gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2021 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,UxUi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode || <MODE>mode == TImode)
  45         {
  46           if (which_alternative == 7)
  47             output_asm_insn ("vstrw.32\t%q1, %E0", operands);
  48           else
  49             output_asm_insn ("vldrw.u32\t%q0, %E1",operands);
  50         }
  51       else
  52         {
  53           if (which_alternative == 7)
  54             output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0", operands);
  55           else
  56             output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1", operands);
  57         }
  58       return "";
  59     }
  60   switch (which_alternative)
  61     {
  62     case 0:
  63       return "vmov\t%q0, %q1";
  64     case 1:
  65       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  66     case 2:
  67       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  68     case 5:
  69       return output_move_quad (operands);
  70     case 8:
  71         return output_move_neon (operands);
  72     default:
  73       gcc_unreachable ();
  74       return "";
  75     }
  76 }
  77   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
  78    (set_attr "length" "4,8,8,4,8,8,4,4,4")
  79    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
  80    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
  81
  82 (define_insn "*mve_vdup<mode>"
  83   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  84         (vec_duplicate:MVE_vecs
  85           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  86   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  87   "vdup.<V_sz_elem>\t%q0, %1"
  88   [(set_attr "length" "4")
  89    (set_attr "type" "mve_move")])
  90
  91 ;;
  92 ;; [vst4q])
  93 ;;
  94 (define_insn "mve_vst4q<mode>"
  95   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
  96         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
  97                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
  98          VST4Q))
  99   ]
 100   "TARGET_HAVE_MVE"
 101 {
 102    rtx ops[6];
 103    int regno = REGNO (operands[1]);
 104    ops[0] = gen_rtx_REG (TImode, regno);
 105    ops[1] = gen_rtx_REG (TImode, regno+4);
 106    ops[2] = gen_rtx_REG (TImode, regno+8);
 107    ops[3] = gen_rtx_REG (TImode, regno+12);
 108    rtx reg  = operands[0];
 109    while (reg && !REG_P (reg))
 110     reg = XEXP (reg, 0);
 111    gcc_assert (REG_P (reg));
 112    ops[4] = reg;
 113    ops[5] = operands[0];
 114    /* Here in first three instructions data is stored to ops[4]'s location but
 115       in the fourth instruction data is stored to operands[0], this is to
 116       support the writeback.  */
 117    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 118                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 119                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 120                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 121    return "";
 122 }
 123   [(set_attr "length" "16")])
 124
 125 ;;
 126 ;; [vrndq_m_f])
 127 ;;
 128 (define_insn "mve_vrndq_m_f<mode>"
 129   [
 130    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 131         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 132                        (match_operand:MVE_0 2 "s_register_operand" "w")
 133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 134          VRNDQ_M_F))
 135   ]
 136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 137   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 138   [(set_attr "type" "mve_move")
 139    (set_attr "length""8")])
 140
 141 ;;
 142 ;; [vrndxq_f])
 143 ;;
 144 (define_insn "mve_vrndxq_f<mode>"
 145   [
 146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 148          VRNDXQ_F))
 149   ]
 150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 151   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 152   [(set_attr "type" "mve_move")
 153 ])
 154
 155 ;;
 156 ;; [vrndq_f])
 157 ;;
 158 (define_insn "mve_vrndq_f<mode>"
 159   [
 160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 162          VRNDQ_F))
 163   ]
 164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 165   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 166   [(set_attr "type" "mve_move")
 167 ])
 168
 169 ;;
 170 ;; [vrndpq_f])
 171 ;;
 172 (define_insn "mve_vrndpq_f<mode>"
 173   [
 174    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 175         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 176          VRNDPQ_F))
 177   ]
 178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 179   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 180   [(set_attr "type" "mve_move")
 181 ])
 182
 183 ;;
 184 ;; [vrndnq_f])
 185 ;;
 186 (define_insn "mve_vrndnq_f<mode>"
 187   [
 188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 190          VRNDNQ_F))
 191   ]
 192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 193   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 194   [(set_attr "type" "mve_move")
 195 ])
 196
 197 ;;
 198 ;; [vrndmq_f])
 199 ;;
 200 (define_insn "mve_vrndmq_f<mode>"
 201   [
 202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 204          VRNDMQ_F))
 205   ]
 206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 207   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 208   [(set_attr "type" "mve_move")
 209 ])
 210
 211 ;;
 212 ;; [vrndaq_f])
 213 ;;
 214 (define_insn "mve_vrndaq_f<mode>"
 215   [
 216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 217         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 218          VRNDAQ_F))
 219   ]
 220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 221   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 222   [(set_attr "type" "mve_move")
 223 ])
 224
 225 ;;
 226 ;; [vrev64q_f])
 227 ;;
 228 (define_insn "mve_vrev64q_f<mode>"
 229   [
 230    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 232          VREV64Q_F))
 233   ]
 234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 235   "vrev64.%#<V_sz_elem> %q0, %q1"
 236   [(set_attr "type" "mve_move")
 237 ])
 238
 239 ;;
 240 ;; [vnegq_f])
 241 ;;
 242 (define_insn "mve_vnegq_f<mode>"
 243   [
 244    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 245         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 246   ]
 247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 248   "vneg.f%#<V_sz_elem>  %q0, %q1"
 249   [(set_attr "type" "mve_move")
 250 ])
 251
 252 ;;
 253 ;; [vdupq_n_f])
 254 ;;
 255 (define_insn "mve_vdupq_n_f<mode>"
 256   [
 257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 258         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 259          VDUPQ_N_F))
 260   ]
 261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 262   "vdup.%#<V_sz_elem>   %q0, %1"
 263   [(set_attr "type" "mve_move")
 264 ])
 265
 266 ;;
 267 ;; [vabsq_f])
 268 ;;
 269 (define_insn "mve_vabsq_f<mode>"
 270   [
 271    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 272         (abs:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 273   ]
 274   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 275   "vabs.f%#<V_sz_elem>  %q0, %q1"
 276   [(set_attr "type" "mve_move")
 277 ])
 278
 279 ;;
 280 ;; [vrev32q_f])
 281 ;;
 282 (define_insn "mve_vrev32q_fv8hf"
 283   [
 284    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 285         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 286          VREV32Q_F))
 287   ]
 288   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 289   "vrev32.16 %q0, %q1"
 290   [(set_attr "type" "mve_move")
 291 ])
 292 ;;
 293 ;; [vcvttq_f32_f16])
 294 ;;
 295 (define_insn "mve_vcvttq_f32_f16v4sf"
 296   [
 297    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 298         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 299          VCVTTQ_F32_F16))
 300   ]
 301   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 302   "vcvtt.f32.f16 %q0, %q1"
 303   [(set_attr "type" "mve_move")
 304 ])
 305
 306 ;;
 307 ;; [vcvtbq_f32_f16])
 308 ;;
 309 (define_insn "mve_vcvtbq_f32_f16v4sf"
 310   [
 311    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 312         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 313          VCVTBQ_F32_F16))
 314   ]
 315   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 316   "vcvtb.f32.f16 %q0, %q1"
 317   [(set_attr "type" "mve_move")
 318 ])
 319
 320 ;;
 321 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 322 ;;
 323 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 324   [
 325    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 326         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 327          VCVTQ_TO_F))
 328   ]
 329   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 330   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 331   [(set_attr "type" "mve_move")
 332 ])
 333
 334 ;;
 335 ;; [vrev64q_u, vrev64q_s])
 336 ;;
 337 (define_insn "mve_vrev64q_<supf><mode>"
 338   [
 339    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 340         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 341          VREV64Q))
 342   ]
 343   "TARGET_HAVE_MVE"
 344   "vrev64.%#<V_sz_elem> %q0, %q1"
 345   [(set_attr "type" "mve_move")
 346 ])
 347
 348 ;;
 349 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 350 ;;
 351 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 352   [
 353    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 354         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 355          VCVTQ_FROM_F))
 356   ]
 357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 358   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 359   [(set_attr "type" "mve_move")
 360 ])
 361 ;; [vqnegq_s])
 362 ;;
 363 (define_insn "mve_vqnegq_s<mode>"
 364   [
 365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 367          VQNEGQ_S))
 368   ]
 369   "TARGET_HAVE_MVE"
 370   "vqneg.s%#<V_sz_elem> %q0, %q1"
 371   [(set_attr "type" "mve_move")
 372 ])
 373
 374 ;;
 375 ;; [vqabsq_s])
 376 ;;
 377 (define_insn "mve_vqabsq_s<mode>"
 378   [
 379    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 380         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 381          VQABSQ_S))
 382   ]
 383   "TARGET_HAVE_MVE"
 384   "vqabs.s%#<V_sz_elem> %q0, %q1"
 385   [(set_attr "type" "mve_move")
 386 ])
 387
 388 ;;
 389 ;; [vnegq_s])
 390 ;;
 391 (define_insn "mve_vnegq_s<mode>"
 392   [
 393    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 394         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 395   ]
 396   "TARGET_HAVE_MVE"
 397   "vneg.s%#<V_sz_elem>  %q0, %q1"
 398   [(set_attr "type" "mve_move")
 399 ])
 400
 401 ;;
 402 ;; [vmvnq_u, vmvnq_s])
 403 ;;
 404 (define_insn "mve_vmvnq_u<mode>"
 405   [
 406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 407         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 408   ]
 409   "TARGET_HAVE_MVE"
 410   "vmvn\t%q0, %q1"
 411   [(set_attr "type" "mve_move")
 412 ])
 413 (define_expand "mve_vmvnq_s<mode>"
 414   [
 415    (set (match_operand:MVE_2 0 "s_register_operand")
 416         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 417   ]
 418   "TARGET_HAVE_MVE"
 419 )
 420
 421 ;;
 422 ;; [vdupq_n_u, vdupq_n_s])
 423 ;;
 424 (define_insn "mve_vdupq_n_<supf><mode>"
 425   [
 426    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 427         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 428          VDUPQ_N))
 429   ]
 430   "TARGET_HAVE_MVE"
 431   "vdup.%#<V_sz_elem>   %q0, %1"
 432   [(set_attr "type" "mve_move")
 433 ])
 434
 435 ;;
 436 ;; [vclzq_u, vclzq_s])
 437 ;;
 438 (define_insn "@mve_vclzq_s<mode>"
 439   [
 440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 441         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 442   ]
 443   "TARGET_HAVE_MVE"
 444   "vclz.i%#<V_sz_elem>  %q0, %q1"
 445   [(set_attr "type" "mve_move")
 446 ])
 447 (define_expand "mve_vclzq_u<mode>"
 448   [
 449    (set (match_operand:MVE_2 0 "s_register_operand")
 450         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 451   ]
 452   "TARGET_HAVE_MVE"
 453 )
 454
 455 ;;
 456 ;; [vclsq_s])
 457 ;;
 458 (define_insn "mve_vclsq_s<mode>"
 459   [
 460    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 461         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 462          VCLSQ_S))
 463   ]
 464   "TARGET_HAVE_MVE"
 465   "vcls.s%#<V_sz_elem>  %q0, %q1"
 466   [(set_attr "type" "mve_move")
 467 ])
 468
 469 ;;
 470 ;; [vaddvq_s, vaddvq_u])
 471 ;;
 472 (define_insn "@mve_vaddvq_<supf><mode>"
 473   [
 474    (set (match_operand:SI 0 "s_register_operand" "=Te")
 475         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 476          VADDVQ))
 477   ]
 478   "TARGET_HAVE_MVE"
 479   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 480   [(set_attr "type" "mve_move")
 481 ])
 482
 483 ;;
 484 ;; [vabsq_s])
 485 ;;
 486 (define_insn "mve_vabsq_s<mode>"
 487   [
 488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 489         (abs:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 490   ]
 491   "TARGET_HAVE_MVE"
 492   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 493   [(set_attr "type" "mve_move")
 494 ])
 495
 496 ;;
 497 ;; [vrev32q_u, vrev32q_s])
 498 ;;
 499 (define_insn "mve_vrev32q_<supf><mode>"
 500   [
 501    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 502         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 503          VREV32Q))
 504   ]
 505   "TARGET_HAVE_MVE"
 506   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 507   [(set_attr "type" "mve_move")
 508 ])
 509
 510 ;;
 511 ;; [vmovltq_u, vmovltq_s])
 512 ;;
 513 (define_insn "mve_vmovltq_<supf><mode>"
 514   [
 515    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 516         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 517          VMOVLTQ))
 518   ]
 519   "TARGET_HAVE_MVE"
 520   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 521   [(set_attr "type" "mve_move")
 522 ])
 523
 524 ;;
 525 ;; [vmovlbq_s, vmovlbq_u])
 526 ;;
 527 (define_insn "mve_vmovlbq_<supf><mode>"
 528   [
 529    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 530         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 531          VMOVLBQ))
 532   ]
 533   "TARGET_HAVE_MVE"
 534   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 535   [(set_attr "type" "mve_move")
 536 ])
 537
 538 ;;
 539 ;; [vcvtpq_s, vcvtpq_u])
 540 ;;
 541 (define_insn "mve_vcvtpq_<supf><mode>"
 542   [
 543    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 544         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 545          VCVTPQ))
 546   ]
 547   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 548   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 549   [(set_attr "type" "mve_move")
 550 ])
 551
 552 ;;
 553 ;; [vcvtnq_s, vcvtnq_u])
 554 ;;
 555 (define_insn "mve_vcvtnq_<supf><mode>"
 556   [
 557    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 558         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 559          VCVTNQ))
 560   ]
 561   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 562   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 563   [(set_attr "type" "mve_move")
 564 ])
 565
 566 ;;
 567 ;; [vcvtmq_s, vcvtmq_u])
 568 ;;
 569 (define_insn "mve_vcvtmq_<supf><mode>"
 570   [
 571    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 572         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 573          VCVTMQ))
 574   ]
 575   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 576   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 577   [(set_attr "type" "mve_move")
 578 ])
 579
 580 ;;
 581 ;; [vcvtaq_u, vcvtaq_s])
 582 ;;
 583 (define_insn "mve_vcvtaq_<supf><mode>"
 584   [
 585    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 586         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 587          VCVTAQ))
 588   ]
 589   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 590   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 591   [(set_attr "type" "mve_move")
 592 ])
 593
 594 ;;
 595 ;; [vmvnq_n_u, vmvnq_n_s])
 596 ;;
 597 (define_insn "mve_vmvnq_n_<supf><mode>"
 598   [
 599    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 600         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 601          VMVNQ_N))
 602   ]
 603   "TARGET_HAVE_MVE"
 604   "vmvn.i%#<V_sz_elem>  %q0, %1"
 605   [(set_attr "type" "mve_move")
 606 ])
 607
 608 ;;
 609 ;; [vrev16q_u, vrev16q_s])
 610 ;;
 611 (define_insn "mve_vrev16q_<supf>v16qi"
 612   [
 613    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 614         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 615          VREV16Q))
 616   ]
 617   "TARGET_HAVE_MVE"
 618   "vrev16.8 %q0, %q1"
 619   [(set_attr "type" "mve_move")
 620 ])
 621
 622 ;;
 623 ;; [vaddlvq_s vaddlvq_u])
 624 ;;
 625 (define_insn "mve_vaddlvq_<supf>v4si"
 626   [
 627    (set (match_operand:DI 0 "s_register_operand" "=r")
 628         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 629          VADDLVQ))
 630   ]
 631   "TARGET_HAVE_MVE"
 632   "vaddlv.<supf>32 %Q0, %R0, %q1"
 633   [(set_attr "type" "mve_move")
 634 ])
 635
 636 ;;
 637 ;; [vctp8q vctp16q vctp32q vctp64q])
 638 ;;
 639 (define_insn "mve_vctp<mode1>qhi"
 640   [
 641    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 642         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 643         VCTPQ))
 644   ]
 645   "TARGET_HAVE_MVE"
 646   "vctp.<mode1> %1"
 647   [(set_attr "type" "mve_move")
 648 ])
 649
 650 ;;
 651 ;; [vpnot])
 652 ;;
 653 (define_insn "mve_vpnothi"
 654   [
 655    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 656         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 657          VPNOT))
 658   ]
 659   "TARGET_HAVE_MVE"
 660   "vpnot"
 661   [(set_attr "type" "mve_move")
 662 ])
 663
 664 ;;
 665 ;; [vsubq_n_f])
 666 ;;
 667 (define_insn "mve_vsubq_n_f<mode>"
 668   [
 669    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 670         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 671                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 672          VSUBQ_N_F))
 673   ]
 674   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 675   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 676   [(set_attr "type" "mve_move")
 677 ])
 678
 679 ;;
 680 ;; [vbrsrq_n_f])
 681 ;;
 682 (define_insn "mve_vbrsrq_n_f<mode>"
 683   [
 684    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 685         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 686                        (match_operand:SI 2 "s_register_operand" "r")]
 687          VBRSRQ_N_F))
 688   ]
 689   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 690   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 691   [(set_attr "type" "mve_move")
 692 ])
 693
 694 ;;
 695 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 696 ;;
 697 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 698   [
 699    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 700         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 701                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 702          VCVTQ_N_TO_F))
 703   ]
 704   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 705   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 706   [(set_attr "type" "mve_move")
 707 ])
 708
 709 ;; [vcreateq_f])
 710 ;;
 711 (define_insn "mve_vcreateq_f<mode>"
 712   [
 713    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 714         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 715                        (match_operand:DI 2 "s_register_operand" "r")]
 716          VCREATEQ_F))
 717   ]
 718   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 719   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 720   [(set_attr "type" "mve_move")
 721    (set_attr "length""8")])
 722
 723 ;;
 724 ;; [vcreateq_u, vcreateq_s])
 725 ;;
 726 (define_insn "mve_vcreateq_<supf><mode>"
 727   [
 728    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 729         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 730                        (match_operand:DI 2 "s_register_operand" "r")]
 731          VCREATEQ))
 732   ]
 733   "TARGET_HAVE_MVE"
 734   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 735   [(set_attr "type" "mve_move")
 736    (set_attr "length""8")])
 737
 738 ;;
 739 ;; [vshrq_n_s, vshrq_n_u])
 740 ;;
 741 ;; Version that takes an immediate as operand 2.
 742 (define_insn "mve_vshrq_n_<supf><mode>"
 743   [
 744    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 745         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 746                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 747          VSHRQ_N))
 748   ]
 749   "TARGET_HAVE_MVE"
 750   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 751   [(set_attr "type" "mve_move")
 752 ])
 753
 754 ;; Versions that take constant vectors as operand 2 (with all elements
 755 ;; equal).
 756 (define_insn "mve_vshrq_n_s<mode>_imm"
 757   [
 758    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 759         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 760                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 761   ]
 762   "TARGET_HAVE_MVE"
 763   {
 764     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 765                                         <MODE>mode,
 766                                         VALID_NEON_QREG_MODE (<MODE>mode),
 767                                         true);
 768   }
 769   [(set_attr "type" "mve_move")
 770 ])
 771 (define_insn "mve_vshrq_n_u<mode>_imm"
 772   [
 773    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 774         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 775                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 776   ]
 777   "TARGET_HAVE_MVE"
 778   {
 779     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 780                                         <MODE>mode,
 781                                         VALID_NEON_QREG_MODE (<MODE>mode),
 782                                         true);
 783   }
 784   [(set_attr "type" "mve_move")
 785 ])
 786
 787 ;;
 788 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 789 ;;
 790 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 791   [
 792    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 793         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 794                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 795          VCVTQ_N_FROM_F))
 796   ]
 797   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 798   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 799   [(set_attr "type" "mve_move")
 800 ])
 801
 802 ;;
 803 ;; [vaddlvq_p_s])
 804 ;;
 805 (define_insn "mve_vaddlvq_p_<supf>v4si"
 806   [
 807    (set (match_operand:DI 0 "s_register_operand" "=r")
 808         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 809                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 810          VADDLVQ_P))
 811   ]
 812   "TARGET_HAVE_MVE"
 813   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 814   [(set_attr "type" "mve_move")
 815    (set_attr "length""8")])
 816
 817 ;;
 818 ;; [vcmpneq_, vcmpcsq_, vcmpeqq_, vcmpgeq_, vcmpgtq_, vcmphiq_, vcmpleq_, vcmpltq_])
 819 ;;
 820 (define_insn "@mve_vcmp<mve_cmp_op>q_<mode>"
 821   [
 822    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 823         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 824                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 825   ]
 826   "TARGET_HAVE_MVE"
 827   "vcmp.<mve_cmp_type>%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
 828   [(set_attr "type" "mve_move")
 829 ])
 830
 831 ;;
 832 ;; [vcmpcsq_n_, vcmpeqq_n_, vcmpgeq_n_, vcmpgtq_n_, vcmphiq_n_, vcmpleq_n_, vcmpltq_n_, vcmpneq_n_])
 833 ;;
 834 (define_insn "mve_vcmp<mve_cmp_op>q_n_<mode>"
 835   [
 836    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 837         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 838                     (match_operand:<V_elem> 2 "s_register_operand" "r")))
 839   ]
 840   "TARGET_HAVE_MVE"
 841   "vcmp.<mve_cmp_type>%#<V_sz_elem>     <mve_cmp_op>, %q1, %2"
 842   [(set_attr "type" "mve_move")
 843 ])
 844
 845 ;;
 846 ;; [vshlq_s, vshlq_u])
 847 ;; See vec-common.md
 848
 849 ;;
 850 ;; [vabdq_s, vabdq_u])
 851 ;;
 852 (define_insn "mve_vabdq_<supf><mode>"
 853   [
 854    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 855         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 856                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 857          VABDQ))
 858   ]
 859   "TARGET_HAVE_MVE"
 860   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 861   [(set_attr "type" "mve_move")
 862 ])
 863
 864 ;;
 865 ;; [vaddq_n_s, vaddq_n_u])
 866 ;;
 867 (define_insn "mve_vaddq_n_<supf><mode>"
 868   [
 869    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 870         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 871                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 872          VADDQ_N))
 873   ]
 874   "TARGET_HAVE_MVE"
 875   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 876   [(set_attr "type" "mve_move")
 877 ])
 878
 879 ;;
 880 ;; [vaddvaq_s, vaddvaq_u])
 881 ;;
 882 (define_insn "mve_vaddvaq_<supf><mode>"
 883   [
 884    (set (match_operand:SI 0 "s_register_operand" "=Te")
 885         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 886                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 887          VADDVAQ))
 888   ]
 889   "TARGET_HAVE_MVE"
 890   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 891   [(set_attr "type" "mve_move")
 892 ])
 893
 894 ;;
 895 ;; [vaddvq_p_u, vaddvq_p_s])
 896 ;;
 897 (define_insn "mve_vaddvq_p_<supf><mode>"
 898   [
 899    (set (match_operand:SI 0 "s_register_operand" "=Te")
 900         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 901                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 902          VADDVQ_P))
 903   ]
 904   "TARGET_HAVE_MVE"
 905   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 906   [(set_attr "type" "mve_move")
 907    (set_attr "length""8")])
 908
 909 ;;
 910 ;; [vandq_u, vandq_s])
 911 ;;
 912 ;; signed and unsigned versions are the same: define the unsigned
 913 ;; insn, and use an expander for the signed one as we still reference
 914 ;; both names from arm_mve.h.
 915 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 916 (define_insn "mve_vandq_u<mode>"
 917   [
 918    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 919         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 920                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 921   ]
 922   "TARGET_HAVE_MVE"
 923   "@
 924    vand\t%q0, %q1, %q2
 925    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 926   [(set_attr "type" "mve_move")
 927 ])
 928 (define_expand "mve_vandq_s<mode>"
 929   [
 930    (set (match_operand:MVE_2 0 "s_register_operand")
 931         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 932                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 933   ]
 934   "TARGET_HAVE_MVE"
 935 )
 936
 937 ;;
 938 ;; [vbicq_s, vbicq_u])
 939 ;;
 940 (define_insn "mve_vbicq_u<mode>"
 941   [
 942    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 943         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 944                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 945   ]
 946   "TARGET_HAVE_MVE"
 947   "vbic\t%q0, %q1, %q2"
 948   [(set_attr "type" "mve_move")
 949 ])
 950
 951 (define_expand "mve_vbicq_s<mode>"
 952   [
 953    (set (match_operand:MVE_2 0 "s_register_operand")
 954         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 955                    (match_operand:MVE_2 1 "s_register_operand")))
 956   ]
 957   "TARGET_HAVE_MVE"
 958 )
 959
 960 ;;
 961 ;; [vbrsrq_n_u, vbrsrq_n_s])
 962 ;;
 963 (define_insn "mve_vbrsrq_n_<supf><mode>"
 964   [
 965    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 966         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 967                        (match_operand:SI 2 "s_register_operand" "r")]
 968          VBRSRQ_N))
 969   ]
 970   "TARGET_HAVE_MVE"
 971   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 972   [(set_attr "type" "mve_move")
 973 ])
 974
 975 ;;
 976 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 977 ;;
 978 (define_insn "mve_vcaddq<mve_rot><mode>"
 979   [
 980    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 981         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 982                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 983          VCADD))
 984   ]
 985   "TARGET_HAVE_MVE"
 986   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 987   [(set_attr "type" "mve_move")
 988 ])
 989
 990 ;; Auto vectorizer pattern for int vcadd
 991 (define_expand "cadd<rot><mode>3"
 992   [(set (match_operand:MVE_2 0 "register_operand")
 993         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
 994                        (match_operand:MVE_2 2 "register_operand")]
 995           VCADD))]
 996   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
 997 )
 998
 999 ;;
1000 ;; [veorq_u, veorq_s])
1001 ;;
1002 (define_insn "mve_veorq_u<mode>"
1003   [
1004    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1005         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1006                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1007   ]
1008   "TARGET_HAVE_MVE"
1009   "veor\t%q0, %q1, %q2"
1010   [(set_attr "type" "mve_move")
1011 ])
1012 (define_expand "mve_veorq_s<mode>"
1013   [
1014    (set (match_operand:MVE_2 0 "s_register_operand")
1015         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1016                    (match_operand:MVE_2 2 "s_register_operand")))
1017   ]
1018   "TARGET_HAVE_MVE"
1019 )
1020
1021 ;;
1022 ;; [vhaddq_n_u, vhaddq_n_s])
1023 ;;
1024 (define_insn "mve_vhaddq_n_<supf><mode>"
1025   [
1026    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1027         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1028                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1029          VHADDQ_N))
1030   ]
1031   "TARGET_HAVE_MVE"
1032   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1033   [(set_attr "type" "mve_move")
1034 ])
1035
1036 ;;
1037 ;; [vhaddq_s, vhaddq_u])
1038 ;;
1039 (define_insn "@mve_vhaddq_<supf><mode>"
1040   [
1041    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1042         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1043                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1044          VHADDQ))
1045   ]
1046   "TARGET_HAVE_MVE"
1047   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1048   [(set_attr "type" "mve_move")
1049 ])
1050
1051 ;;
1052 ;; [vhcaddq_rot270_s])
1053 ;;
1054 (define_insn "mve_vhcaddq_rot270_s<mode>"
1055   [
1056    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1057         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1058                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1059          VHCADDQ_ROT270_S))
1060   ]
1061   "TARGET_HAVE_MVE"
1062   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1063   [(set_attr "type" "mve_move")
1064 ])
1065
1066 ;;
1067 ;; [vhcaddq_rot90_s])
1068 ;;
1069 (define_insn "mve_vhcaddq_rot90_s<mode>"
1070   [
1071    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1072         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1073                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1074          VHCADDQ_ROT90_S))
1075   ]
1076   "TARGET_HAVE_MVE"
1077   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1078   [(set_attr "type" "mve_move")
1079 ])
1080
1081 ;;
1082 ;; [vhsubq_n_u, vhsubq_n_s])
1083 ;;
1084 (define_insn "mve_vhsubq_n_<supf><mode>"
1085   [
1086    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1087         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1088                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1089          VHSUBQ_N))
1090   ]
1091   "TARGET_HAVE_MVE"
1092   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1093   [(set_attr "type" "mve_move")
1094 ])
1095
1096 ;;
1097 ;; [vhsubq_s, vhsubq_u])
1098 ;;
1099 (define_insn "mve_vhsubq_<supf><mode>"
1100   [
1101    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1102         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1103                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1104          VHSUBQ))
1105   ]
1106   "TARGET_HAVE_MVE"
1107   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1108   [(set_attr "type" "mve_move")
1109 ])
1110
1111 ;;
1112 ;; [vmaxaq_s])
1113 ;;
1114 (define_insn "mve_vmaxaq_s<mode>"
1115   [
1116    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1117         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1118                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1119          VMAXAQ_S))
1120   ]
1121   "TARGET_HAVE_MVE"
1122   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1123   [(set_attr "type" "mve_move")
1124 ])
1125
1126 ;;
1127 ;; [vmaxavq_s])
1128 ;;
1129 (define_insn "mve_vmaxavq_s<mode>"
1130   [
1131    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1132         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1133                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1134          VMAXAVQ_S))
1135   ]
1136   "TARGET_HAVE_MVE"
1137   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1138   [(set_attr "type" "mve_move")
1139 ])
1140
1141 ;;
1142 ;; [vmaxq_u, vmaxq_s])
1143 ;;
1144 (define_insn "mve_vmaxq_s<mode>"
1145   [
1146    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1147         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1148                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1149   ]
1150   "TARGET_HAVE_MVE"
1151   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1152   [(set_attr "type" "mve_move")
1153 ])
1154
1155 (define_insn "mve_vmaxq_u<mode>"
1156   [
1157    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1158         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1159                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1160   ]
1161   "TARGET_HAVE_MVE"
1162   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1163   [(set_attr "type" "mve_move")
1164 ])
1165
1166 ;;
1167 ;; [vmaxvq_u, vmaxvq_s])
1168 ;;
1169 (define_insn "mve_vmaxvq_<supf><mode>"
1170   [
1171    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1172         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1173                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1174          VMAXVQ))
1175   ]
1176   "TARGET_HAVE_MVE"
1177   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1178   [(set_attr "type" "mve_move")
1179 ])
1180
1181 ;;
1182 ;; [vminaq_s])
1183 ;;
1184 (define_insn "mve_vminaq_s<mode>"
1185   [
1186    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1187         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1188                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1189          VMINAQ_S))
1190   ]
1191   "TARGET_HAVE_MVE"
1192   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1193   [(set_attr "type" "mve_move")
1194 ])
1195
1196 ;;
1197 ;; [vminavq_s])
1198 ;;
1199 (define_insn "mve_vminavq_s<mode>"
1200   [
1201    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1202         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1203                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1204          VMINAVQ_S))
1205   ]
1206   "TARGET_HAVE_MVE"
1207   "vminav.s%#<V_sz_elem>\t%0, %q2"
1208   [(set_attr "type" "mve_move")
1209 ])
1210
1211 ;;
1212 ;; [vminq_s, vminq_u])
1213 ;;
1214 (define_insn "mve_vminq_s<mode>"
1215   [
1216    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1217         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1218                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1219   ]
1220   "TARGET_HAVE_MVE"
1221   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1222   [(set_attr "type" "mve_move")
1223 ])
1224
1225 (define_insn "mve_vminq_u<mode>"
1226   [
1227    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1228         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1229                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1230   ]
1231   "TARGET_HAVE_MVE"
1232   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1233   [(set_attr "type" "mve_move")
1234 ])
1235
1236 ;;
1237 ;; [vminvq_u, vminvq_s])
1238 ;;
1239 (define_insn "mve_vminvq_<supf><mode>"
1240   [
1241    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1242         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1243                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1244          VMINVQ))
1245   ]
1246   "TARGET_HAVE_MVE"
1247   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1248   [(set_attr "type" "mve_move")
1249 ])
1250
1251 ;;
1252 ;; [vmladavq_u, vmladavq_s])
1253 ;;
1254 (define_insn "mve_vmladavq_<supf><mode>"
1255   [
1256    (set (match_operand:SI 0 "s_register_operand" "=Te")
1257         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1258                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1259          VMLADAVQ))
1260   ]
1261   "TARGET_HAVE_MVE"
1262   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1263   [(set_attr "type" "mve_move")
1264 ])
1265
1266 ;;
1267 ;; [vmladavxq_s])
1268 ;;
1269 (define_insn "mve_vmladavxq_s<mode>"
1270   [
1271    (set (match_operand:SI 0 "s_register_operand" "=Te")
1272         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1273                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1274          VMLADAVXQ_S))
1275   ]
1276   "TARGET_HAVE_MVE"
1277   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1278   [(set_attr "type" "mve_move")
1279 ])
1280
1281 ;;
1282 ;; [vmlsdavq_s])
1283 ;;
1284 (define_insn "mve_vmlsdavq_s<mode>"
1285   [
1286    (set (match_operand:SI 0 "s_register_operand" "=Te")
1287         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1288                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1289          VMLSDAVQ_S))
1290   ]
1291   "TARGET_HAVE_MVE"
1292   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1293   [(set_attr "type" "mve_move")
1294 ])
1295
1296 ;;
1297 ;; [vmlsdavxq_s])
1298 ;;
1299 (define_insn "mve_vmlsdavxq_s<mode>"
1300   [
1301    (set (match_operand:SI 0 "s_register_operand" "=Te")
1302         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1303                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1304          VMLSDAVXQ_S))
1305   ]
1306   "TARGET_HAVE_MVE"
1307   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1308   [(set_attr "type" "mve_move")
1309 ])
1310
1311 ;;
1312 ;; [vmulhq_s, vmulhq_u])
1313 ;;
1314 (define_insn "mve_vmulhq_<supf><mode>"
1315   [
1316    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1317         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1318                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1319          VMULHQ))
1320   ]
1321   "TARGET_HAVE_MVE"
1322   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1323   [(set_attr "type" "mve_move")
1324 ])
1325
1326 ;;
1327 ;; [vmullbq_int_u, vmullbq_int_s])
1328 ;;
1329 (define_insn "mve_vmullbq_int_<supf><mode>"
1330   [
1331    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1332         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1333                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1334          VMULLBQ_INT))
1335   ]
1336   "TARGET_HAVE_MVE"
1337   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1338   [(set_attr "type" "mve_move")
1339 ])
1340
1341 ;;
1342 ;; [vmulltq_int_u, vmulltq_int_s])
1343 ;;
1344 (define_insn "mve_vmulltq_int_<supf><mode>"
1345   [
1346    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1347         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1348                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1349          VMULLTQ_INT))
1350   ]
1351   "TARGET_HAVE_MVE"
1352   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1353   [(set_attr "type" "mve_move")
1354 ])
1355
1356 ;;
1357 ;; [vmulq_n_u, vmulq_n_s])
1358 ;;
1359 (define_insn "mve_vmulq_n_<supf><mode>"
1360   [
1361    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1362         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1363                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1364          VMULQ_N))
1365   ]
1366   "TARGET_HAVE_MVE"
1367   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1368   [(set_attr "type" "mve_move")
1369 ])
1370
1371 ;;
1372 ;; [vmulq_u, vmulq_s])
1373 ;;
1374 (define_insn "mve_vmulq_<supf><mode>"
1375   [
1376    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1377         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1378                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1379          VMULQ))
1380   ]
1381   "TARGET_HAVE_MVE"
1382   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1383   [(set_attr "type" "mve_move")
1384 ])
1385
1386 (define_insn "mve_vmulq<mode>"
1387   [
1388    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1389         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1390                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1391   ]
1392   "TARGET_HAVE_MVE"
1393   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1394   [(set_attr "type" "mve_move")
1395 ])
1396
1397 ;;
1398 ;; [vornq_u, vornq_s])
1399 ;;
1400 (define_insn "mve_vornq_s<mode>"
1401   [
1402    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1403         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1404                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1405   ]
1406   "TARGET_HAVE_MVE"
1407    "vorn\t%q0, %q1, %q2"
1408   [(set_attr "type" "mve_move")
1409 ])
1410
1411 (define_expand "mve_vornq_u<mode>"
1412   [
1413    (set (match_operand:MVE_2 0 "s_register_operand")
1414         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1415                    (match_operand:MVE_2 1 "s_register_operand")))
1416   ]
1417   "TARGET_HAVE_MVE"
1418 )
1419
1420 ;;
1421 ;; [vorrq_s, vorrq_u])
1422 ;;
1423 ;; signed and unsigned versions are the same: define the unsigned
1424 ;; insn, and use an expander for the signed one as we still reference
1425 ;; both names from arm_mve.h.
1426 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1427 (define_insn "mve_vorrq_s<mode>"
1428   [
1429    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1430         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1431                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1432   ]
1433   "TARGET_HAVE_MVE"
1434   "@
1435    vorr\t%q0, %q1, %q2
1436    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1437   [(set_attr "type" "mve_move")
1438 ])
1439 (define_expand "mve_vorrq_u<mode>"
1440   [
1441    (set (match_operand:MVE_2 0 "s_register_operand")
1442         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1443                    (match_operand:MVE_2 2 "neon_logic_op2")))
1444   ]
1445   "TARGET_HAVE_MVE"
1446 )
1447
1448 ;;
1449 ;; [vqaddq_n_s, vqaddq_n_u])
1450 ;;
1451 (define_insn "mve_vqaddq_n_<supf><mode>"
1452   [
1453    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1454         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1455                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1456          VQADDQ_N))
1457   ]
1458   "TARGET_HAVE_MVE"
1459   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1460   [(set_attr "type" "mve_move")
1461 ])
1462
1463 ;;
1464 ;; [vqaddq_u, vqaddq_s])
1465 ;;
1466 (define_insn "mve_vqaddq_<supf><mode>"
1467   [
1468    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1469         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1470                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1471          VQADDQ))
1472   ]
1473   "TARGET_HAVE_MVE"
1474   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1475   [(set_attr "type" "mve_move")
1476 ])
1477
1478 ;;
1479 ;; [vqdmulhq_n_s])
1480 ;;
1481 (define_insn "mve_vqdmulhq_n_s<mode>"
1482   [
1483    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1484         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1485                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1486          VQDMULHQ_N_S))
1487   ]
1488   "TARGET_HAVE_MVE"
1489   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1490   [(set_attr "type" "mve_move")
1491 ])
1492
1493 ;;
1494 ;; [vqdmulhq_s])
1495 ;;
1496 (define_insn "mve_vqdmulhq_s<mode>"
1497   [
1498    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1499         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1500                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1501          VQDMULHQ_S))
1502   ]
1503   "TARGET_HAVE_MVE"
1504   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1505   [(set_attr "type" "mve_move")
1506 ])
1507
1508 ;;
1509 ;; [vqrdmulhq_n_s])
1510 ;;
1511 (define_insn "mve_vqrdmulhq_n_s<mode>"
1512   [
1513    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1514         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1515                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1516          VQRDMULHQ_N_S))
1517   ]
1518   "TARGET_HAVE_MVE"
1519   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1520   [(set_attr "type" "mve_move")
1521 ])
1522
1523 ;;
1524 ;; [vqrdmulhq_s])
1525 ;;
1526 (define_insn "mve_vqrdmulhq_s<mode>"
1527   [
1528    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1529         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1530                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1531          VQRDMULHQ_S))
1532   ]
1533   "TARGET_HAVE_MVE"
1534   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1535   [(set_attr "type" "mve_move")
1536 ])
1537
1538 ;;
1539 ;; [vqrshlq_n_s, vqrshlq_n_u])
1540 ;;
1541 (define_insn "mve_vqrshlq_n_<supf><mode>"
1542   [
1543    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1544         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1545                        (match_operand:SI 2 "s_register_operand" "r")]
1546          VQRSHLQ_N))
1547   ]
1548   "TARGET_HAVE_MVE"
1549   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1550   [(set_attr "type" "mve_move")
1551 ])
1552
1553 ;;
1554 ;; [vqrshlq_s, vqrshlq_u])
1555 ;;
1556 (define_insn "mve_vqrshlq_<supf><mode>"
1557   [
1558    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1559         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1560                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1561          VQRSHLQ))
1562   ]
1563   "TARGET_HAVE_MVE"
1564   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1565   [(set_attr "type" "mve_move")
1566 ])
1567
1568 ;;
1569 ;; [vqshlq_n_s, vqshlq_n_u])
1570 ;;
1571 (define_insn "mve_vqshlq_n_<supf><mode>"
1572   [
1573    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1574         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1575                        (match_operand:SI 2 "immediate_operand" "i")]
1576          VQSHLQ_N))
1577   ]
1578   "TARGET_HAVE_MVE"
1579   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1580   [(set_attr "type" "mve_move")
1581 ])
1582
1583 ;;
1584 ;; [vqshlq_r_u, vqshlq_r_s])
1585 ;;
1586 (define_insn "mve_vqshlq_r_<supf><mode>"
1587   [
1588    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1589         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1590                        (match_operand:SI 2 "s_register_operand" "r")]
1591          VQSHLQ_R))
1592   ]
1593   "TARGET_HAVE_MVE"
1594   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1595   [(set_attr "type" "mve_move")
1596 ])
1597
1598 ;;
1599 ;; [vqshlq_s, vqshlq_u])
1600 ;;
1601 (define_insn "mve_vqshlq_<supf><mode>"
1602   [
1603    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1604         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1605                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1606          VQSHLQ))
1607   ]
1608   "TARGET_HAVE_MVE"
1609   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1610   [(set_attr "type" "mve_move")
1611 ])
1612
1613 ;;
1614 ;; [vqshluq_n_s])
1615 ;;
1616 (define_insn "mve_vqshluq_n_s<mode>"
1617   [
1618    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1619         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1620                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1621          VQSHLUQ_N_S))
1622   ]
1623   "TARGET_HAVE_MVE"
1624   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1625   [(set_attr "type" "mve_move")
1626 ])
1627
1628 ;;
1629 ;; [vqsubq_n_s, vqsubq_n_u])
1630 ;;
1631 (define_insn "mve_vqsubq_n_<supf><mode>"
1632   [
1633    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1634         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1635                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1636          VQSUBQ_N))
1637   ]
1638   "TARGET_HAVE_MVE"
1639   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1640   [(set_attr "type" "mve_move")
1641 ])
1642
1643 ;;
1644 ;; [vqsubq_u, vqsubq_s])
1645 ;;
1646 (define_insn "mve_vqsubq_<supf><mode>"
1647   [
1648    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1649         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1650                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1651          VQSUBQ))
1652   ]
1653   "TARGET_HAVE_MVE"
1654   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1655   [(set_attr "type" "mve_move")
1656 ])
1657
1658 ;;
1659 ;; [vrhaddq_s, vrhaddq_u])
1660 ;;
1661 (define_insn "@mve_vrhaddq_<supf><mode>"
1662   [
1663    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1664         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1665                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1666          VRHADDQ))
1667   ]
1668   "TARGET_HAVE_MVE"
1669   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1670   [(set_attr "type" "mve_move")
1671 ])
1672
1673 ;;
1674 ;; [vrmulhq_s, vrmulhq_u])
1675 ;;
1676 (define_insn "mve_vrmulhq_<supf><mode>"
1677   [
1678    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1679         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1680                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1681          VRMULHQ))
1682   ]
1683   "TARGET_HAVE_MVE"
1684   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1685   [(set_attr "type" "mve_move")
1686 ])
1687
1688 ;;
1689 ;; [vrshlq_n_u, vrshlq_n_s])
1690 ;;
1691 (define_insn "mve_vrshlq_n_<supf><mode>"
1692   [
1693    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1694         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1695                        (match_operand:SI 2 "s_register_operand" "r")]
1696          VRSHLQ_N))
1697   ]
1698   "TARGET_HAVE_MVE"
1699   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1700   [(set_attr "type" "mve_move")
1701 ])
1702
1703 ;;
1704 ;; [vrshlq_s, vrshlq_u])
1705 ;;
1706 (define_insn "mve_vrshlq_<supf><mode>"
1707   [
1708    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1709         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1710                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1711          VRSHLQ))
1712   ]
1713   "TARGET_HAVE_MVE"
1714   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1715   [(set_attr "type" "mve_move")
1716 ])
1717
1718 ;;
1719 ;; [vrshrq_n_s, vrshrq_n_u])
1720 ;;
1721 (define_insn "mve_vrshrq_n_<supf><mode>"
1722   [
1723    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1724         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1725                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1726          VRSHRQ_N))
1727   ]
1728   "TARGET_HAVE_MVE"
1729   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1730   [(set_attr "type" "mve_move")
1731 ])
1732
1733 ;;
1734 ;; [vshlq_n_u, vshlq_n_s])
1735 ;;
1736 (define_insn "mve_vshlq_n_<supf><mode>"
1737   [
1738    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1739         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1740                        (match_operand:SI 2 "immediate_operand" "i")]
1741          VSHLQ_N))
1742   ]
1743   "TARGET_HAVE_MVE"
1744   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1745   [(set_attr "type" "mve_move")
1746 ])
1747
1748 ;;
1749 ;; [vshlq_r_s, vshlq_r_u])
1750 ;;
1751 (define_insn "mve_vshlq_r_<supf><mode>"
1752   [
1753    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1754         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1755                        (match_operand:SI 2 "s_register_operand" "r")]
1756          VSHLQ_R))
1757   ]
1758   "TARGET_HAVE_MVE"
1759   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1760   [(set_attr "type" "mve_move")
1761 ])
1762
1763 ;;
1764 ;; [vsubq_n_s, vsubq_n_u])
1765 ;;
1766 (define_insn "mve_vsubq_n_<supf><mode>"
1767   [
1768    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1769         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1770                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1771          VSUBQ_N))
1772   ]
1773   "TARGET_HAVE_MVE"
1774   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1775   [(set_attr "type" "mve_move")
1776 ])
1777
1778 ;;
1779 ;; [vsubq_s, vsubq_u])
1780 ;;
1781 (define_insn "mve_vsubq_<supf><mode>"
1782   [
1783    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1784         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1785                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1786          VSUBQ))
1787   ]
1788   "TARGET_HAVE_MVE"
1789   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1790   [(set_attr "type" "mve_move")
1791 ])
1792
1793 (define_insn "mve_vsubq<mode>"
1794   [
1795    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1796         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1797                      (match_operand:MVE_2 2 "s_register_operand" "w")))
1798   ]
1799   "TARGET_HAVE_MVE"
1800   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1801   [(set_attr "type" "mve_move")
1802 ])
1803
1804 ;;
1805 ;; [vabdq_f])
1806 ;;
1807 (define_insn "mve_vabdq_f<mode>"
1808   [
1809    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1810         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1811                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1812          VABDQ_F))
1813   ]
1814   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1815   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
1816   [(set_attr "type" "mve_move")
1817 ])
1818
1819 ;;
1820 ;; [vaddlvaq_s vaddlvaq_u])
1821 ;;
1822 (define_insn "mve_vaddlvaq_<supf>v4si"
1823   [
1824    (set (match_operand:DI 0 "s_register_operand" "=r")
1825         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1826                     (match_operand:V4SI 2 "s_register_operand" "w")]
1827          VADDLVAQ))
1828   ]
1829   "TARGET_HAVE_MVE"
1830   "vaddlva.<supf>32 %Q0, %R0, %q2"
1831   [(set_attr "type" "mve_move")
1832 ])
1833
1834 ;;
1835 ;; [vaddq_n_f])
1836 ;;
1837 (define_insn "mve_vaddq_n_f<mode>"
1838   [
1839    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1840         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1841                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1842          VADDQ_N_F))
1843   ]
1844   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1845   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
1846   [(set_attr "type" "mve_move")
1847 ])
1848
1849 ;;
1850 ;; [vandq_f])
1851 ;;
1852 (define_insn "mve_vandq_f<mode>"
1853   [
1854    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1855         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1856                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1857   ]
1858   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1859   "vand %q0, %q1, %q2"
1860   [(set_attr "type" "mve_move")
1861 ])
1862
1863 ;;
1864 ;; [vbicq_f])
1865 ;;
1866 (define_insn "mve_vbicq_f<mode>"
1867   [
1868    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1869         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
1870                               (match_operand:MVE_0 2 "s_register_operand" "w")))
1871   ]
1872   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1873   "vbic %q0, %q1, %q2"
1874   [(set_attr "type" "mve_move")
1875 ])
1876
1877 ;;
1878 ;; [vbicq_n_s, vbicq_n_u])
1879 ;;
1880 (define_insn "mve_vbicq_n_<supf><mode>"
1881   [
1882    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1883         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1884                        (match_operand:SI 2 "immediate_operand" "i")]
1885          VBICQ_N))
1886   ]
1887   "TARGET_HAVE_MVE"
1888   "vbic.i%#<V_sz_elem>  %q0, %2"
1889   [(set_attr "type" "mve_move")
1890 ])
1891
1892 ;;
1893 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
1894 ;;
1895 (define_insn "mve_vcaddq<mve_rot><mode>"
1896   [
1897    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1898         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1899                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1900          VCADD))
1901   ]
1902   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1903   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1904   [(set_attr "type" "mve_move")
1905 ])
1906
1907 ;;
1908 ;; [vcmpeqq_f, vcmpgeq_f, vcmpgtq_f, vcmpleq_f, vcmpltq_f, vcmpneq_f])
1909 ;;
1910 (define_insn "@mve_vcmp<mve_cmp_op>q_f<mode>"
1911   [
1912    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1913         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1914                                (match_operand:MVE_0 2 "s_register_operand" "w")))
1915   ]
1916   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1917   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
1918   [(set_attr "type" "mve_move")
1919 ])
1920
1921 ;;
1922 ;; [vcmpeqq_n_f, vcmpgeq_n_f, vcmpgtq_n_f, vcmpleq_n_f, vcmpltq_n_f, vcmpneq_n_f])
1923 ;;
1924 (define_insn "@mve_vcmp<mve_cmp_op>q_n_f<mode>"
1925   [
1926    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1927         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1928                                (match_operand:<V_elem> 2 "s_register_operand" "r")))
1929   ]
1930   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1931   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %2"
1932   [(set_attr "type" "mve_move")
1933 ])
1934
1935 ;;
1936 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
1937 ;;
1938 (define_insn "mve_vcmulq<mve_rot><mode>"
1939   [
1940    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1941         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1942                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1943          VCMUL))
1944   ]
1945   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1946   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1947   [(set_attr "type" "mve_move")
1948 ])
1949
1950 ;;
1951 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
1952 ;;
1953 (define_insn "mve_vctp<mode1>q_mhi"
1954   [
1955    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1956         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
1957                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1958          VCTPQ_M))
1959   ]
1960   "TARGET_HAVE_MVE"
1961   "vpst\;vctpt.<mode1> %1"
1962   [(set_attr "type" "mve_move")
1963    (set_attr "length""8")])
1964
1965 ;;
1966 ;; [vcvtbq_f16_f32])
1967 ;;
1968 (define_insn "mve_vcvtbq_f16_f32v8hf"
1969   [
1970    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1971         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1972                       (match_operand:V4SF 2 "s_register_operand" "w")]
1973          VCVTBQ_F16_F32))
1974   ]
1975   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1976   "vcvtb.f16.f32 %q0, %q2"
1977   [(set_attr "type" "mve_move")
1978 ])
1979
1980 ;;
1981 ;; [vcvttq_f16_f32])
1982 ;;
1983 (define_insn "mve_vcvttq_f16_f32v8hf"
1984   [
1985    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1986         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1987                       (match_operand:V4SF 2 "s_register_operand" "w")]
1988          VCVTTQ_F16_F32))
1989   ]
1990   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1991   "vcvtt.f16.f32 %q0, %q2"
1992   [(set_attr "type" "mve_move")
1993 ])
1994
1995 ;;
1996 ;; [veorq_f])
1997 ;;
1998 (define_insn "mve_veorq_f<mode>"
1999   [
2000    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2001         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2002                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2003   ]
2004   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2005   "veor %q0, %q1, %q2"
2006   [(set_attr "type" "mve_move")
2007 ])
2008
2009 ;;
2010 ;; [vmaxnmaq_f])
2011 ;;
2012 (define_insn "mve_vmaxnmaq_f<mode>"
2013   [
2014    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2015         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2016                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2017          VMAXNMAQ_F))
2018   ]
2019   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2020   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2021   [(set_attr "type" "mve_move")
2022 ])
2023
2024 ;;
2025 ;; [vmaxnmavq_f])
2026 ;;
2027 (define_insn "mve_vmaxnmavq_f<mode>"
2028   [
2029    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2030         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2031                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2032          VMAXNMAVQ_F))
2033   ]
2034   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2035   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2036   [(set_attr "type" "mve_move")
2037 ])
2038
2039 ;;
2040 ;; [vmaxnmq_f])
2041 ;;
2042 (define_insn "mve_vmaxnmq_f<mode>"
2043   [
2044    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2045         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2046                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2047   ]
2048   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2049   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2050   [(set_attr "type" "mve_move")
2051 ])
2052
2053 ;;
2054 ;; [vmaxnmvq_f])
2055 ;;
2056 (define_insn "mve_vmaxnmvq_f<mode>"
2057   [
2058    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2059         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2060                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2061          VMAXNMVQ_F))
2062   ]
2063   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2064   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2065   [(set_attr "type" "mve_move")
2066 ])
2067
2068 ;;
2069 ;; [vminnmaq_f])
2070 ;;
2071 (define_insn "mve_vminnmaq_f<mode>"
2072   [
2073    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2074         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2075                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2076          VMINNMAQ_F))
2077   ]
2078   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2079   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2080   [(set_attr "type" "mve_move")
2081 ])
2082
2083 ;;
2084 ;; [vminnmavq_f])
2085 ;;
2086 (define_insn "mve_vminnmavq_f<mode>"
2087   [
2088    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2089         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2090                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2091          VMINNMAVQ_F))
2092   ]
2093   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2094   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2095   [(set_attr "type" "mve_move")
2096 ])
2097
2098 ;;
2099 ;; [vminnmq_f])
2100 ;;
2101 (define_insn "mve_vminnmq_f<mode>"
2102   [
2103    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2104         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2105                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2106   ]
2107   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2108   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2109   [(set_attr "type" "mve_move")
2110 ])
2111
2112 ;;
2113 ;; [vminnmvq_f])
2114 ;;
2115 (define_insn "mve_vminnmvq_f<mode>"
2116   [
2117    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2118         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2119                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2120          VMINNMVQ_F))
2121   ]
2122   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2123   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2124   [(set_attr "type" "mve_move")
2125 ])
2126
2127 ;;
2128 ;; [vmlaldavq_u, vmlaldavq_s])
2129 ;;
2130 (define_insn "mve_vmlaldavq_<supf><mode>"
2131   [
2132    (set (match_operand:DI 0 "s_register_operand" "=r")
2133         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2134                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2135          VMLALDAVQ))
2136   ]
2137   "TARGET_HAVE_MVE"
2138   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2139   [(set_attr "type" "mve_move")
2140 ])
2141
2142 ;;
2143 ;; [vmlaldavxq_s])
2144 ;;
2145 (define_insn "mve_vmlaldavxq_s<mode>"
2146   [
2147    (set (match_operand:DI 0 "s_register_operand" "=r")
2148         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2149                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2150          VMLALDAVXQ_S))
2151   ]
2152   "TARGET_HAVE_MVE"
2153   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2154   [(set_attr "type" "mve_move")
2155 ])
2156
2157 ;;
2158 ;; [vmlsldavq_s])
2159 ;;
2160 (define_insn "mve_vmlsldavq_s<mode>"
2161   [
2162    (set (match_operand:DI 0 "s_register_operand" "=r")
2163         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2164                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2165          VMLSLDAVQ_S))
2166   ]
2167   "TARGET_HAVE_MVE"
2168   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2169   [(set_attr "type" "mve_move")
2170 ])
2171
2172 ;;
2173 ;; [vmlsldavxq_s])
2174 ;;
2175 (define_insn "mve_vmlsldavxq_s<mode>"
2176   [
2177    (set (match_operand:DI 0 "s_register_operand" "=r")
2178         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2179                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2180          VMLSLDAVXQ_S))
2181   ]
2182   "TARGET_HAVE_MVE"
2183   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2184   [(set_attr "type" "mve_move")
2185 ])
2186
2187 ;;
2188 ;; [vmovnbq_u, vmovnbq_s])
2189 ;;
2190 (define_insn "mve_vmovnbq_<supf><mode>"
2191   [
2192    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2193         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2194                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2195          VMOVNBQ))
2196   ]
2197   "TARGET_HAVE_MVE"
2198   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2199   [(set_attr "type" "mve_move")
2200 ])
2201
2202 ;;
2203 ;; [vmovntq_s, vmovntq_u])
2204 ;;
2205 (define_insn "mve_vmovntq_<supf><mode>"
2206   [
2207    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2208         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2209                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2210          VMOVNTQ))
2211   ]
2212   "TARGET_HAVE_MVE"
2213   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2214   [(set_attr "type" "mve_move")
2215 ])
2216
2217 ;;
2218 ;; [vmulq_f])
2219 ;;
2220 (define_insn "mve_vmulq_f<mode>"
2221   [
2222    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2223         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2224                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2225   ]
2226   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2227   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2228   [(set_attr "type" "mve_move")
2229 ])
2230
2231 ;;
2232 ;; [vmulq_n_f])
2233 ;;
2234 (define_insn "mve_vmulq_n_f<mode>"
2235   [
2236    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2237         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2238                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2239          VMULQ_N_F))
2240   ]
2241   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2242   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2243   [(set_attr "type" "mve_move")
2244 ])
2245
2246 ;;
2247 ;; [vornq_f])
2248 ;;
2249 (define_insn "mve_vornq_f<mode>"
2250   [
2251    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2252         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
2253                    (match_operand:MVE_0 1 "s_register_operand" "w")))
2254   ]
2255   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2256   "vorn %q0, %q1, %q2"
2257   [(set_attr "type" "mve_move")
2258 ])
2259
2260 ;;
2261 ;; [vorrq_f])
2262 ;;
2263 (define_insn "mve_vorrq_f<mode>"
2264   [
2265    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2266         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2267                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2268   ]
2269   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2270   "vorr %q0, %q1, %q2"
2271   [(set_attr "type" "mve_move")
2272 ])
2273
2274 ;;
2275 ;; [vorrq_n_u, vorrq_n_s])
2276 ;;
2277 (define_insn "mve_vorrq_n_<supf><mode>"
2278   [
2279    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2280         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2281                        (match_operand:SI 2 "immediate_operand" "i")]
2282          VORRQ_N))
2283   ]
2284   "TARGET_HAVE_MVE"
2285   "vorr.i%#<V_sz_elem>  %q0, %2"
2286   [(set_attr "type" "mve_move")
2287 ])
2288
2289 ;;
2290 ;; [vqdmullbq_n_s])
2291 ;;
2292 (define_insn "mve_vqdmullbq_n_s<mode>"
2293   [
2294    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2295         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2296                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2297          VQDMULLBQ_N_S))
2298   ]
2299   "TARGET_HAVE_MVE"
2300   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2301   [(set_attr "type" "mve_move")
2302 ])
2303
2304 ;;
2305 ;; [vqdmullbq_s])
2306 ;;
2307 (define_insn "mve_vqdmullbq_s<mode>"
2308   [
2309    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2310         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2311                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2312          VQDMULLBQ_S))
2313   ]
2314   "TARGET_HAVE_MVE"
2315   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2316   [(set_attr "type" "mve_move")
2317 ])
2318
2319 ;;
2320 ;; [vqdmulltq_n_s])
2321 ;;
2322 (define_insn "mve_vqdmulltq_n_s<mode>"
2323   [
2324    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2325         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2326                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2327          VQDMULLTQ_N_S))
2328   ]
2329   "TARGET_HAVE_MVE"
2330   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2331   [(set_attr "type" "mve_move")
2332 ])
2333
2334 ;;
2335 ;; [vqdmulltq_s])
2336 ;;
2337 (define_insn "mve_vqdmulltq_s<mode>"
2338   [
2339    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2340         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2341                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2342          VQDMULLTQ_S))
2343   ]
2344   "TARGET_HAVE_MVE"
2345   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2346   [(set_attr "type" "mve_move")
2347 ])
2348
2349 ;;
2350 ;; [vqmovnbq_u, vqmovnbq_s])
2351 ;;
2352 (define_insn "mve_vqmovnbq_<supf><mode>"
2353   [
2354    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2355         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2356                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2357          VQMOVNBQ))
2358   ]
2359   "TARGET_HAVE_MVE"
2360   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2361   [(set_attr "type" "mve_move")
2362 ])
2363
2364 ;;
2365 ;; [vqmovntq_u, vqmovntq_s])
2366 ;;
2367 (define_insn "mve_vqmovntq_<supf><mode>"
2368   [
2369    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2370         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2371                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2372          VQMOVNTQ))
2373   ]
2374   "TARGET_HAVE_MVE"
2375   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2376   [(set_attr "type" "mve_move")
2377 ])
2378
2379 ;;
2380 ;; [vqmovunbq_s])
2381 ;;
2382 (define_insn "mve_vqmovunbq_s<mode>"
2383   [
2384    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2385         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2386                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2387          VQMOVUNBQ_S))
2388   ]
2389   "TARGET_HAVE_MVE"
2390   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2391   [(set_attr "type" "mve_move")
2392 ])
2393
2394 ;;
2395 ;; [vqmovuntq_s])
2396 ;;
2397 (define_insn "mve_vqmovuntq_s<mode>"
2398   [
2399    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2400         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2401                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2402          VQMOVUNTQ_S))
2403   ]
2404   "TARGET_HAVE_MVE"
2405   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2406   [(set_attr "type" "mve_move")
2407 ])
2408
2409 ;;
2410 ;; [vrmlaldavhxq_s])
2411 ;;
2412 (define_insn "mve_vrmlaldavhxq_sv4si"
2413   [
2414    (set (match_operand:DI 0 "s_register_operand" "=r")
2415         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2416                     (match_operand:V4SI 2 "s_register_operand" "w")]
2417          VRMLALDAVHXQ_S))
2418   ]
2419   "TARGET_HAVE_MVE"
2420   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2421   [(set_attr "type" "mve_move")
2422 ])
2423
2424 ;;
2425 ;; [vrmlsldavhq_s])
2426 ;;
2427 (define_insn "mve_vrmlsldavhq_sv4si"
2428   [
2429    (set (match_operand:DI 0 "s_register_operand" "=r")
2430         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2431                     (match_operand:V4SI 2 "s_register_operand" "w")]
2432          VRMLSLDAVHQ_S))
2433   ]
2434   "TARGET_HAVE_MVE"
2435   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2436   [(set_attr "type" "mve_move")
2437 ])
2438
2439 ;;
2440 ;; [vrmlsldavhxq_s])
2441 ;;
2442 (define_insn "mve_vrmlsldavhxq_sv4si"
2443   [
2444    (set (match_operand:DI 0 "s_register_operand" "=r")
2445         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2446                     (match_operand:V4SI 2 "s_register_operand" "w")]
2447          VRMLSLDAVHXQ_S))
2448   ]
2449   "TARGET_HAVE_MVE"
2450   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2451   [(set_attr "type" "mve_move")
2452 ])
2453
2454 ;;
2455 ;; [vshllbq_n_s, vshllbq_n_u])
2456 ;;
2457 (define_insn "mve_vshllbq_n_<supf><mode>"
2458   [
2459    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2460         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2461                                   (match_operand:SI 2 "immediate_operand" "i")]
2462          VSHLLBQ_N))
2463   ]
2464   "TARGET_HAVE_MVE"
2465   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2466   [(set_attr "type" "mve_move")
2467 ])
2468
2469 ;;
2470 ;; [vshlltq_n_u, vshlltq_n_s])
2471 ;;
2472 (define_insn "mve_vshlltq_n_<supf><mode>"
2473   [
2474    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2475         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2476                                   (match_operand:SI 2 "immediate_operand" "i")]
2477          VSHLLTQ_N))
2478   ]
2479   "TARGET_HAVE_MVE"
2480   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2481   [(set_attr "type" "mve_move")
2482 ])
2483
2484 ;;
2485 ;; [vsubq_f])
2486 ;;
2487 (define_insn "mve_vsubq_f<mode>"
2488   [
2489    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2490         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2491                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2492   ]
2493   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2494   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2495   [(set_attr "type" "mve_move")
2496 ])
2497
2498 ;;
2499 ;; [vmulltq_poly_p])
2500 ;;
2501 (define_insn "mve_vmulltq_poly_p<mode>"
2502   [
2503    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2504         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2505                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2506          VMULLTQ_POLY_P))
2507   ]
2508   "TARGET_HAVE_MVE"
2509   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2510   [(set_attr "type" "mve_move")
2511 ])
2512
2513 ;;
2514 ;; [vmullbq_poly_p])
2515 ;;
2516 (define_insn "mve_vmullbq_poly_p<mode>"
2517   [
2518    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2519         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2520                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2521          VMULLBQ_POLY_P))
2522   ]
2523   "TARGET_HAVE_MVE"
2524   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2525   [(set_attr "type" "mve_move")
2526 ])
2527
2528 ;;
2529 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2530 ;;
2531 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2532   [
2533    (set (match_operand:DI 0 "s_register_operand" "=r")
2534         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2535                     (match_operand:V4SI 2 "s_register_operand" "w")]
2536          VRMLALDAVHQ))
2537   ]
2538   "TARGET_HAVE_MVE"
2539   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2540   [(set_attr "type" "mve_move")
2541 ])
2542
2543 ;;
2544 ;; [vbicq_m_n_s, vbicq_m_n_u])
2545 ;;
2546 (define_insn "mve_vbicq_m_n_<supf><mode>"
2547   [
2548    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2549         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2550                        (match_operand:SI 2 "immediate_operand" "i")
2551                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2552          VBICQ_M_N))
2553   ]
2554   "TARGET_HAVE_MVE"
2555   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2556   [(set_attr "type" "mve_move")
2557    (set_attr "length""8")])
2558 ;;
2559 ;; [vcmpeqq_m_f])
2560 ;;
2561 (define_insn "mve_vcmpeqq_m_f<mode>"
2562   [
2563    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2564         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2565                     (match_operand:MVE_0 2 "s_register_operand" "w")
2566                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2567          VCMPEQQ_M_F))
2568   ]
2569   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2570   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2571   [(set_attr "type" "mve_move")
2572    (set_attr "length""8")])
2573 ;;
2574 ;; [vcvtaq_m_u, vcvtaq_m_s])
2575 ;;
2576 (define_insn "mve_vcvtaq_m_<supf><mode>"
2577   [
2578    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2579         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2580                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2581                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2582          VCVTAQ_M))
2583   ]
2584   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2585   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2586   [(set_attr "type" "mve_move")
2587    (set_attr "length""8")])
2588 ;;
2589 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2590 ;;
2591 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2592   [
2593    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2594         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2595                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2596                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2597          VCVTQ_M_TO_F))
2598   ]
2599   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2600   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2601   [(set_attr "type" "mve_move")
2602    (set_attr "length""8")])
2603 ;;
2604 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2605 ;;
2606 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2607   [
2608    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2609         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2610                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2611                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2612          VQRSHRNBQ_N))
2613   ]
2614   "TARGET_HAVE_MVE"
2615   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2616   [(set_attr "type" "mve_move")
2617 ])
2618 ;;
2619 ;; [vqrshrunbq_n_s])
2620 ;;
2621 (define_insn "mve_vqrshrunbq_n_s<mode>"
2622   [
2623    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2624         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2625                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2626                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2627          VQRSHRUNBQ_N_S))
2628   ]
2629   "TARGET_HAVE_MVE"
2630   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
2631   [(set_attr "type" "mve_move")
2632 ])
2633 ;;
2634 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
2635 ;;
2636 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
2637   [
2638    (set (match_operand:DI 0 "s_register_operand" "=r")
2639         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2640                     (match_operand:V4SI 2 "s_register_operand" "w")
2641                     (match_operand:V4SI 3 "s_register_operand" "w")]
2642          VRMLALDAVHAQ))
2643   ]
2644   "TARGET_HAVE_MVE"
2645   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
2646   [(set_attr "type" "mve_move")
2647 ])
2648
2649 ;;
2650 ;; [vabavq_s, vabavq_u])
2651 ;;
2652 (define_insn "mve_vabavq_<supf><mode>"
2653   [
2654    (set (match_operand:SI 0 "s_register_operand" "=r")
2655         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2656                     (match_operand:MVE_2 2 "s_register_operand" "w")
2657                     (match_operand:MVE_2 3 "s_register_operand" "w")]
2658          VABAVQ))
2659   ]
2660   "TARGET_HAVE_MVE"
2661   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2662   [(set_attr "type" "mve_move")
2663 ])
2664
2665 ;;
2666 ;; [vshlcq_u vshlcq_s]
2667 ;;
2668 (define_expand "mve_vshlcq_vec_<supf><mode>"
2669  [(match_operand:MVE_2 0 "s_register_operand")
2670   (match_operand:MVE_2 1 "s_register_operand")
2671   (match_operand:SI 2 "s_register_operand")
2672   (match_operand:SI 3 "mve_imm_32")
2673   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2674  "TARGET_HAVE_MVE"
2675 {
2676   rtx ignore_wb = gen_reg_rtx (SImode);
2677   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
2678                                       operands[2], operands[3]));
2679   DONE;
2680 })
2681
2682 (define_expand "mve_vshlcq_carry_<supf><mode>"
2683  [(match_operand:SI 0 "s_register_operand")
2684   (match_operand:MVE_2 1 "s_register_operand")
2685   (match_operand:SI 2 "s_register_operand")
2686   (match_operand:SI 3 "mve_imm_32")
2687   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2688  "TARGET_HAVE_MVE"
2689 {
2690   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
2691   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
2692                                       operands[2], operands[3]));
2693   DONE;
2694 })
2695
2696 (define_insn "mve_vshlcq_<supf><mode>"
2697  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
2698        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
2699                       (match_operand:SI 3 "s_register_operand" "1")
2700                       (match_operand:SI 4 "mve_imm_32" "Rf")]
2701         VSHLCQ))
2702   (set (match_operand:SI  1 "s_register_operand" "=r")
2703        (unspec:SI [(match_dup 2)
2704                    (match_dup 3)
2705                    (match_dup 4)]
2706         VSHLCQ))]
2707  "TARGET_HAVE_MVE"
2708  "vshlc %q0, %1, %4")
2709
2710 ;;
2711 ;; [vabsq_m_s])
2712 ;;
2713 (define_insn "mve_vabsq_m_s<mode>"
2714   [
2715    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2716         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2717                        (match_operand:MVE_2 2 "s_register_operand" "w")
2718                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2719          VABSQ_M_S))
2720   ]
2721   "TARGET_HAVE_MVE"
2722   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
2723   [(set_attr "type" "mve_move")
2724    (set_attr "length""8")])
2725
2726 ;;
2727 ;; [vaddvaq_p_u, vaddvaq_p_s])
2728 ;;
2729 (define_insn "mve_vaddvaq_p_<supf><mode>"
2730   [
2731    (set (match_operand:SI 0 "s_register_operand" "=Te")
2732         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2733                        (match_operand:MVE_2 2 "s_register_operand" "w")
2734                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2735          VADDVAQ_P))
2736   ]
2737   "TARGET_HAVE_MVE"
2738   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
2739   [(set_attr "type" "mve_move")
2740    (set_attr "length""8")])
2741
2742 ;;
2743 ;; [vclsq_m_s])
2744 ;;
2745 (define_insn "mve_vclsq_m_s<mode>"
2746   [
2747    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2748         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2749                        (match_operand:MVE_2 2 "s_register_operand" "w")
2750                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2751          VCLSQ_M_S))
2752   ]
2753   "TARGET_HAVE_MVE"
2754   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
2755   [(set_attr "type" "mve_move")
2756    (set_attr "length""8")])
2757
2758 ;;
2759 ;; [vclzq_m_s, vclzq_m_u])
2760 ;;
2761 (define_insn "mve_vclzq_m_<supf><mode>"
2762   [
2763    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2764         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2765                        (match_operand:MVE_2 2 "s_register_operand" "w")
2766                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2767          VCLZQ_M))
2768   ]
2769   "TARGET_HAVE_MVE"
2770   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
2771   [(set_attr "type" "mve_move")
2772    (set_attr "length""8")])
2773
2774 ;;
2775 ;; [vcmpcsq_m_n_u])
2776 ;;
2777 (define_insn "mve_vcmpcsq_m_n_u<mode>"
2778   [
2779    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2780         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2781                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2782                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2783          VCMPCSQ_M_N_U))
2784   ]
2785   "TARGET_HAVE_MVE"
2786   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
2787   [(set_attr "type" "mve_move")
2788    (set_attr "length""8")])
2789
2790 ;;
2791 ;; [vcmpcsq_m_u])
2792 ;;
2793 (define_insn "mve_vcmpcsq_m_u<mode>"
2794   [
2795    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2796         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2797                        (match_operand:MVE_2 2 "s_register_operand" "w")
2798                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2799          VCMPCSQ_M_U))
2800   ]
2801   "TARGET_HAVE_MVE"
2802   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
2803   [(set_attr "type" "mve_move")
2804    (set_attr "length""8")])
2805
2806 ;;
2807 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
2808 ;;
2809 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
2810   [
2811    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2812         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2813                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2814                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2815          VCMPEQQ_M_N))
2816   ]
2817   "TARGET_HAVE_MVE"
2818   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
2819   [(set_attr "type" "mve_move")
2820    (set_attr "length""8")])
2821
2822 ;;
2823 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
2824 ;;
2825 (define_insn "mve_vcmpeqq_m_<supf><mode>"
2826   [
2827    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2828         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2829                        (match_operand:MVE_2 2 "s_register_operand" "w")
2830                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2831          VCMPEQQ_M))
2832   ]
2833   "TARGET_HAVE_MVE"
2834   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
2835   [(set_attr "type" "mve_move")
2836    (set_attr "length""8")])
2837
2838 ;;
2839 ;; [vcmpgeq_m_n_s])
2840 ;;
2841 (define_insn "mve_vcmpgeq_m_n_s<mode>"
2842   [
2843    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2844         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2845                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2846                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2847          VCMPGEQ_M_N_S))
2848   ]
2849   "TARGET_HAVE_MVE"
2850   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
2851   [(set_attr "type" "mve_move")
2852    (set_attr "length""8")])
2853
2854 ;;
2855 ;; [vcmpgeq_m_s])
2856 ;;
2857 (define_insn "mve_vcmpgeq_m_s<mode>"
2858   [
2859    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2860         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2861                        (match_operand:MVE_2 2 "s_register_operand" "w")
2862                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2863          VCMPGEQ_M_S))
2864   ]
2865   "TARGET_HAVE_MVE"
2866   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
2867   [(set_attr "type" "mve_move")
2868    (set_attr "length""8")])
2869
2870 ;;
2871 ;; [vcmpgtq_m_n_s])
2872 ;;
2873 (define_insn "mve_vcmpgtq_m_n_s<mode>"
2874   [
2875    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2876         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2877                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2878                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2879          VCMPGTQ_M_N_S))
2880   ]
2881   "TARGET_HAVE_MVE"
2882   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
2883   [(set_attr "type" "mve_move")
2884    (set_attr "length""8")])
2885
2886 ;;
2887 ;; [vcmpgtq_m_s])
2888 ;;
2889 (define_insn "mve_vcmpgtq_m_s<mode>"
2890   [
2891    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2892         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2893                        (match_operand:MVE_2 2 "s_register_operand" "w")
2894                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2895          VCMPGTQ_M_S))
2896   ]
2897   "TARGET_HAVE_MVE"
2898   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
2899   [(set_attr "type" "mve_move")
2900    (set_attr "length""8")])
2901
2902 ;;
2903 ;; [vcmphiq_m_n_u])
2904 ;;
2905 (define_insn "mve_vcmphiq_m_n_u<mode>"
2906   [
2907    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2908         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2909                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2910                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2911          VCMPHIQ_M_N_U))
2912   ]
2913   "TARGET_HAVE_MVE"
2914   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
2915   [(set_attr "type" "mve_move")
2916    (set_attr "length""8")])
2917
2918 ;;
2919 ;; [vcmphiq_m_u])
2920 ;;
2921 (define_insn "mve_vcmphiq_m_u<mode>"
2922   [
2923    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2924         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2925                        (match_operand:MVE_2 2 "s_register_operand" "w")
2926                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2927          VCMPHIQ_M_U))
2928   ]
2929   "TARGET_HAVE_MVE"
2930   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
2931   [(set_attr "type" "mve_move")
2932    (set_attr "length""8")])
2933
2934 ;;
2935 ;; [vcmpleq_m_n_s])
2936 ;;
2937 (define_insn "mve_vcmpleq_m_n_s<mode>"
2938   [
2939    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2940         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2941                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2942                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2943          VCMPLEQ_M_N_S))
2944   ]
2945   "TARGET_HAVE_MVE"
2946   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
2947   [(set_attr "type" "mve_move")
2948    (set_attr "length""8")])
2949
2950 ;;
2951 ;; [vcmpleq_m_s])
2952 ;;
2953 (define_insn "mve_vcmpleq_m_s<mode>"
2954   [
2955    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2956         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2957                        (match_operand:MVE_2 2 "s_register_operand" "w")
2958                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2959          VCMPLEQ_M_S))
2960   ]
2961   "TARGET_HAVE_MVE"
2962   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
2963   [(set_attr "type" "mve_move")
2964    (set_attr "length""8")])
2965
2966 ;;
2967 ;; [vcmpltq_m_n_s])
2968 ;;
2969 (define_insn "mve_vcmpltq_m_n_s<mode>"
2970   [
2971    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2972         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2973                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2974                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2975          VCMPLTQ_M_N_S))
2976   ]
2977   "TARGET_HAVE_MVE"
2978   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
2979   [(set_attr "type" "mve_move")
2980    (set_attr "length""8")])
2981
2982 ;;
2983 ;; [vcmpltq_m_s])
2984 ;;
2985 (define_insn "mve_vcmpltq_m_s<mode>"
2986   [
2987    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2988         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2989                        (match_operand:MVE_2 2 "s_register_operand" "w")
2990                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2991          VCMPLTQ_M_S))
2992   ]
2993   "TARGET_HAVE_MVE"
2994   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
2995   [(set_attr "type" "mve_move")
2996    (set_attr "length""8")])
2997
2998 ;;
2999 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3000 ;;
3001 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3002   [
3003    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3004         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3005                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3006                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3007          VCMPNEQ_M_N))
3008   ]
3009   "TARGET_HAVE_MVE"
3010   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3011   [(set_attr "type" "mve_move")
3012    (set_attr "length""8")])
3013
3014 ;;
3015 ;; [vcmpneq_m_s, vcmpneq_m_u])
3016 ;;
3017 (define_insn "mve_vcmpneq_m_<supf><mode>"
3018   [
3019    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3020         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3021                        (match_operand:MVE_2 2 "s_register_operand" "w")
3022                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3023          VCMPNEQ_M))
3024   ]
3025   "TARGET_HAVE_MVE"
3026   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3027   [(set_attr "type" "mve_move")
3028    (set_attr "length""8")])
3029
3030 ;;
3031 ;; [vdupq_m_n_s, vdupq_m_n_u])
3032 ;;
3033 (define_insn "mve_vdupq_m_n_<supf><mode>"
3034   [
3035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3037                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3038                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3039          VDUPQ_M_N))
3040   ]
3041   "TARGET_HAVE_MVE"
3042   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3043   [(set_attr "type" "mve_move")
3044    (set_attr "length""8")])
3045
3046 ;;
3047 ;; [vmaxaq_m_s])
3048 ;;
3049 (define_insn "mve_vmaxaq_m_s<mode>"
3050   [
3051    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3052         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3053                        (match_operand:MVE_2 2 "s_register_operand" "w")
3054                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3055          VMAXAQ_M_S))
3056   ]
3057   "TARGET_HAVE_MVE"
3058   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3059   [(set_attr "type" "mve_move")
3060    (set_attr "length""8")])
3061
3062 ;;
3063 ;; [vmaxavq_p_s])
3064 ;;
3065 (define_insn "mve_vmaxavq_p_s<mode>"
3066   [
3067    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3068         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3069                        (match_operand:MVE_2 2 "s_register_operand" "w")
3070                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3071          VMAXAVQ_P_S))
3072   ]
3073   "TARGET_HAVE_MVE"
3074   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3075   [(set_attr "type" "mve_move")
3076    (set_attr "length""8")])
3077
3078 ;;
3079 ;; [vmaxvq_p_u, vmaxvq_p_s])
3080 ;;
3081 (define_insn "mve_vmaxvq_p_<supf><mode>"
3082   [
3083    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3084         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3085                        (match_operand:MVE_2 2 "s_register_operand" "w")
3086                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3087          VMAXVQ_P))
3088   ]
3089   "TARGET_HAVE_MVE"
3090   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3091   [(set_attr "type" "mve_move")
3092    (set_attr "length""8")])
3093
3094 ;;
3095 ;; [vminaq_m_s])
3096 ;;
3097 (define_insn "mve_vminaq_m_s<mode>"
3098   [
3099    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3100         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3101                        (match_operand:MVE_2 2 "s_register_operand" "w")
3102                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3103          VMINAQ_M_S))
3104   ]
3105   "TARGET_HAVE_MVE"
3106   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3107   [(set_attr "type" "mve_move")
3108    (set_attr "length""8")])
3109
3110 ;;
3111 ;; [vminavq_p_s])
3112 ;;
3113 (define_insn "mve_vminavq_p_s<mode>"
3114   [
3115    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3116         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3117                        (match_operand:MVE_2 2 "s_register_operand" "w")
3118                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3119          VMINAVQ_P_S))
3120   ]
3121   "TARGET_HAVE_MVE"
3122   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3123   [(set_attr "type" "mve_move")
3124    (set_attr "length""8")])
3125
3126 ;;
3127 ;; [vminvq_p_s, vminvq_p_u])
3128 ;;
3129 (define_insn "mve_vminvq_p_<supf><mode>"
3130   [
3131    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3132         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3133                        (match_operand:MVE_2 2 "s_register_operand" "w")
3134                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3135          VMINVQ_P))
3136   ]
3137   "TARGET_HAVE_MVE"
3138   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3139   [(set_attr "type" "mve_move")
3140    (set_attr "length""8")])
3141
3142 ;;
3143 ;; [vmladavaq_u, vmladavaq_s])
3144 ;;
3145 (define_insn "mve_vmladavaq_<supf><mode>"
3146   [
3147    (set (match_operand:SI 0 "s_register_operand" "=Te")
3148         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3149                        (match_operand:MVE_2 2 "s_register_operand" "w")
3150                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3151          VMLADAVAQ))
3152   ]
3153   "TARGET_HAVE_MVE"
3154   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3155   [(set_attr "type" "mve_move")
3156 ])
3157
3158 ;;
3159 ;; [vmladavq_p_u, vmladavq_p_s])
3160 ;;
3161 (define_insn "mve_vmladavq_p_<supf><mode>"
3162   [
3163    (set (match_operand:SI 0 "s_register_operand" "=Te")
3164         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3165                        (match_operand:MVE_2 2 "s_register_operand" "w")
3166                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3167          VMLADAVQ_P))
3168   ]
3169   "TARGET_HAVE_MVE"
3170   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3171   [(set_attr "type" "mve_move")
3172    (set_attr "length""8")])
3173
3174 ;;
3175 ;; [vmladavxq_p_s])
3176 ;;
3177 (define_insn "mve_vmladavxq_p_s<mode>"
3178   [
3179    (set (match_operand:SI 0 "s_register_operand" "=Te")
3180         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3181                        (match_operand:MVE_2 2 "s_register_operand" "w")
3182                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3183          VMLADAVXQ_P_S))
3184   ]
3185   "TARGET_HAVE_MVE"
3186   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3187   [(set_attr "type" "mve_move")
3188    (set_attr "length""8")])
3189
3190 ;;
3191 ;; [vmlaq_n_u, vmlaq_n_s])
3192 ;;
3193 (define_insn "mve_vmlaq_n_<supf><mode>"
3194   [
3195    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3196         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3197                        (match_operand:MVE_2 2 "s_register_operand" "w")
3198                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3199          VMLAQ_N))
3200   ]
3201   "TARGET_HAVE_MVE"
3202   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3203   [(set_attr "type" "mve_move")
3204 ])
3205
3206 ;;
3207 ;; [vmlasq_n_u, vmlasq_n_s])
3208 ;;
3209 (define_insn "mve_vmlasq_n_<supf><mode>"
3210   [
3211    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3212         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3213                        (match_operand:MVE_2 2 "s_register_operand" "w")
3214                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3215          VMLASQ_N))
3216   ]
3217   "TARGET_HAVE_MVE"
3218   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3219   [(set_attr "type" "mve_move")
3220 ])
3221
3222 ;;
3223 ;; [vmlsdavq_p_s])
3224 ;;
3225 (define_insn "mve_vmlsdavq_p_s<mode>"
3226   [
3227    (set (match_operand:SI 0 "s_register_operand" "=Te")
3228         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3229                        (match_operand:MVE_2 2 "s_register_operand" "w")
3230                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3231          VMLSDAVQ_P_S))
3232   ]
3233   "TARGET_HAVE_MVE"
3234   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3235   [(set_attr "type" "mve_move")
3236    (set_attr "length""8")])
3237
3238 ;;
3239 ;; [vmlsdavxq_p_s])
3240 ;;
3241 (define_insn "mve_vmlsdavxq_p_s<mode>"
3242   [
3243    (set (match_operand:SI 0 "s_register_operand" "=Te")
3244         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3245                        (match_operand:MVE_2 2 "s_register_operand" "w")
3246                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3247          VMLSDAVXQ_P_S))
3248   ]
3249   "TARGET_HAVE_MVE"
3250   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3251   [(set_attr "type" "mve_move")
3252    (set_attr "length""8")])
3253
3254 ;;
3255 ;; [vmvnq_m_s, vmvnq_m_u])
3256 ;;
3257 (define_insn "mve_vmvnq_m_<supf><mode>"
3258   [
3259    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3260         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3261                        (match_operand:MVE_2 2 "s_register_operand" "w")
3262                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3263          VMVNQ_M))
3264   ]
3265   "TARGET_HAVE_MVE"
3266   "vpst\;vmvnt %q0, %q2"
3267   [(set_attr "type" "mve_move")
3268    (set_attr "length""8")])
3269
3270 ;;
3271 ;; [vnegq_m_s])
3272 ;;
3273 (define_insn "mve_vnegq_m_s<mode>"
3274   [
3275    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3276         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3277                        (match_operand:MVE_2 2 "s_register_operand" "w")
3278                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3279          VNEGQ_M_S))
3280   ]
3281   "TARGET_HAVE_MVE"
3282   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3283   [(set_attr "type" "mve_move")
3284    (set_attr "length""8")])
3285
3286 ;;
3287 ;; [vpselq_u, vpselq_s])
3288 ;;
3289 (define_insn "@mve_vpselq_<supf><mode>"
3290   [
3291    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3292         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3293                        (match_operand:MVE_1 2 "s_register_operand" "w")
3294                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3295          VPSELQ))
3296   ]
3297   "TARGET_HAVE_MVE"
3298   "vpsel %q0, %q1, %q2"
3299   [(set_attr "type" "mve_move")
3300 ])
3301
3302 ;;
3303 ;; [vqabsq_m_s])
3304 ;;
3305 (define_insn "mve_vqabsq_m_s<mode>"
3306   [
3307    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3308         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3309                        (match_operand:MVE_2 2 "s_register_operand" "w")
3310                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3311          VQABSQ_M_S))
3312   ]
3313   "TARGET_HAVE_MVE"
3314   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3315   [(set_attr "type" "mve_move")
3316    (set_attr "length""8")])
3317
3318 ;;
3319 ;; [vqdmlahq_n_s])
3320 ;;
3321 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3322   [
3323    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3324         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3325                        (match_operand:MVE_2 2 "s_register_operand" "w")
3326                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3327          VQDMLAHQ_N))
3328   ]
3329   "TARGET_HAVE_MVE"
3330   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3331   [(set_attr "type" "mve_move")
3332 ])
3333
3334 ;;
3335 ;; [vqdmlashq_n_s])
3336 ;;
3337 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3338   [
3339    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3340         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3341                        (match_operand:MVE_2 2 "s_register_operand" "w")
3342                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3343          VQDMLASHQ_N))
3344   ]
3345   "TARGET_HAVE_MVE"
3346   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3347   [(set_attr "type" "mve_move")
3348 ])
3349
3350 ;;
3351 ;; [vqnegq_m_s])
3352 ;;
3353 (define_insn "mve_vqnegq_m_s<mode>"
3354   [
3355    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3356         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3357                        (match_operand:MVE_2 2 "s_register_operand" "w")
3358                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3359          VQNEGQ_M_S))
3360   ]
3361   "TARGET_HAVE_MVE"
3362   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3363   [(set_attr "type" "mve_move")
3364    (set_attr "length""8")])
3365
3366 ;;
3367 ;; [vqrdmladhq_s])
3368 ;;
3369 (define_insn "mve_vqrdmladhq_s<mode>"
3370   [
3371    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3372         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3373                        (match_operand:MVE_2 2 "s_register_operand" "w")
3374                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3375          VQRDMLADHQ_S))
3376   ]
3377   "TARGET_HAVE_MVE"
3378   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3379   [(set_attr "type" "mve_move")
3380 ])
3381
3382 ;;
3383 ;; [vqrdmladhxq_s])
3384 ;;
3385 (define_insn "mve_vqrdmladhxq_s<mode>"
3386   [
3387    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3388         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3389                        (match_operand:MVE_2 2 "s_register_operand" "w")
3390                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3391          VQRDMLADHXQ_S))
3392   ]
3393   "TARGET_HAVE_MVE"
3394   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3395   [(set_attr "type" "mve_move")
3396 ])
3397
3398 ;;
3399 ;; [vqrdmlahq_n_s])
3400 ;;
3401 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3402   [
3403    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3404         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3405                        (match_operand:MVE_2 2 "s_register_operand" "w")
3406                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3407          VQRDMLAHQ_N))
3408   ]
3409   "TARGET_HAVE_MVE"
3410   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3411   [(set_attr "type" "mve_move")
3412 ])
3413
3414 ;;
3415 ;; [vqrdmlashq_n_s])
3416 ;;
3417 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3418   [
3419    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3420         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3421                        (match_operand:MVE_2 2 "s_register_operand" "w")
3422                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3423          VQRDMLASHQ_N))
3424   ]
3425   "TARGET_HAVE_MVE"
3426   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3427   [(set_attr "type" "mve_move")
3428 ])
3429
3430 ;;
3431 ;; [vqrdmlsdhq_s])
3432 ;;
3433 (define_insn "mve_vqrdmlsdhq_s<mode>"
3434   [
3435    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3436         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3437                        (match_operand:MVE_2 2 "s_register_operand" "w")
3438                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3439          VQRDMLSDHQ_S))
3440   ]
3441   "TARGET_HAVE_MVE"
3442   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3443   [(set_attr "type" "mve_move")
3444 ])
3445
3446 ;;
3447 ;; [vqrdmlsdhxq_s])
3448 ;;
3449 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3450   [
3451    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3452         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3453                        (match_operand:MVE_2 2 "s_register_operand" "w")
3454                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3455          VQRDMLSDHXQ_S))
3456   ]
3457   "TARGET_HAVE_MVE"
3458   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3459   [(set_attr "type" "mve_move")
3460 ])
3461
3462 ;;
3463 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3464 ;;
3465 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3466   [
3467    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3468         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3469                        (match_operand:SI 2 "s_register_operand" "r")
3470                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3471          VQRSHLQ_M_N))
3472   ]
3473   "TARGET_HAVE_MVE"
3474   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3475   [(set_attr "type" "mve_move")
3476    (set_attr "length""8")])
3477
3478 ;;
3479 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3480 ;;
3481 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3482   [
3483    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3484         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3485                        (match_operand:SI 2 "s_register_operand" "r")
3486                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3487          VQSHLQ_M_R))
3488   ]
3489   "TARGET_HAVE_MVE"
3490   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3491   [(set_attr "type" "mve_move")
3492    (set_attr "length""8")])
3493
3494 ;;
3495 ;; [vrev64q_m_u, vrev64q_m_s])
3496 ;;
3497 (define_insn "mve_vrev64q_m_<supf><mode>"
3498   [
3499    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3500         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3501                        (match_operand:MVE_2 2 "s_register_operand" "w")
3502                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3503          VREV64Q_M))
3504   ]
3505   "TARGET_HAVE_MVE"
3506   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3507   [(set_attr "type" "mve_move")
3508    (set_attr "length""8")])
3509
3510 ;;
3511 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3512 ;;
3513 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3514   [
3515    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3516         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3517                        (match_operand:SI 2 "s_register_operand" "r")
3518                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3519          VRSHLQ_M_N))
3520   ]
3521   "TARGET_HAVE_MVE"
3522   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3523   [(set_attr "type" "mve_move")
3524    (set_attr "length""8")])
3525
3526 ;;
3527 ;; [vshlq_m_r_u, vshlq_m_r_s])
3528 ;;
3529 (define_insn "mve_vshlq_m_r_<supf><mode>"
3530   [
3531    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3532         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3533                        (match_operand:SI 2 "s_register_operand" "r")
3534                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3535          VSHLQ_M_R))
3536   ]
3537   "TARGET_HAVE_MVE"
3538   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3539   [(set_attr "type" "mve_move")
3540    (set_attr "length""8")])
3541
3542 ;;
3543 ;; [vsliq_n_u, vsliq_n_s])
3544 ;;
3545 (define_insn "mve_vsliq_n_<supf><mode>"
3546   [
3547    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3548         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3549                        (match_operand:MVE_2 2 "s_register_operand" "w")
3550                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3551          VSLIQ_N))
3552   ]
3553   "TARGET_HAVE_MVE"
3554   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3555   [(set_attr "type" "mve_move")
3556 ])
3557
3558 ;;
3559 ;; [vsriq_n_u, vsriq_n_s])
3560 ;;
3561 (define_insn "mve_vsriq_n_<supf><mode>"
3562   [
3563    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3564         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3565                        (match_operand:MVE_2 2 "s_register_operand" "w")
3566                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3567          VSRIQ_N))
3568   ]
3569   "TARGET_HAVE_MVE"
3570   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3571   [(set_attr "type" "mve_move")
3572 ])
3573
3574 ;;
3575 ;; [vqdmlsdhxq_s])
3576 ;;
3577 (define_insn "mve_vqdmlsdhxq_s<mode>"
3578   [
3579    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3580         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3581                        (match_operand:MVE_2 2 "s_register_operand" "w")
3582                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3583          VQDMLSDHXQ_S))
3584   ]
3585   "TARGET_HAVE_MVE"
3586   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3587   [(set_attr "type" "mve_move")
3588 ])
3589
3590 ;;
3591 ;; [vqdmlsdhq_s])
3592 ;;
3593 (define_insn "mve_vqdmlsdhq_s<mode>"
3594   [
3595    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3596         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3597                        (match_operand:MVE_2 2 "s_register_operand" "w")
3598                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3599          VQDMLSDHQ_S))
3600   ]
3601   "TARGET_HAVE_MVE"
3602   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3603   [(set_attr "type" "mve_move")
3604 ])
3605
3606 ;;
3607 ;; [vqdmladhxq_s])
3608 ;;
3609 (define_insn "mve_vqdmladhxq_s<mode>"
3610   [
3611    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3612         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3613                        (match_operand:MVE_2 2 "s_register_operand" "w")
3614                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3615          VQDMLADHXQ_S))
3616   ]
3617   "TARGET_HAVE_MVE"
3618   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3619   [(set_attr "type" "mve_move")
3620 ])
3621
3622 ;;
3623 ;; [vqdmladhq_s])
3624 ;;
3625 (define_insn "mve_vqdmladhq_s<mode>"
3626   [
3627    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3628         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3629                        (match_operand:MVE_2 2 "s_register_operand" "w")
3630                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3631          VQDMLADHQ_S))
3632   ]
3633   "TARGET_HAVE_MVE"
3634   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3635   [(set_attr "type" "mve_move")
3636 ])
3637
3638 ;;
3639 ;; [vmlsdavaxq_s])
3640 ;;
3641 (define_insn "mve_vmlsdavaxq_s<mode>"
3642   [
3643    (set (match_operand:SI 0 "s_register_operand" "=Te")
3644         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3645                     (match_operand:MVE_2 2 "s_register_operand" "w")
3646                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3647          VMLSDAVAXQ_S))
3648   ]
3649   "TARGET_HAVE_MVE"
3650   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3651   [(set_attr "type" "mve_move")
3652 ])
3653
3654 ;;
3655 ;; [vmlsdavaq_s])
3656 ;;
3657 (define_insn "mve_vmlsdavaq_s<mode>"
3658   [
3659    (set (match_operand:SI 0 "s_register_operand" "=Te")
3660         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3661                     (match_operand:MVE_2 2 "s_register_operand" "w")
3662                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3663          VMLSDAVAQ_S))
3664   ]
3665   "TARGET_HAVE_MVE"
3666   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
3667   [(set_attr "type" "mve_move")
3668 ])
3669
3670 ;;
3671 ;; [vmladavaxq_s])
3672 ;;
3673 (define_insn "mve_vmladavaxq_s<mode>"
3674   [
3675    (set (match_operand:SI 0 "s_register_operand" "=Te")
3676         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3677                     (match_operand:MVE_2 2 "s_register_operand" "w")
3678                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3679          VMLADAVAXQ_S))
3680   ]
3681   "TARGET_HAVE_MVE"
3682   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3683   [(set_attr "type" "mve_move")
3684 ])
3685 ;;
3686 ;; [vabsq_m_f])
3687 ;;
3688 (define_insn "mve_vabsq_m_f<mode>"
3689   [
3690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3691         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3692                        (match_operand:MVE_0 2 "s_register_operand" "w")
3693                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3694          VABSQ_M_F))
3695   ]
3696   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3697   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
3698   [(set_attr "type" "mve_move")
3699    (set_attr "length""8")])
3700
3701 ;;
3702 ;; [vaddlvaq_p_s vaddlvaq_p_u])
3703 ;;
3704 (define_insn "mve_vaddlvaq_p_<supf>v4si"
3705   [
3706    (set (match_operand:DI 0 "s_register_operand" "=r")
3707         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3708                        (match_operand:V4SI 2 "s_register_operand" "w")
3709                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3710          VADDLVAQ_P))
3711   ]
3712   "TARGET_HAVE_MVE"
3713   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
3714   [(set_attr "type" "mve_move")
3715    (set_attr "length""8")])
3716 ;;
3717 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
3718 ;;
3719 (define_insn "mve_vcmlaq<mve_rot><mode>"
3720   [
3721    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
3722         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
3723                     (unspec:MVE_0
3724                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
3725                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
3726                      VCMLA)))
3727   ]
3728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3729   "@
3730    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
3731    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
3732   [(set_attr "type" "mve_move")
3733 ])
3734
3735 ;;
3736 ;; [vcmpeqq_m_n_f])
3737 ;;
3738 (define_insn "mve_vcmpeqq_m_n_f<mode>"
3739   [
3740    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3741         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3742                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3743                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3744          VCMPEQQ_M_N_F))
3745   ]
3746   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3747   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
3748   [(set_attr "type" "mve_move")
3749    (set_attr "length""8")])
3750
3751 ;;
3752 ;; [vcmpgeq_m_f])
3753 ;;
3754 (define_insn "mve_vcmpgeq_m_f<mode>"
3755   [
3756    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3757         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3758                        (match_operand:MVE_0 2 "s_register_operand" "w")
3759                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3760          VCMPGEQ_M_F))
3761   ]
3762   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3763   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
3764   [(set_attr "type" "mve_move")
3765    (set_attr "length""8")])
3766
3767 ;;
3768 ;; [vcmpgeq_m_n_f])
3769 ;;
3770 (define_insn "mve_vcmpgeq_m_n_f<mode>"
3771   [
3772    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3773         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3774                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3775                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3776          VCMPGEQ_M_N_F))
3777   ]
3778   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3779   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
3780   [(set_attr "type" "mve_move")
3781    (set_attr "length""8")])
3782
3783 ;;
3784 ;; [vcmpgtq_m_f])
3785 ;;
3786 (define_insn "mve_vcmpgtq_m_f<mode>"
3787   [
3788    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3789         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3790                        (match_operand:MVE_0 2 "s_register_operand" "w")
3791                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3792          VCMPGTQ_M_F))
3793   ]
3794   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3795   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
3796   [(set_attr "type" "mve_move")
3797    (set_attr "length""8")])
3798
3799 ;;
3800 ;; [vcmpgtq_m_n_f])
3801 ;;
3802 (define_insn "mve_vcmpgtq_m_n_f<mode>"
3803   [
3804    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3805         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3806                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3807                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3808          VCMPGTQ_M_N_F))
3809   ]
3810   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3811   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
3812   [(set_attr "type" "mve_move")
3813    (set_attr "length""8")])
3814
3815 ;;
3816 ;; [vcmpleq_m_f])
3817 ;;
3818 (define_insn "mve_vcmpleq_m_f<mode>"
3819   [
3820    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3821         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3822                        (match_operand:MVE_0 2 "s_register_operand" "w")
3823                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3824          VCMPLEQ_M_F))
3825   ]
3826   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3827   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
3828   [(set_attr "type" "mve_move")
3829    (set_attr "length""8")])
3830
3831 ;;
3832 ;; [vcmpleq_m_n_f])
3833 ;;
3834 (define_insn "mve_vcmpleq_m_n_f<mode>"
3835   [
3836    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3837         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3838                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3839                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3840          VCMPLEQ_M_N_F))
3841   ]
3842   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3843   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
3844   [(set_attr "type" "mve_move")
3845    (set_attr "length""8")])
3846
3847 ;;
3848 ;; [vcmpltq_m_f])
3849 ;;
3850 (define_insn "mve_vcmpltq_m_f<mode>"
3851   [
3852    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3853         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3854                        (match_operand:MVE_0 2 "s_register_operand" "w")
3855                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3856          VCMPLTQ_M_F))
3857   ]
3858   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3859   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
3860   [(set_attr "type" "mve_move")
3861    (set_attr "length""8")])
3862
3863 ;;
3864 ;; [vcmpltq_m_n_f])
3865 ;;
3866 (define_insn "mve_vcmpltq_m_n_f<mode>"
3867   [
3868    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3869         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3870                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3871                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3872          VCMPLTQ_M_N_F))
3873   ]
3874   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3875   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
3876   [(set_attr "type" "mve_move")
3877    (set_attr "length""8")])
3878
3879 ;;
3880 ;; [vcmpneq_m_f])
3881 ;;
3882 (define_insn "mve_vcmpneq_m_f<mode>"
3883   [
3884    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3885         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3886                        (match_operand:MVE_0 2 "s_register_operand" "w")
3887                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3888          VCMPNEQ_M_F))
3889   ]
3890   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3891   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
3892   [(set_attr "type" "mve_move")
3893    (set_attr "length""8")])
3894
3895 ;;
3896 ;; [vcmpneq_m_n_f])
3897 ;;
3898 (define_insn "mve_vcmpneq_m_n_f<mode>"
3899   [
3900    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3901         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3902                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3903                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3904          VCMPNEQ_M_N_F))
3905   ]
3906   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3907   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
3908   [(set_attr "type" "mve_move")
3909    (set_attr "length""8")])
3910
3911 ;;
3912 ;; [vcvtbq_m_f16_f32])
3913 ;;
3914 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
3915   [
3916    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3917         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3918                        (match_operand:V4SF 2 "s_register_operand" "w")
3919                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3920          VCVTBQ_M_F16_F32))
3921   ]
3922   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3923   "vpst\;vcvtbt.f16.f32 %q0, %q2"
3924   [(set_attr "type" "mve_move")
3925    (set_attr "length""8")])
3926
3927 ;;
3928 ;; [vcvtbq_m_f32_f16])
3929 ;;
3930 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
3931   [
3932    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3933         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3934                        (match_operand:V8HF 2 "s_register_operand" "w")
3935                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3936          VCVTBQ_M_F32_F16))
3937   ]
3938   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3939   "vpst\;vcvtbt.f32.f16 %q0, %q2"
3940   [(set_attr "type" "mve_move")
3941    (set_attr "length""8")])
3942
3943 ;;
3944 ;; [vcvttq_m_f16_f32])
3945 ;;
3946 (define_insn "mve_vcvttq_m_f16_f32v8hf"
3947   [
3948    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3949         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3950                        (match_operand:V4SF 2 "s_register_operand" "w")
3951                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3952          VCVTTQ_M_F16_F32))
3953   ]
3954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3955   "vpst\;vcvttt.f16.f32 %q0, %q2"
3956   [(set_attr "type" "mve_move")
3957    (set_attr "length""8")])
3958
3959 ;;
3960 ;; [vcvttq_m_f32_f16])
3961 ;;
3962 (define_insn "mve_vcvttq_m_f32_f16v4sf"
3963   [
3964    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3965         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3966                        (match_operand:V8HF 2 "s_register_operand" "w")
3967                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3968          VCVTTQ_M_F32_F16))
3969   ]
3970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3971   "vpst\;vcvttt.f32.f16 %q0, %q2"
3972   [(set_attr "type" "mve_move")
3973    (set_attr "length""8")])
3974
3975 ;;
3976 ;; [vdupq_m_n_f])
3977 ;;
3978 (define_insn "mve_vdupq_m_n_f<mode>"
3979   [
3980    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3981         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3982                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3983                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3984          VDUPQ_M_N_F))
3985   ]
3986   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3987   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3988   [(set_attr "type" "mve_move")
3989    (set_attr "length""8")])
3990
3991 ;;
3992 ;; [vfmaq_f])
3993 ;;
3994 (define_insn "mve_vfmaq_f<mode>"
3995   [
3996    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3997         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3998                        (match_operand:MVE_0 2 "s_register_operand" "w")
3999                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4000          VFMAQ_F))
4001   ]
4002   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4003   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4004   [(set_attr "type" "mve_move")
4005 ])
4006
4007 ;;
4008 ;; [vfmaq_n_f])
4009 ;;
4010 (define_insn "mve_vfmaq_n_f<mode>"
4011   [
4012    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4013         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4014                        (match_operand:MVE_0 2 "s_register_operand" "w")
4015                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4016          VFMAQ_N_F))
4017   ]
4018   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4019   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4020   [(set_attr "type" "mve_move")
4021 ])
4022
4023 ;;
4024 ;; [vfmasq_n_f])
4025 ;;
4026 (define_insn "mve_vfmasq_n_f<mode>"
4027   [
4028    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4029         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4030                        (match_operand:MVE_0 2 "s_register_operand" "w")
4031                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4032          VFMASQ_N_F))
4033   ]
4034   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4035   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4036   [(set_attr "type" "mve_move")
4037 ])
4038 ;;
4039 ;; [vfmsq_f])
4040 ;;
4041 (define_insn "mve_vfmsq_f<mode>"
4042   [
4043    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4044         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4045                        (match_operand:MVE_0 2 "s_register_operand" "w")
4046                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4047          VFMSQ_F))
4048   ]
4049   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4050   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4051   [(set_attr "type" "mve_move")
4052 ])
4053
4054 ;;
4055 ;; [vmaxnmaq_m_f])
4056 ;;
4057 (define_insn "mve_vmaxnmaq_m_f<mode>"
4058   [
4059    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4060         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4061                        (match_operand:MVE_0 2 "s_register_operand" "w")
4062                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4063          VMAXNMAQ_M_F))
4064   ]
4065   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4066   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4067   [(set_attr "type" "mve_move")
4068    (set_attr "length""8")])
4069 ;;
4070 ;; [vmaxnmavq_p_f])
4071 ;;
4072 (define_insn "mve_vmaxnmavq_p_f<mode>"
4073   [
4074    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4075         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4076                        (match_operand:MVE_0 2 "s_register_operand" "w")
4077                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4078          VMAXNMAVQ_P_F))
4079   ]
4080   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4081   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4082   [(set_attr "type" "mve_move")
4083    (set_attr "length""8")])
4084
4085 ;;
4086 ;; [vmaxnmvq_p_f])
4087 ;;
4088 (define_insn "mve_vmaxnmvq_p_f<mode>"
4089   [
4090    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4091         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4092                        (match_operand:MVE_0 2 "s_register_operand" "w")
4093                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4094          VMAXNMVQ_P_F))
4095   ]
4096   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4097   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4098   [(set_attr "type" "mve_move")
4099    (set_attr "length""8")])
4100 ;;
4101 ;; [vminnmaq_m_f])
4102 ;;
4103 (define_insn "mve_vminnmaq_m_f<mode>"
4104   [
4105    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4106         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4107                        (match_operand:MVE_0 2 "s_register_operand" "w")
4108                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4109          VMINNMAQ_M_F))
4110   ]
4111   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4112   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4113   [(set_attr "type" "mve_move")
4114    (set_attr "length""8")])
4115
4116 ;;
4117 ;; [vminnmavq_p_f])
4118 ;;
4119 (define_insn "mve_vminnmavq_p_f<mode>"
4120   [
4121    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4122         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4123                        (match_operand:MVE_0 2 "s_register_operand" "w")
4124                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4125          VMINNMAVQ_P_F))
4126   ]
4127   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4128   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4129   [(set_attr "type" "mve_move")
4130    (set_attr "length""8")])
4131 ;;
4132 ;; [vminnmvq_p_f])
4133 ;;
4134 (define_insn "mve_vminnmvq_p_f<mode>"
4135   [
4136    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4137         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4138                        (match_operand:MVE_0 2 "s_register_operand" "w")
4139                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4140          VMINNMVQ_P_F))
4141   ]
4142   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4143   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4144   [(set_attr "type" "mve_move")
4145    (set_attr "length""8")])
4146
4147 ;;
4148 ;; [vmlaldavaq_s, vmlaldavaq_u])
4149 ;;
4150 (define_insn "mve_vmlaldavaq_<supf><mode>"
4151   [
4152    (set (match_operand:DI 0 "s_register_operand" "=r")
4153         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4154                        (match_operand:MVE_5 2 "s_register_operand" "w")
4155                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4156          VMLALDAVAQ))
4157   ]
4158   "TARGET_HAVE_MVE"
4159   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4160   [(set_attr "type" "mve_move")
4161 ])
4162
4163 ;;
4164 ;; [vmlaldavaxq_s])
4165 ;;
4166 (define_insn "mve_vmlaldavaxq_s<mode>"
4167   [
4168    (set (match_operand:DI 0 "s_register_operand" "=r")
4169         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4170                        (match_operand:MVE_5 2 "s_register_operand" "w")
4171                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4172          VMLALDAVAXQ_S))
4173   ]
4174   "TARGET_HAVE_MVE"
4175   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4176   [(set_attr "type" "mve_move")
4177 ])
4178
4179 ;;
4180 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4181 ;;
4182 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4183   [
4184    (set (match_operand:DI 0 "s_register_operand" "=r")
4185         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4186                        (match_operand:MVE_5 2 "s_register_operand" "w")
4187                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4188          VMLALDAVQ_P))
4189   ]
4190   "TARGET_HAVE_MVE"
4191   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4192   [(set_attr "type" "mve_move")
4193    (set_attr "length""8")])
4194
4195 ;;
4196 ;; [vmlaldavxq_p_s])
4197 ;;
4198 (define_insn "mve_vmlaldavxq_p_s<mode>"
4199   [
4200    (set (match_operand:DI 0 "s_register_operand" "=r")
4201         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4202                        (match_operand:MVE_5 2 "s_register_operand" "w")
4203                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4204          VMLALDAVXQ_P_S))
4205   ]
4206   "TARGET_HAVE_MVE"
4207   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4208   [(set_attr "type" "mve_move")
4209    (set_attr "length""8")])
4210 ;;
4211 ;; [vmlsldavaq_s])
4212 ;;
4213 (define_insn "mve_vmlsldavaq_s<mode>"
4214   [
4215    (set (match_operand:DI 0 "s_register_operand" "=r")
4216         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4217                        (match_operand:MVE_5 2 "s_register_operand" "w")
4218                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4219          VMLSLDAVAQ_S))
4220   ]
4221   "TARGET_HAVE_MVE"
4222   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4223   [(set_attr "type" "mve_move")
4224 ])
4225
4226 ;;
4227 ;; [vmlsldavaxq_s])
4228 ;;
4229 (define_insn "mve_vmlsldavaxq_s<mode>"
4230   [
4231    (set (match_operand:DI 0 "s_register_operand" "=r")
4232         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4233                        (match_operand:MVE_5 2 "s_register_operand" "w")
4234                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4235          VMLSLDAVAXQ_S))
4236   ]
4237   "TARGET_HAVE_MVE"
4238   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4239   [(set_attr "type" "mve_move")
4240 ])
4241
4242 ;;
4243 ;; [vmlsldavq_p_s])
4244 ;;
4245 (define_insn "mve_vmlsldavq_p_s<mode>"
4246   [
4247    (set (match_operand:DI 0 "s_register_operand" "=r")
4248         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4249                        (match_operand:MVE_5 2 "s_register_operand" "w")
4250                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4251          VMLSLDAVQ_P_S))
4252   ]
4253   "TARGET_HAVE_MVE"
4254   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4255   [(set_attr "type" "mve_move")
4256    (set_attr "length""8")])
4257
4258 ;;
4259 ;; [vmlsldavxq_p_s])
4260 ;;
4261 (define_insn "mve_vmlsldavxq_p_s<mode>"
4262   [
4263    (set (match_operand:DI 0 "s_register_operand" "=r")
4264         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4265                        (match_operand:MVE_5 2 "s_register_operand" "w")
4266                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4267          VMLSLDAVXQ_P_S))
4268   ]
4269   "TARGET_HAVE_MVE"
4270   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4271   [(set_attr "type" "mve_move")
4272    (set_attr "length""8")])
4273 ;;
4274 ;; [vmovlbq_m_u, vmovlbq_m_s])
4275 ;;
4276 (define_insn "mve_vmovlbq_m_<supf><mode>"
4277   [
4278    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4279         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4280                        (match_operand:MVE_3 2 "s_register_operand" "w")
4281                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4282          VMOVLBQ_M))
4283   ]
4284   "TARGET_HAVE_MVE"
4285   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4286   [(set_attr "type" "mve_move")
4287    (set_attr "length""8")])
4288 ;;
4289 ;; [vmovltq_m_u, vmovltq_m_s])
4290 ;;
4291 (define_insn "mve_vmovltq_m_<supf><mode>"
4292   [
4293    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4294         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4295                        (match_operand:MVE_3 2 "s_register_operand" "w")
4296                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4297          VMOVLTQ_M))
4298   ]
4299   "TARGET_HAVE_MVE"
4300   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4301   [(set_attr "type" "mve_move")
4302    (set_attr "length""8")])
4303 ;;
4304 ;; [vmovnbq_m_u, vmovnbq_m_s])
4305 ;;
4306 (define_insn "mve_vmovnbq_m_<supf><mode>"
4307   [
4308    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4309         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4310                        (match_operand:MVE_5 2 "s_register_operand" "w")
4311                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4312          VMOVNBQ_M))
4313   ]
4314   "TARGET_HAVE_MVE"
4315   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4316   [(set_attr "type" "mve_move")
4317    (set_attr "length""8")])
4318
4319 ;;
4320 ;; [vmovntq_m_u, vmovntq_m_s])
4321 ;;
4322 (define_insn "mve_vmovntq_m_<supf><mode>"
4323   [
4324    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4325         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4326                        (match_operand:MVE_5 2 "s_register_operand" "w")
4327                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4328          VMOVNTQ_M))
4329   ]
4330   "TARGET_HAVE_MVE"
4331   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4332   [(set_attr "type" "mve_move")
4333    (set_attr "length""8")])
4334
4335 ;;
4336 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4337 ;;
4338 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4339   [
4340    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4341         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4342                        (match_operand:SI 2 "immediate_operand" "i")
4343                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4344          VMVNQ_M_N))
4345   ]
4346   "TARGET_HAVE_MVE"
4347   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4348   [(set_attr "type" "mve_move")
4349    (set_attr "length""8")])
4350 ;;
4351 ;; [vnegq_m_f])
4352 ;;
4353 (define_insn "mve_vnegq_m_f<mode>"
4354   [
4355    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4356         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4357                        (match_operand:MVE_0 2 "s_register_operand" "w")
4358                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4359          VNEGQ_M_F))
4360   ]
4361   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4362   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4363   [(set_attr "type" "mve_move")
4364    (set_attr "length""8")])
4365
4366 ;;
4367 ;; [vorrq_m_n_s, vorrq_m_n_u])
4368 ;;
4369 (define_insn "mve_vorrq_m_n_<supf><mode>"
4370   [
4371    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4372         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4373                        (match_operand:SI 2 "immediate_operand" "i")
4374                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4375          VORRQ_M_N))
4376   ]
4377   "TARGET_HAVE_MVE"
4378   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4379   [(set_attr "type" "mve_move")
4380    (set_attr "length""8")])
4381 ;;
4382 ;; [vpselq_f])
4383 ;;
4384 (define_insn "@mve_vpselq_f<mode>"
4385   [
4386    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4387         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4388                        (match_operand:MVE_0 2 "s_register_operand" "w")
4389                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4390          VPSELQ_F))
4391   ]
4392   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4393   "vpsel %q0, %q1, %q2"
4394   [(set_attr "type" "mve_move")
4395 ])
4396
4397 ;;
4398 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4399 ;;
4400 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4401   [
4402    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4403         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4404                        (match_operand:MVE_5 2 "s_register_operand" "w")
4405                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4406          VQMOVNBQ_M))
4407   ]
4408   "TARGET_HAVE_MVE"
4409   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4410   [(set_attr "type" "mve_move")
4411    (set_attr "length""8")])
4412
4413 ;;
4414 ;; [vqmovntq_m_u, vqmovntq_m_s])
4415 ;;
4416 (define_insn "mve_vqmovntq_m_<supf><mode>"
4417   [
4418    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4419         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4420                        (match_operand:MVE_5 2 "s_register_operand" "w")
4421                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4422          VQMOVNTQ_M))
4423   ]
4424   "TARGET_HAVE_MVE"
4425   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4426   [(set_attr "type" "mve_move")
4427    (set_attr "length""8")])
4428
4429 ;;
4430 ;; [vqmovunbq_m_s])
4431 ;;
4432 (define_insn "mve_vqmovunbq_m_s<mode>"
4433   [
4434    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4435         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4436                        (match_operand:MVE_5 2 "s_register_operand" "w")
4437                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4438          VQMOVUNBQ_M_S))
4439   ]
4440   "TARGET_HAVE_MVE"
4441   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4442   [(set_attr "type" "mve_move")
4443    (set_attr "length""8")])
4444
4445 ;;
4446 ;; [vqmovuntq_m_s])
4447 ;;
4448 (define_insn "mve_vqmovuntq_m_s<mode>"
4449   [
4450    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4451         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4452                        (match_operand:MVE_5 2 "s_register_operand" "w")
4453                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4454          VQMOVUNTQ_M_S))
4455   ]
4456   "TARGET_HAVE_MVE"
4457   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4458   [(set_attr "type" "mve_move")
4459    (set_attr "length""8")])
4460
4461 ;;
4462 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4463 ;;
4464 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4465   [
4466    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4467         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4468                        (match_operand:MVE_5 2 "s_register_operand" "w")
4469                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4470          VQRSHRNTQ_N))
4471   ]
4472   "TARGET_HAVE_MVE"
4473   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4474   [(set_attr "type" "mve_move")
4475 ])
4476
4477 ;;
4478 ;; [vqrshruntq_n_s])
4479 ;;
4480 (define_insn "mve_vqrshruntq_n_s<mode>"
4481   [
4482    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4483         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4484                        (match_operand:MVE_5 2 "s_register_operand" "w")
4485                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4486          VQRSHRUNTQ_N_S))
4487   ]
4488   "TARGET_HAVE_MVE"
4489   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4490   [(set_attr "type" "mve_move")
4491 ])
4492
4493 ;;
4494 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4495 ;;
4496 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4497   [
4498    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4499         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4500                        (match_operand:MVE_5 2 "s_register_operand" "w")
4501                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4502          VQSHRNBQ_N))
4503   ]
4504   "TARGET_HAVE_MVE"
4505   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4506   [(set_attr "type" "mve_move")
4507 ])
4508
4509 ;;
4510 ;; [vqshrntq_n_u, vqshrntq_n_s])
4511 ;;
4512 (define_insn "mve_vqshrntq_n_<supf><mode>"
4513   [
4514    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4515         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4516                        (match_operand:MVE_5 2 "s_register_operand" "w")
4517                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4518          VQSHRNTQ_N))
4519   ]
4520   "TARGET_HAVE_MVE"
4521   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4522   [(set_attr "type" "mve_move")
4523 ])
4524
4525 ;;
4526 ;; [vqshrunbq_n_s])
4527 ;;
4528 (define_insn "mve_vqshrunbq_n_s<mode>"
4529   [
4530    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4531         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4532                        (match_operand:MVE_5 2 "s_register_operand" "w")
4533                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4534          VQSHRUNBQ_N_S))
4535   ]
4536   "TARGET_HAVE_MVE"
4537   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4538   [(set_attr "type" "mve_move")
4539 ])
4540
4541 ;;
4542 ;; [vqshruntq_n_s])
4543 ;;
4544 (define_insn "mve_vqshruntq_n_s<mode>"
4545   [
4546    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4547         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4548                        (match_operand:MVE_5 2 "s_register_operand" "w")
4549                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4550          VQSHRUNTQ_N_S))
4551   ]
4552   "TARGET_HAVE_MVE"
4553   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4554   [(set_attr "type" "mve_move")
4555 ])
4556
4557 ;;
4558 ;; [vrev32q_m_f])
4559 ;;
4560 (define_insn "mve_vrev32q_m_fv8hf"
4561   [
4562    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4563         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4564                        (match_operand:V8HF 2 "s_register_operand" "w")
4565                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4566          VREV32Q_M_F))
4567   ]
4568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4569   "vpst\;vrev32t.16 %q0, %q2"
4570   [(set_attr "type" "mve_move")
4571    (set_attr "length""8")])
4572
4573 ;;
4574 ;; [vrev32q_m_s, vrev32q_m_u])
4575 ;;
4576 (define_insn "mve_vrev32q_m_<supf><mode>"
4577   [
4578    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4579         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4580                        (match_operand:MVE_3 2 "s_register_operand" "w")
4581                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4582          VREV32Q_M))
4583   ]
4584   "TARGET_HAVE_MVE"
4585   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4586   [(set_attr "type" "mve_move")
4587    (set_attr "length""8")])
4588
4589 ;;
4590 ;; [vrev64q_m_f])
4591 ;;
4592 (define_insn "mve_vrev64q_m_f<mode>"
4593   [
4594    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4595         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4596                        (match_operand:MVE_0 2 "s_register_operand" "w")
4597                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4598          VREV64Q_M_F))
4599   ]
4600   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4601   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4602   [(set_attr "type" "mve_move")
4603    (set_attr "length""8")])
4604
4605 ;;
4606 ;; [vrmlaldavhaxq_s])
4607 ;;
4608 (define_insn "mve_vrmlaldavhaxq_sv4si"
4609   [
4610    (set (match_operand:DI 0 "s_register_operand" "=r")
4611         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4612                        (match_operand:V4SI 2 "s_register_operand" "w")
4613                        (match_operand:V4SI 3 "s_register_operand" "w")]
4614          VRMLALDAVHAXQ_S))
4615   ]
4616   "TARGET_HAVE_MVE"
4617   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
4618   [(set_attr "type" "mve_move")
4619 ])
4620
4621 ;;
4622 ;; [vrmlaldavhxq_p_s])
4623 ;;
4624 (define_insn "mve_vrmlaldavhxq_p_sv4si"
4625   [
4626    (set (match_operand:DI 0 "s_register_operand" "=r")
4627         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4628                        (match_operand:V4SI 2 "s_register_operand" "w")
4629                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4630          VRMLALDAVHXQ_P_S))
4631   ]
4632   "TARGET_HAVE_MVE"
4633   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
4634   [(set_attr "type" "mve_move")
4635    (set_attr "length""8")])
4636
4637 ;;
4638 ;; [vrmlsldavhaxq_s])
4639 ;;
4640 (define_insn "mve_vrmlsldavhaxq_sv4si"
4641   [
4642    (set (match_operand:DI 0 "s_register_operand" "=r")
4643         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4644                        (match_operand:V4SI 2 "s_register_operand" "w")
4645                        (match_operand:V4SI 3 "s_register_operand" "w")]
4646          VRMLSLDAVHAXQ_S))
4647   ]
4648   "TARGET_HAVE_MVE"
4649   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
4650   [(set_attr "type" "mve_move")
4651 ])
4652
4653 ;;
4654 ;; [vrmlsldavhq_p_s])
4655 ;;
4656 (define_insn "mve_vrmlsldavhq_p_sv4si"
4657   [
4658    (set (match_operand:DI 0 "s_register_operand" "=r")
4659         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4660                        (match_operand:V4SI 2 "s_register_operand" "w")
4661                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4662          VRMLSLDAVHQ_P_S))
4663   ]
4664   "TARGET_HAVE_MVE"
4665   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
4666   [(set_attr "type" "mve_move")
4667    (set_attr "length""8")])
4668
4669 ;;
4670 ;; [vrmlsldavhxq_p_s])
4671 ;;
4672 (define_insn "mve_vrmlsldavhxq_p_sv4si"
4673   [
4674    (set (match_operand:DI 0 "s_register_operand" "=r")
4675         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4676                        (match_operand:V4SI 2 "s_register_operand" "w")
4677                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4678          VRMLSLDAVHXQ_P_S))
4679   ]
4680   "TARGET_HAVE_MVE"
4681   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
4682   [(set_attr "type" "mve_move")
4683    (set_attr "length""8")])
4684
4685 ;;
4686 ;; [vrndaq_m_f])
4687 ;;
4688 (define_insn "mve_vrndaq_m_f<mode>"
4689   [
4690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4691         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4692                        (match_operand:MVE_0 2 "s_register_operand" "w")
4693                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4694          VRNDAQ_M_F))
4695   ]
4696   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4697   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
4698   [(set_attr "type" "mve_move")
4699    (set_attr "length""8")])
4700
4701 ;;
4702 ;; [vrndmq_m_f])
4703 ;;
4704 (define_insn "mve_vrndmq_m_f<mode>"
4705   [
4706    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4707         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4708                        (match_operand:MVE_0 2 "s_register_operand" "w")
4709                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4710          VRNDMQ_M_F))
4711   ]
4712   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4713   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
4714   [(set_attr "type" "mve_move")
4715    (set_attr "length""8")])
4716
4717 ;;
4718 ;; [vrndnq_m_f])
4719 ;;
4720 (define_insn "mve_vrndnq_m_f<mode>"
4721   [
4722    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4723         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4724                        (match_operand:MVE_0 2 "s_register_operand" "w")
4725                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4726          VRNDNQ_M_F))
4727   ]
4728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4729   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
4730   [(set_attr "type" "mve_move")
4731    (set_attr "length""8")])
4732
4733 ;;
4734 ;; [vrndpq_m_f])
4735 ;;
4736 (define_insn "mve_vrndpq_m_f<mode>"
4737   [
4738    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4739         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4740                        (match_operand:MVE_0 2 "s_register_operand" "w")
4741                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4742          VRNDPQ_M_F))
4743   ]
4744   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4745   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
4746   [(set_attr "type" "mve_move")
4747    (set_attr "length""8")])
4748
4749 ;;
4750 ;; [vrndxq_m_f])
4751 ;;
4752 (define_insn "mve_vrndxq_m_f<mode>"
4753   [
4754    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4755         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4756                        (match_operand:MVE_0 2 "s_register_operand" "w")
4757                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4758          VRNDXQ_M_F))
4759   ]
4760   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4761   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
4762   [(set_attr "type" "mve_move")
4763    (set_attr "length""8")])
4764
4765 ;;
4766 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
4767 ;;
4768 (define_insn "mve_vrshrnbq_n_<supf><mode>"
4769   [
4770    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4771         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4772                        (match_operand:MVE_5 2 "s_register_operand" "w")
4773                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4774          VRSHRNBQ_N))
4775   ]
4776   "TARGET_HAVE_MVE"
4777   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
4778   [(set_attr "type" "mve_move")
4779 ])
4780
4781 ;;
4782 ;; [vrshrntq_n_u, vrshrntq_n_s])
4783 ;;
4784 (define_insn "mve_vrshrntq_n_<supf><mode>"
4785   [
4786    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4787         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4788                        (match_operand:MVE_5 2 "s_register_operand" "w")
4789                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4790          VRSHRNTQ_N))
4791   ]
4792   "TARGET_HAVE_MVE"
4793   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
4794   [(set_attr "type" "mve_move")
4795 ])
4796
4797 ;;
4798 ;; [vshrnbq_n_u, vshrnbq_n_s])
4799 ;;
4800 (define_insn "mve_vshrnbq_n_<supf><mode>"
4801   [
4802    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4803         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4804                        (match_operand:MVE_5 2 "s_register_operand" "w")
4805                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4806          VSHRNBQ_N))
4807   ]
4808   "TARGET_HAVE_MVE"
4809   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
4810   [(set_attr "type" "mve_move")
4811 ])
4812
4813 ;;
4814 ;; [vshrntq_n_s, vshrntq_n_u])
4815 ;;
4816 (define_insn "mve_vshrntq_n_<supf><mode>"
4817   [
4818    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4819         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4820                                  (match_operand:MVE_5 2 "s_register_operand" "w")
4821                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4822          VSHRNTQ_N))
4823   ]
4824   "TARGET_HAVE_MVE"
4825   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
4826   [(set_attr "type" "mve_move")
4827 ])
4828
4829 ;;
4830 ;; [vcvtmq_m_s, vcvtmq_m_u])
4831 ;;
4832 (define_insn "mve_vcvtmq_m_<supf><mode>"
4833   [
4834    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4835         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4836                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4837                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4838          VCVTMQ_M))
4839   ]
4840   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4841   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4842   [(set_attr "type" "mve_move")
4843    (set_attr "length""8")])
4844
4845 ;;
4846 ;; [vcvtpq_m_u, vcvtpq_m_s])
4847 ;;
4848 (define_insn "mve_vcvtpq_m_<supf><mode>"
4849   [
4850    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4851         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4852                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4853                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4854          VCVTPQ_M))
4855   ]
4856   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4857   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4858   [(set_attr "type" "mve_move")
4859    (set_attr "length""8")])
4860
4861 ;;
4862 ;; [vcvtnq_m_s, vcvtnq_m_u])
4863 ;;
4864 (define_insn "mve_vcvtnq_m_<supf><mode>"
4865   [
4866    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4867         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4868                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4869                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4870          VCVTNQ_M))
4871   ]
4872   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4873   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4874   [(set_attr "type" "mve_move")
4875    (set_attr "length""8")])
4876
4877 ;;
4878 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
4879 ;;
4880 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
4881   [
4882    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4883         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4884                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4885                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
4886                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4887          VCVTQ_M_N_FROM_F))
4888   ]
4889   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4890   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
4891   [(set_attr "type" "mve_move")
4892    (set_attr "length""8")])
4893
4894 ;;
4895 ;; [vrev16q_m_u, vrev16q_m_s])
4896 ;;
4897 (define_insn "mve_vrev16q_m_<supf>v16qi"
4898   [
4899    (set (match_operand:V16QI 0 "s_register_operand" "=w")
4900         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
4901                        (match_operand:V16QI 2 "s_register_operand" "w")
4902                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4903          VREV16Q_M))
4904   ]
4905   "TARGET_HAVE_MVE"
4906   "vpst\;vrev16t.8 %q0, %q2"
4907   [(set_attr "type" "mve_move")
4908    (set_attr "length""8")])
4909
4910 ;;
4911 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
4912 ;;
4913 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
4914   [
4915    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4916         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4917                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4918                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4919          VCVTQ_M_FROM_F))
4920   ]
4921   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4922   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4923   [(set_attr "type" "mve_move")
4924    (set_attr "length""8")])
4925
4926 ;;
4927 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
4928 ;;
4929 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
4930   [
4931    (set (match_operand:DI 0 "s_register_operand" "=r")
4932         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4933                     (match_operand:V4SI 2 "s_register_operand" "w")
4934                     (match_operand:HI 3 "vpr_register_operand" "Up")]
4935          VRMLALDAVHQ_P))
4936   ]
4937   "TARGET_HAVE_MVE"
4938   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
4939   [(set_attr "type" "mve_move")
4940    (set_attr "length""8")])
4941
4942 ;;
4943 ;; [vrmlsldavhaq_s])
4944 ;;
4945 (define_insn "mve_vrmlsldavhaq_sv4si"
4946   [
4947    (set (match_operand:DI 0 "s_register_operand" "=r")
4948         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4949                     (match_operand:V4SI 2 "s_register_operand" "w")
4950                     (match_operand:V4SI 3 "s_register_operand" "w")]
4951          VRMLSLDAVHAQ_S))
4952   ]
4953   "TARGET_HAVE_MVE"
4954   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
4955   [(set_attr "type" "mve_move")
4956 ])
4957
4958 ;;
4959 ;; [vabavq_p_s, vabavq_p_u])
4960 ;;
4961 (define_insn "mve_vabavq_p_<supf><mode>"
4962   [
4963    (set (match_operand:SI 0 "s_register_operand" "=r")
4964         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4965                     (match_operand:MVE_2 2 "s_register_operand" "w")
4966                     (match_operand:MVE_2 3 "s_register_operand" "w")
4967                     (match_operand:HI 4 "vpr_register_operand" "Up")]
4968          VABAVQ_P))
4969   ]
4970   "TARGET_HAVE_MVE"
4971   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
4972   [(set_attr "type" "mve_move")
4973 ])
4974
4975 ;;
4976 ;; [vqshluq_m_n_s])
4977 ;;
4978 (define_insn "mve_vqshluq_m_n_s<mode>"
4979   [
4980    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4981         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4982                        (match_operand:MVE_2 2 "s_register_operand" "w")
4983                        (match_operand:SI 3 "mve_imm_7" "Ra")
4984                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4985          VQSHLUQ_M_N_S))
4986   ]
4987   "TARGET_HAVE_MVE"
4988   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
4989   [(set_attr "type" "mve_move")])
4990
4991 ;;
4992 ;; [vshlq_m_s, vshlq_m_u])
4993 ;;
4994 (define_insn "mve_vshlq_m_<supf><mode>"
4995   [
4996    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4997         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4998                        (match_operand:MVE_2 2 "s_register_operand" "w")
4999                        (match_operand:MVE_2 3 "s_register_operand" "w")
5000                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5001          VSHLQ_M))
5002   ]
5003   "TARGET_HAVE_MVE"
5004   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5005   [(set_attr "type" "mve_move")])
5006
5007 ;;
5008 ;; [vsriq_m_n_s, vsriq_m_n_u])
5009 ;;
5010 (define_insn "mve_vsriq_m_n_<supf><mode>"
5011   [
5012    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5013         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5014                        (match_operand:MVE_2 2 "s_register_operand" "w")
5015                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5016                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5017          VSRIQ_M_N))
5018   ]
5019   "TARGET_HAVE_MVE"
5020   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5021   [(set_attr "type" "mve_move")])
5022
5023 ;;
5024 ;; [vsubq_m_u, vsubq_m_s])
5025 ;;
5026 (define_insn "mve_vsubq_m_<supf><mode>"
5027   [
5028    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5029         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5030                        (match_operand:MVE_2 2 "s_register_operand" "w")
5031                        (match_operand:MVE_2 3 "s_register_operand" "w")
5032                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5033          VSUBQ_M))
5034   ]
5035   "TARGET_HAVE_MVE"
5036   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5037   [(set_attr "type" "mve_move")])
5038
5039 ;;
5040 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5041 ;;
5042 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5043   [
5044    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5045         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5046                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5047                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5048                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5049          VCVTQ_M_N_TO_F))
5050   ]
5051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5052   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5053   [(set_attr "type" "mve_move")
5054    (set_attr "length""8")])
5055 ;;
5056 ;; [vabdq_m_s, vabdq_m_u])
5057 ;;
5058 (define_insn "mve_vabdq_m_<supf><mode>"
5059   [
5060    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5061         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5062                        (match_operand:MVE_2 2 "s_register_operand" "w")
5063                        (match_operand:MVE_2 3 "s_register_operand" "w")
5064                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5065          VABDQ_M))
5066   ]
5067   "TARGET_HAVE_MVE"
5068   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5069   [(set_attr "type" "mve_move")
5070    (set_attr "length""8")])
5071
5072 ;;
5073 ;; [vaddq_m_n_s, vaddq_m_n_u])
5074 ;;
5075 (define_insn "mve_vaddq_m_n_<supf><mode>"
5076   [
5077    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5078         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5079                        (match_operand:MVE_2 2 "s_register_operand" "w")
5080                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5081                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5082          VADDQ_M_N))
5083   ]
5084   "TARGET_HAVE_MVE"
5085   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5086   [(set_attr "type" "mve_move")
5087    (set_attr "length""8")])
5088
5089 ;;
5090 ;; [vaddq_m_u, vaddq_m_s])
5091 ;;
5092 (define_insn "mve_vaddq_m_<supf><mode>"
5093   [
5094    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5095         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5096                        (match_operand:MVE_2 2 "s_register_operand" "w")
5097                        (match_operand:MVE_2 3 "s_register_operand" "w")
5098                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5099          VADDQ_M))
5100   ]
5101   "TARGET_HAVE_MVE"
5102   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5103   [(set_attr "type" "mve_move")
5104    (set_attr "length""8")])
5105
5106 ;;
5107 ;; [vandq_m_u, vandq_m_s])
5108 ;;
5109 (define_insn "mve_vandq_m_<supf><mode>"
5110   [
5111    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5112         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5113                        (match_operand:MVE_2 2 "s_register_operand" "w")
5114                        (match_operand:MVE_2 3 "s_register_operand" "w")
5115                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5116          VANDQ_M))
5117   ]
5118   "TARGET_HAVE_MVE"
5119   "vpst\;vandt %q0, %q2, %q3"
5120   [(set_attr "type" "mve_move")
5121    (set_attr "length""8")])
5122
5123 ;;
5124 ;; [vbicq_m_u, vbicq_m_s])
5125 ;;
5126 (define_insn "mve_vbicq_m_<supf><mode>"
5127   [
5128    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5129         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5130                        (match_operand:MVE_2 2 "s_register_operand" "w")
5131                        (match_operand:MVE_2 3 "s_register_operand" "w")
5132                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5133          VBICQ_M))
5134   ]
5135   "TARGET_HAVE_MVE"
5136   "vpst\;vbict %q0, %q2, %q3"
5137   [(set_attr "type" "mve_move")
5138    (set_attr "length""8")])
5139
5140 ;;
5141 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5142 ;;
5143 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5144   [
5145    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5146         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5147                        (match_operand:MVE_2 2 "s_register_operand" "w")
5148                        (match_operand:SI 3 "s_register_operand" "r")
5149                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5150          VBRSRQ_M_N))
5151   ]
5152   "TARGET_HAVE_MVE"
5153   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5154   [(set_attr "type" "mve_move")
5155    (set_attr "length""8")])
5156
5157 ;;
5158 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5159 ;;
5160 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5161   [
5162    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5163         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5164                        (match_operand:MVE_2 2 "s_register_operand" "w")
5165                        (match_operand:MVE_2 3 "s_register_operand" "w")
5166                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5167          VCADDQ_ROT270_M))
5168   ]
5169   "TARGET_HAVE_MVE"
5170   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5171   [(set_attr "type" "mve_move")
5172    (set_attr "length""8")])
5173
5174 ;;
5175 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5176 ;;
5177 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5178   [
5179    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5180         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5181                        (match_operand:MVE_2 2 "s_register_operand" "w")
5182                        (match_operand:MVE_2 3 "s_register_operand" "w")
5183                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5184          VCADDQ_ROT90_M))
5185   ]
5186   "TARGET_HAVE_MVE"
5187   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5188   [(set_attr "type" "mve_move")
5189    (set_attr "length""8")])
5190
5191 ;;
5192 ;; [veorq_m_s, veorq_m_u])
5193 ;;
5194 (define_insn "mve_veorq_m_<supf><mode>"
5195   [
5196    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5197         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5198                        (match_operand:MVE_2 2 "s_register_operand" "w")
5199                        (match_operand:MVE_2 3 "s_register_operand" "w")
5200                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5201          VEORQ_M))
5202   ]
5203   "TARGET_HAVE_MVE"
5204   "vpst\;veort %q0, %q2, %q3"
5205   [(set_attr "type" "mve_move")
5206    (set_attr "length""8")])
5207
5208 ;;
5209 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5210 ;;
5211 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5212   [
5213    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5214         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5215                        (match_operand:MVE_2 2 "s_register_operand" "w")
5216                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5217                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5218          VHADDQ_M_N))
5219   ]
5220   "TARGET_HAVE_MVE"
5221   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5222   [(set_attr "type" "mve_move")
5223    (set_attr "length""8")])
5224
5225 ;;
5226 ;; [vhaddq_m_s, vhaddq_m_u])
5227 ;;
5228 (define_insn "mve_vhaddq_m_<supf><mode>"
5229   [
5230    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5231         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5232                        (match_operand:MVE_2 2 "s_register_operand" "w")
5233                        (match_operand:MVE_2 3 "s_register_operand" "w")
5234                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5235          VHADDQ_M))
5236   ]
5237   "TARGET_HAVE_MVE"
5238   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5239   [(set_attr "type" "mve_move")
5240    (set_attr "length""8")])
5241
5242 ;;
5243 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5244 ;;
5245 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5246   [
5247    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5248         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5249                        (match_operand:MVE_2 2 "s_register_operand" "w")
5250                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5251                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5252          VHSUBQ_M_N))
5253   ]
5254   "TARGET_HAVE_MVE"
5255   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5256   [(set_attr "type" "mve_move")
5257    (set_attr "length""8")])
5258
5259 ;;
5260 ;; [vhsubq_m_s, vhsubq_m_u])
5261 ;;
5262 (define_insn "mve_vhsubq_m_<supf><mode>"
5263   [
5264    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5265         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5266                        (match_operand:MVE_2 2 "s_register_operand" "w")
5267                        (match_operand:MVE_2 3 "s_register_operand" "w")
5268                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5269          VHSUBQ_M))
5270   ]
5271   "TARGET_HAVE_MVE"
5272   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5273   [(set_attr "type" "mve_move")
5274    (set_attr "length""8")])
5275
5276 ;;
5277 ;; [vmaxq_m_s, vmaxq_m_u])
5278 ;;
5279 (define_insn "mve_vmaxq_m_<supf><mode>"
5280   [
5281    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5282         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5283                        (match_operand:MVE_2 2 "s_register_operand" "w")
5284                        (match_operand:MVE_2 3 "s_register_operand" "w")
5285                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5286          VMAXQ_M))
5287   ]
5288   "TARGET_HAVE_MVE"
5289   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5290   [(set_attr "type" "mve_move")
5291    (set_attr "length""8")])
5292
5293 ;;
5294 ;; [vminq_m_s, vminq_m_u])
5295 ;;
5296 (define_insn "mve_vminq_m_<supf><mode>"
5297   [
5298    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5299         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5300                        (match_operand:MVE_2 2 "s_register_operand" "w")
5301                        (match_operand:MVE_2 3 "s_register_operand" "w")
5302                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5303          VMINQ_M))
5304   ]
5305   "TARGET_HAVE_MVE"
5306   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5307   [(set_attr "type" "mve_move")
5308    (set_attr "length""8")])
5309
5310 ;;
5311 ;; [vmladavaq_p_u, vmladavaq_p_s])
5312 ;;
5313 (define_insn "mve_vmladavaq_p_<supf><mode>"
5314   [
5315    (set (match_operand:SI 0 "s_register_operand" "=Te")
5316         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5317                     (match_operand:MVE_2 2 "s_register_operand" "w")
5318                     (match_operand:MVE_2 3 "s_register_operand" "w")
5319                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5320          VMLADAVAQ_P))
5321   ]
5322   "TARGET_HAVE_MVE"
5323   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5324   [(set_attr "type" "mve_move")
5325    (set_attr "length""8")])
5326
5327 ;;
5328 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5329 ;;
5330 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5331   [
5332    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5333         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5334                        (match_operand:MVE_2 2 "s_register_operand" "w")
5335                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5336                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5337          VMLAQ_M_N))
5338   ]
5339   "TARGET_HAVE_MVE"
5340   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5341   [(set_attr "type" "mve_move")
5342    (set_attr "length""8")])
5343
5344 ;;
5345 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5346 ;;
5347 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5348   [
5349    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5350         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5351                        (match_operand:MVE_2 2 "s_register_operand" "w")
5352                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5353                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5354          VMLASQ_M_N))
5355   ]
5356   "TARGET_HAVE_MVE"
5357   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5358   [(set_attr "type" "mve_move")
5359    (set_attr "length""8")])
5360
5361 ;;
5362 ;; [vmulhq_m_s, vmulhq_m_u])
5363 ;;
5364 (define_insn "mve_vmulhq_m_<supf><mode>"
5365   [
5366    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5367         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5368                        (match_operand:MVE_2 2 "s_register_operand" "w")
5369                        (match_operand:MVE_2 3 "s_register_operand" "w")
5370                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5371          VMULHQ_M))
5372   ]
5373   "TARGET_HAVE_MVE"
5374   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5375   [(set_attr "type" "mve_move")
5376    (set_attr "length""8")])
5377
5378 ;;
5379 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5380 ;;
5381 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5382   [
5383    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5384         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5385                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5386                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5387                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5388          VMULLBQ_INT_M))
5389   ]
5390   "TARGET_HAVE_MVE"
5391   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5392   [(set_attr "type" "mve_move")
5393    (set_attr "length""8")])
5394
5395 ;;
5396 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5397 ;;
5398 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5399   [
5400    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5401         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5402                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5403                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5404                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5405          VMULLTQ_INT_M))
5406   ]
5407   "TARGET_HAVE_MVE"
5408   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5409   [(set_attr "type" "mve_move")
5410    (set_attr "length""8")])
5411
5412 ;;
5413 ;; [vmulq_m_n_u, vmulq_m_n_s])
5414 ;;
5415 (define_insn "mve_vmulq_m_n_<supf><mode>"
5416   [
5417    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5418         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5419                        (match_operand:MVE_2 2 "s_register_operand" "w")
5420                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5421                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5422          VMULQ_M_N))
5423   ]
5424   "TARGET_HAVE_MVE"
5425   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5426   [(set_attr "type" "mve_move")
5427    (set_attr "length""8")])
5428
5429 ;;
5430 ;; [vmulq_m_s, vmulq_m_u])
5431 ;;
5432 (define_insn "mve_vmulq_m_<supf><mode>"
5433   [
5434    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5435         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5436                        (match_operand:MVE_2 2 "s_register_operand" "w")
5437                        (match_operand:MVE_2 3 "s_register_operand" "w")
5438                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5439          VMULQ_M))
5440   ]
5441   "TARGET_HAVE_MVE"
5442   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5443   [(set_attr "type" "mve_move")
5444    (set_attr "length""8")])
5445
5446 ;;
5447 ;; [vornq_m_u, vornq_m_s])
5448 ;;
5449 (define_insn "mve_vornq_m_<supf><mode>"
5450   [
5451    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5452         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5453                        (match_operand:MVE_2 2 "s_register_operand" "w")
5454                        (match_operand:MVE_2 3 "s_register_operand" "w")
5455                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5456          VORNQ_M))
5457   ]
5458   "TARGET_HAVE_MVE"
5459   "vpst\;vornt %q0, %q2, %q3"
5460   [(set_attr "type" "mve_move")
5461    (set_attr "length""8")])
5462
5463 ;;
5464 ;; [vorrq_m_s, vorrq_m_u])
5465 ;;
5466 (define_insn "mve_vorrq_m_<supf><mode>"
5467   [
5468    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5469         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5470                        (match_operand:MVE_2 2 "s_register_operand" "w")
5471                        (match_operand:MVE_2 3 "s_register_operand" "w")
5472                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5473          VORRQ_M))
5474   ]
5475   "TARGET_HAVE_MVE"
5476   "vpst\;vorrt %q0, %q2, %q3"
5477   [(set_attr "type" "mve_move")
5478    (set_attr "length""8")])
5479
5480 ;;
5481 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5482 ;;
5483 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5484   [
5485    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5486         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5487                        (match_operand:MVE_2 2 "s_register_operand" "w")
5488                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5489                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5490          VQADDQ_M_N))
5491   ]
5492   "TARGET_HAVE_MVE"
5493   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5494   [(set_attr "type" "mve_move")
5495    (set_attr "length""8")])
5496
5497 ;;
5498 ;; [vqaddq_m_u, vqaddq_m_s])
5499 ;;
5500 (define_insn "mve_vqaddq_m_<supf><mode>"
5501   [
5502    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5503         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5504                        (match_operand:MVE_2 2 "s_register_operand" "w")
5505                        (match_operand:MVE_2 3 "s_register_operand" "w")
5506                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5507          VQADDQ_M))
5508   ]
5509   "TARGET_HAVE_MVE"
5510   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5511   [(set_attr "type" "mve_move")
5512    (set_attr "length""8")])
5513
5514 ;;
5515 ;; [vqdmlahq_m_n_s])
5516 ;;
5517 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5518   [
5519    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5520         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5521                        (match_operand:MVE_2 2 "s_register_operand" "w")
5522                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5523                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5524          VQDMLAHQ_M_N_S))
5525   ]
5526   "TARGET_HAVE_MVE"
5527   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5528   [(set_attr "type" "mve_move")
5529    (set_attr "length""8")])
5530
5531 ;;
5532 ;; [vqdmlashq_m_n_s])
5533 ;;
5534 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5535   [
5536    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5537         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5538                        (match_operand:MVE_2 2 "s_register_operand" "w")
5539                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5540                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5541          VQDMLASHQ_M_N_S))
5542   ]
5543   "TARGET_HAVE_MVE"
5544   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5545   [(set_attr "type" "mve_move")
5546    (set_attr "length""8")])
5547
5548 ;;
5549 ;; [vqrdmlahq_m_n_s])
5550 ;;
5551 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5552   [
5553    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5554         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5555                        (match_operand:MVE_2 2 "s_register_operand" "w")
5556                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5557                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5558          VQRDMLAHQ_M_N_S))
5559   ]
5560   "TARGET_HAVE_MVE"
5561   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5562   [(set_attr "type" "mve_move")
5563    (set_attr "length""8")])
5564
5565 ;;
5566 ;; [vqrdmlashq_m_n_s])
5567 ;;
5568 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5569   [
5570    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5571         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5572                        (match_operand:MVE_2 2 "s_register_operand" "w")
5573                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5574                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5575          VQRDMLASHQ_M_N_S))
5576   ]
5577   "TARGET_HAVE_MVE"
5578   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5579   [(set_attr "type" "mve_move")
5580    (set_attr "length""8")])
5581
5582 ;;
5583 ;; [vqrshlq_m_u, vqrshlq_m_s])
5584 ;;
5585 (define_insn "mve_vqrshlq_m_<supf><mode>"
5586   [
5587    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5588         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5589                        (match_operand:MVE_2 2 "s_register_operand" "w")
5590                        (match_operand:MVE_2 3 "s_register_operand" "w")
5591                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5592          VQRSHLQ_M))
5593   ]
5594   "TARGET_HAVE_MVE"
5595   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5596   [(set_attr "type" "mve_move")
5597    (set_attr "length""8")])
5598
5599 ;;
5600 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5601 ;;
5602 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5603   [
5604    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5605         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5606                        (match_operand:MVE_2 2 "s_register_operand" "w")
5607                        (match_operand:SI 3 "immediate_operand" "i")
5608                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5609          VQSHLQ_M_N))
5610   ]
5611   "TARGET_HAVE_MVE"
5612   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5613   [(set_attr "type" "mve_move")
5614    (set_attr "length""8")])
5615
5616 ;;
5617 ;; [vqshlq_m_u, vqshlq_m_s])
5618 ;;
5619 (define_insn "mve_vqshlq_m_<supf><mode>"
5620   [
5621    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5622         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5623                        (match_operand:MVE_2 2 "s_register_operand" "w")
5624                        (match_operand:MVE_2 3 "s_register_operand" "w")
5625                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5626          VQSHLQ_M))
5627   ]
5628   "TARGET_HAVE_MVE"
5629   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5630   [(set_attr "type" "mve_move")
5631    (set_attr "length""8")])
5632
5633 ;;
5634 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
5635 ;;
5636 (define_insn "mve_vqsubq_m_n_<supf><mode>"
5637   [
5638    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5639         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5640                        (match_operand:MVE_2 2 "s_register_operand" "w")
5641                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5642                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5643          VQSUBQ_M_N))
5644   ]
5645   "TARGET_HAVE_MVE"
5646   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5647   [(set_attr "type" "mve_move")
5648    (set_attr "length""8")])
5649
5650 ;;
5651 ;; [vqsubq_m_u, vqsubq_m_s])
5652 ;;
5653 (define_insn "mve_vqsubq_m_<supf><mode>"
5654   [
5655    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5656         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5657                        (match_operand:MVE_2 2 "s_register_operand" "w")
5658                        (match_operand:MVE_2 3 "s_register_operand" "w")
5659                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5660          VQSUBQ_M))
5661   ]
5662   "TARGET_HAVE_MVE"
5663   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5664   [(set_attr "type" "mve_move")
5665    (set_attr "length""8")])
5666
5667 ;;
5668 ;; [vrhaddq_m_u, vrhaddq_m_s])
5669 ;;
5670 (define_insn "mve_vrhaddq_m_<supf><mode>"
5671   [
5672    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5673         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5674                        (match_operand:MVE_2 2 "s_register_operand" "w")
5675                        (match_operand:MVE_2 3 "s_register_operand" "w")
5676                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5677          VRHADDQ_M))
5678   ]
5679   "TARGET_HAVE_MVE"
5680   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5681   [(set_attr "type" "mve_move")
5682    (set_attr "length""8")])
5683
5684 ;;
5685 ;; [vrmulhq_m_u, vrmulhq_m_s])
5686 ;;
5687 (define_insn "mve_vrmulhq_m_<supf><mode>"
5688   [
5689    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5690         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5691                        (match_operand:MVE_2 2 "s_register_operand" "w")
5692                        (match_operand:MVE_2 3 "s_register_operand" "w")
5693                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5694          VRMULHQ_M))
5695   ]
5696   "TARGET_HAVE_MVE"
5697   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5698   [(set_attr "type" "mve_move")
5699    (set_attr "length""8")])
5700
5701 ;;
5702 ;; [vrshlq_m_s, vrshlq_m_u])
5703 ;;
5704 (define_insn "mve_vrshlq_m_<supf><mode>"
5705   [
5706    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5707         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5708                        (match_operand:MVE_2 2 "s_register_operand" "w")
5709                        (match_operand:MVE_2 3 "s_register_operand" "w")
5710                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5711          VRSHLQ_M))
5712   ]
5713   "TARGET_HAVE_MVE"
5714   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5715   [(set_attr "type" "mve_move")
5716    (set_attr "length""8")])
5717
5718 ;;
5719 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
5720 ;;
5721 (define_insn "mve_vrshrq_m_n_<supf><mode>"
5722   [
5723    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5724         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5725                        (match_operand:MVE_2 2 "s_register_operand" "w")
5726                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5727                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5728          VRSHRQ_M_N))
5729   ]
5730   "TARGET_HAVE_MVE"
5731   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5732   [(set_attr "type" "mve_move")
5733    (set_attr "length""8")])
5734
5735 ;;
5736 ;; [vshlq_m_n_s, vshlq_m_n_u])
5737 ;;
5738 (define_insn "mve_vshlq_m_n_<supf><mode>"
5739   [
5740    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5741         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5742                        (match_operand:MVE_2 2 "s_register_operand" "w")
5743                        (match_operand:SI 3 "immediate_operand" "i")
5744                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5745          VSHLQ_M_N))
5746   ]
5747   "TARGET_HAVE_MVE"
5748   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5749   [(set_attr "type" "mve_move")
5750    (set_attr "length""8")])
5751
5752 ;;
5753 ;; [vshrq_m_n_s, vshrq_m_n_u])
5754 ;;
5755 (define_insn "mve_vshrq_m_n_<supf><mode>"
5756   [
5757    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5758         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5759                        (match_operand:MVE_2 2 "s_register_operand" "w")
5760                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5761                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5762          VSHRQ_M_N))
5763   ]
5764   "TARGET_HAVE_MVE"
5765   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5766   [(set_attr "type" "mve_move")
5767    (set_attr "length""8")])
5768
5769 ;;
5770 ;; [vsliq_m_n_u, vsliq_m_n_s])
5771 ;;
5772 (define_insn "mve_vsliq_m_n_<supf><mode>"
5773    [
5774    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5775        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5776                        (match_operand:MVE_2 2 "s_register_operand" "w")
5777                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
5778                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5779          VSLIQ_M_N))
5780   ]
5781   "TARGET_HAVE_MVE"
5782   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
5783   [(set_attr "type" "mve_move")
5784    (set_attr "length""8")])
5785
5786 ;;
5787 ;; [vsubq_m_n_s, vsubq_m_n_u])
5788 ;;
5789 (define_insn "mve_vsubq_m_n_<supf><mode>"
5790   [
5791    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5792         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5793                        (match_operand:MVE_2 2 "s_register_operand" "w")
5794                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5795                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5796          VSUBQ_M_N))
5797   ]
5798   "TARGET_HAVE_MVE"
5799   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
5800   [(set_attr "type" "mve_move")
5801    (set_attr "length""8")])
5802
5803 ;;
5804 ;; [vhcaddq_rot270_m_s])
5805 ;;
5806 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
5807   [
5808    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5809         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5810                        (match_operand:MVE_2 2 "s_register_operand" "w")
5811                        (match_operand:MVE_2 3 "s_register_operand" "w")
5812                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5813          VHCADDQ_ROT270_M_S))
5814   ]
5815   "TARGET_HAVE_MVE"
5816   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
5817   [(set_attr "type" "mve_move")
5818    (set_attr "length""8")])
5819
5820 ;;
5821 ;; [vhcaddq_rot90_m_s])
5822 ;;
5823 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
5824   [
5825    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5826         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5827                        (match_operand:MVE_2 2 "s_register_operand" "w")
5828                        (match_operand:MVE_2 3 "s_register_operand" "w")
5829                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5830          VHCADDQ_ROT90_M_S))
5831   ]
5832   "TARGET_HAVE_MVE"
5833   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
5834   [(set_attr "type" "mve_move")
5835    (set_attr "length""8")])
5836
5837 ;;
5838 ;; [vmladavaxq_p_s])
5839 ;;
5840 (define_insn "mve_vmladavaxq_p_s<mode>"
5841   [
5842    (set (match_operand:SI 0 "s_register_operand" "=Te")
5843         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5844                        (match_operand:MVE_2 2 "s_register_operand" "w")
5845                        (match_operand:MVE_2 3 "s_register_operand" "w")
5846                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5847          VMLADAVAXQ_P_S))
5848   ]
5849   "TARGET_HAVE_MVE"
5850   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5851   [(set_attr "type" "mve_move")
5852    (set_attr "length""8")])
5853
5854 ;;
5855 ;; [vmlsdavaq_p_s])
5856 ;;
5857 (define_insn "mve_vmlsdavaq_p_s<mode>"
5858   [
5859    (set (match_operand:SI 0 "s_register_operand" "=Te")
5860         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5861                        (match_operand:MVE_2 2 "s_register_operand" "w")
5862                        (match_operand:MVE_2 3 "s_register_operand" "w")
5863                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5864          VMLSDAVAQ_P_S))
5865   ]
5866   "TARGET_HAVE_MVE"
5867   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
5868   [(set_attr "type" "mve_move")
5869    (set_attr "length""8")])
5870
5871 ;;
5872 ;; [vmlsdavaxq_p_s])
5873 ;;
5874 (define_insn "mve_vmlsdavaxq_p_s<mode>"
5875   [
5876    (set (match_operand:SI 0 "s_register_operand" "=Te")
5877         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5878                        (match_operand:MVE_2 2 "s_register_operand" "w")
5879                        (match_operand:MVE_2 3 "s_register_operand" "w")
5880                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5881          VMLSDAVAXQ_P_S))
5882   ]
5883   "TARGET_HAVE_MVE"
5884   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5885   [(set_attr "type" "mve_move")
5886    (set_attr "length""8")])
5887
5888 ;;
5889 ;; [vqdmladhq_m_s])
5890 ;;
5891 (define_insn "mve_vqdmladhq_m_s<mode>"
5892   [
5893    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5894         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5895                        (match_operand:MVE_2 2 "s_register_operand" "w")
5896                        (match_operand:MVE_2 3 "s_register_operand" "w")
5897                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5898          VQDMLADHQ_M_S))
5899   ]
5900   "TARGET_HAVE_MVE"
5901   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5902   [(set_attr "type" "mve_move")
5903    (set_attr "length""8")])
5904
5905 ;;
5906 ;; [vqdmladhxq_m_s])
5907 ;;
5908 (define_insn "mve_vqdmladhxq_m_s<mode>"
5909   [
5910    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5911         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5912                        (match_operand:MVE_2 2 "s_register_operand" "w")
5913                        (match_operand:MVE_2 3 "s_register_operand" "w")
5914                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5915          VQDMLADHXQ_M_S))
5916   ]
5917   "TARGET_HAVE_MVE"
5918   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5919   [(set_attr "type" "mve_move")
5920    (set_attr "length""8")])
5921
5922 ;;
5923 ;; [vqdmlsdhq_m_s])
5924 ;;
5925 (define_insn "mve_vqdmlsdhq_m_s<mode>"
5926   [
5927    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5928         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5929                        (match_operand:MVE_2 2 "s_register_operand" "w")
5930                        (match_operand:MVE_2 3 "s_register_operand" "w")
5931                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5932          VQDMLSDHQ_M_S))
5933   ]
5934   "TARGET_HAVE_MVE"
5935   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5936   [(set_attr "type" "mve_move")
5937    (set_attr "length""8")])
5938
5939 ;;
5940 ;; [vqdmlsdhxq_m_s])
5941 ;;
5942 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
5943   [
5944    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5945         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5946                        (match_operand:MVE_2 2 "s_register_operand" "w")
5947                        (match_operand:MVE_2 3 "s_register_operand" "w")
5948                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5949          VQDMLSDHXQ_M_S))
5950   ]
5951   "TARGET_HAVE_MVE"
5952   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5953   [(set_attr "type" "mve_move")
5954    (set_attr "length""8")])
5955
5956 ;;
5957 ;; [vqdmulhq_m_n_s])
5958 ;;
5959 (define_insn "mve_vqdmulhq_m_n_s<mode>"
5960   [
5961    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5962         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5963                        (match_operand:MVE_2 2 "s_register_operand" "w")
5964                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5965                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5966          VQDMULHQ_M_N_S))
5967   ]
5968   "TARGET_HAVE_MVE"
5969   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
5970   [(set_attr "type" "mve_move")
5971    (set_attr "length""8")])
5972
5973 ;;
5974 ;; [vqdmulhq_m_s])
5975 ;;
5976 (define_insn "mve_vqdmulhq_m_s<mode>"
5977   [
5978    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5979         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5980                        (match_operand:MVE_2 2 "s_register_operand" "w")
5981                        (match_operand:MVE_2 3 "s_register_operand" "w")
5982                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5983          VQDMULHQ_M_S))
5984   ]
5985   "TARGET_HAVE_MVE"
5986   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5987   [(set_attr "type" "mve_move")
5988    (set_attr "length""8")])
5989
5990 ;;
5991 ;; [vqrdmladhq_m_s])
5992 ;;
5993 (define_insn "mve_vqrdmladhq_m_s<mode>"
5994   [
5995    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5996         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5997                        (match_operand:MVE_2 2 "s_register_operand" "w")
5998                        (match_operand:MVE_2 3 "s_register_operand" "w")
5999                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6000          VQRDMLADHQ_M_S))
6001   ]
6002   "TARGET_HAVE_MVE"
6003   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6004   [(set_attr "type" "mve_move")
6005    (set_attr "length""8")])
6006
6007 ;;
6008 ;; [vqrdmladhxq_m_s])
6009 ;;
6010 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6011   [
6012    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6013         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6014                        (match_operand:MVE_2 2 "s_register_operand" "w")
6015                        (match_operand:MVE_2 3 "s_register_operand" "w")
6016                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6017          VQRDMLADHXQ_M_S))
6018   ]
6019   "TARGET_HAVE_MVE"
6020   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6021   [(set_attr "type" "mve_move")
6022    (set_attr "length""8")])
6023
6024 ;;
6025 ;; [vqrdmlsdhq_m_s])
6026 ;;
6027 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6028   [
6029    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6030         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6031                        (match_operand:MVE_2 2 "s_register_operand" "w")
6032                        (match_operand:MVE_2 3 "s_register_operand" "w")
6033                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6034          VQRDMLSDHQ_M_S))
6035   ]
6036   "TARGET_HAVE_MVE"
6037   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6038   [(set_attr "type" "mve_move")
6039    (set_attr "length""8")])
6040
6041 ;;
6042 ;; [vqrdmlsdhxq_m_s])
6043 ;;
6044 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6045   [
6046    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6047         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6048                        (match_operand:MVE_2 2 "s_register_operand" "w")
6049                        (match_operand:MVE_2 3 "s_register_operand" "w")
6050                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6051          VQRDMLSDHXQ_M_S))
6052   ]
6053   "TARGET_HAVE_MVE"
6054   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6055   [(set_attr "type" "mve_move")
6056    (set_attr "length""8")])
6057
6058 ;;
6059 ;; [vqrdmulhq_m_n_s])
6060 ;;
6061 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6062   [
6063    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6064         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6065                        (match_operand:MVE_2 2 "s_register_operand" "w")
6066                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6067                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6068          VQRDMULHQ_M_N_S))
6069   ]
6070   "TARGET_HAVE_MVE"
6071   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6072   [(set_attr "type" "mve_move")
6073    (set_attr "length""8")])
6074
6075 ;;
6076 ;; [vqrdmulhq_m_s])
6077 ;;
6078 (define_insn "mve_vqrdmulhq_m_s<mode>"
6079   [
6080    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6081         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6082                        (match_operand:MVE_2 2 "s_register_operand" "w")
6083                        (match_operand:MVE_2 3 "s_register_operand" "w")
6084                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6085          VQRDMULHQ_M_S))
6086   ]
6087   "TARGET_HAVE_MVE"
6088   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6089   [(set_attr "type" "mve_move")
6090    (set_attr "length""8")])
6091
6092 ;;
6093 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6094 ;;
6095 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6096   [
6097    (set (match_operand:DI 0 "s_register_operand" "=r")
6098         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6099                        (match_operand:MVE_5 2 "s_register_operand" "w")
6100                        (match_operand:MVE_5 3 "s_register_operand" "w")
6101                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6102          VMLALDAVAQ_P))
6103   ]
6104   "TARGET_HAVE_MVE"
6105   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6106   [(set_attr "type" "mve_move")
6107    (set_attr "length""8")])
6108
6109 ;;
6110 ;; [vmlaldavaxq_p_s])
6111 ;;
6112 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6113   [
6114    (set (match_operand:DI 0 "s_register_operand" "=r")
6115         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6116                        (match_operand:MVE_5 2 "s_register_operand" "w")
6117                        (match_operand:MVE_5 3 "s_register_operand" "w")
6118                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6119          VMLALDAVAXQ_P))
6120   ]
6121   "TARGET_HAVE_MVE"
6122   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6123   [(set_attr "type" "mve_move")
6124    (set_attr "length""8")])
6125
6126 ;;
6127 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6128 ;;
6129 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6130   [
6131    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6132         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6133                        (match_operand:MVE_5 2 "s_register_operand" "w")
6134                        (match_operand:SI 3 "mve_imm_8" "Rb")
6135                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6136          VQRSHRNBQ_M_N))
6137   ]
6138   "TARGET_HAVE_MVE"
6139   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6140   [(set_attr "type" "mve_move")
6141    (set_attr "length""8")])
6142
6143 ;;
6144 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6145 ;;
6146 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6147   [
6148    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6149         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6150                        (match_operand:MVE_5 2 "s_register_operand" "w")
6151                        (match_operand:SI 3 "mve_imm_8" "Rb")
6152                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6153          VQRSHRNTQ_M_N))
6154   ]
6155   "TARGET_HAVE_MVE"
6156   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6157   [(set_attr "type" "mve_move")
6158    (set_attr "length""8")])
6159
6160 ;;
6161 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6162 ;;
6163 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6164   [
6165    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6166         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6167                        (match_operand:MVE_5 2 "s_register_operand" "w")
6168                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6169                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6170          VQSHRNBQ_M_N))
6171   ]
6172   "TARGET_HAVE_MVE"
6173   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6174   [(set_attr "type" "mve_move")
6175    (set_attr "length""8")])
6176
6177 ;;
6178 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6179 ;;
6180 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6181   [
6182    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6183         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6184                        (match_operand:MVE_5 2 "s_register_operand" "w")
6185                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6186                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6187          VQSHRNTQ_M_N))
6188   ]
6189   "TARGET_HAVE_MVE"
6190   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6191   [(set_attr "type" "mve_move")
6192    (set_attr "length""8")])
6193
6194 ;;
6195 ;; [vrmlaldavhaq_p_s])
6196 ;;
6197 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6198   [
6199    (set (match_operand:DI 0 "s_register_operand" "=r")
6200         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6201                        (match_operand:V4SI 2 "s_register_operand" "w")
6202                        (match_operand:V4SI 3 "s_register_operand" "w")
6203                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6204          VRMLALDAVHAQ_P_S))
6205   ]
6206   "TARGET_HAVE_MVE"
6207   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6208   [(set_attr "type" "mve_move")
6209    (set_attr "length""8")])
6210
6211 ;;
6212 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6213 ;;
6214 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6215   [
6216    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6217         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6218                        (match_operand:MVE_5 2 "s_register_operand" "w")
6219                        (match_operand:SI 3 "mve_imm_8" "Rb")
6220                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6221          VRSHRNBQ_M_N))
6222   ]
6223   "TARGET_HAVE_MVE"
6224   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6225   [(set_attr "type" "mve_move")
6226    (set_attr "length""8")])
6227
6228 ;;
6229 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6230 ;;
6231 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6232   [
6233    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6234         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6235                        (match_operand:MVE_5 2 "s_register_operand" "w")
6236                        (match_operand:SI 3 "mve_imm_8" "Rb")
6237                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6238          VRSHRNTQ_M_N))
6239   ]
6240   "TARGET_HAVE_MVE"
6241   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6242   [(set_attr "type" "mve_move")
6243    (set_attr "length""8")])
6244
6245 ;;
6246 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6247 ;;
6248 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6249   [
6250    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6251         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6252                        (match_operand:MVE_3 2 "s_register_operand" "w")
6253                        (match_operand:SI 3 "immediate_operand" "i")
6254                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6255          VSHLLBQ_M_N))
6256   ]
6257   "TARGET_HAVE_MVE"
6258   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6259   [(set_attr "type" "mve_move")
6260    (set_attr "length""8")])
6261
6262 ;;
6263 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6264 ;;
6265 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6266   [
6267    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6268         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6269                        (match_operand:MVE_3 2 "s_register_operand" "w")
6270                        (match_operand:SI 3 "immediate_operand" "i")
6271                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6272          VSHLLTQ_M_N))
6273   ]
6274   "TARGET_HAVE_MVE"
6275   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6276   [(set_attr "type" "mve_move")
6277    (set_attr "length""8")])
6278
6279 ;;
6280 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6281 ;;
6282 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6283   [
6284    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6285         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6286                        (match_operand:MVE_5 2 "s_register_operand" "w")
6287                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6288                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6289          VSHRNBQ_M_N))
6290   ]
6291   "TARGET_HAVE_MVE"
6292   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6293   [(set_attr "type" "mve_move")
6294    (set_attr "length""8")])
6295
6296 ;;
6297 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6298 ;;
6299 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6300   [
6301    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6302         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6303                        (match_operand:MVE_5 2 "s_register_operand" "w")
6304                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6305                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6306          VSHRNTQ_M_N))
6307   ]
6308   "TARGET_HAVE_MVE"
6309   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6310   [(set_attr "type" "mve_move")
6311    (set_attr "length""8")])
6312
6313 ;;
6314 ;; [vmlsldavaq_p_s])
6315 ;;
6316 (define_insn "mve_vmlsldavaq_p_s<mode>"
6317   [
6318    (set (match_operand:DI 0 "s_register_operand" "=r")
6319         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6320                        (match_operand:MVE_5 2 "s_register_operand" "w")
6321                        (match_operand:MVE_5 3 "s_register_operand" "w")
6322                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6323          VMLSLDAVAQ_P_S))
6324   ]
6325   "TARGET_HAVE_MVE"
6326   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6327   [(set_attr "type" "mve_move")
6328    (set_attr "length""8")])
6329
6330 ;;
6331 ;; [vmlsldavaxq_p_s])
6332 ;;
6333 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6334   [
6335    (set (match_operand:DI 0 "s_register_operand" "=r")
6336         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6337                        (match_operand:MVE_5 2 "s_register_operand" "w")
6338                        (match_operand:MVE_5 3 "s_register_operand" "w")
6339                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6340          VMLSLDAVAXQ_P_S))
6341   ]
6342   "TARGET_HAVE_MVE"
6343   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6344   [(set_attr "type" "mve_move")
6345    (set_attr "length""8")])
6346
6347 ;;
6348 ;; [vmullbq_poly_m_p])
6349 ;;
6350 (define_insn "mve_vmullbq_poly_m_p<mode>"
6351   [
6352    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6353         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6354                        (match_operand:MVE_3 2 "s_register_operand" "w")
6355                        (match_operand:MVE_3 3 "s_register_operand" "w")
6356                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6357          VMULLBQ_POLY_M_P))
6358   ]
6359   "TARGET_HAVE_MVE"
6360   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6361   [(set_attr "type" "mve_move")
6362    (set_attr "length""8")])
6363
6364 ;;
6365 ;; [vmulltq_poly_m_p])
6366 ;;
6367 (define_insn "mve_vmulltq_poly_m_p<mode>"
6368   [
6369    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6370         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6371                        (match_operand:MVE_3 2 "s_register_operand" "w")
6372                        (match_operand:MVE_3 3 "s_register_operand" "w")
6373                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6374          VMULLTQ_POLY_M_P))
6375   ]
6376   "TARGET_HAVE_MVE"
6377   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6378   [(set_attr "type" "mve_move")
6379    (set_attr "length""8")])
6380
6381 ;;
6382 ;; [vqdmullbq_m_n_s])
6383 ;;
6384 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6385   [
6386    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6387         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6388                        (match_operand:MVE_5 2 "s_register_operand" "w")
6389                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6390                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6391          VQDMULLBQ_M_N_S))
6392   ]
6393   "TARGET_HAVE_MVE"
6394   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6395   [(set_attr "type" "mve_move")
6396    (set_attr "length""8")])
6397
6398 ;;
6399 ;; [vqdmullbq_m_s])
6400 ;;
6401 (define_insn "mve_vqdmullbq_m_s<mode>"
6402   [
6403    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6404         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6405                        (match_operand:MVE_5 2 "s_register_operand" "w")
6406                        (match_operand:MVE_5 3 "s_register_operand" "w")
6407                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6408          VQDMULLBQ_M_S))
6409   ]
6410   "TARGET_HAVE_MVE"
6411   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6412   [(set_attr "type" "mve_move")
6413    (set_attr "length""8")])
6414
6415 ;;
6416 ;; [vqdmulltq_m_n_s])
6417 ;;
6418 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6419   [
6420    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6421         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6422                        (match_operand:MVE_5 2 "s_register_operand" "w")
6423                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6424                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6425          VQDMULLTQ_M_N_S))
6426   ]
6427   "TARGET_HAVE_MVE"
6428   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6429   [(set_attr "type" "mve_move")
6430    (set_attr "length""8")])
6431
6432 ;;
6433 ;; [vqdmulltq_m_s])
6434 ;;
6435 (define_insn "mve_vqdmulltq_m_s<mode>"
6436   [
6437    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6438         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6439                        (match_operand:MVE_5 2 "s_register_operand" "w")
6440                        (match_operand:MVE_5 3 "s_register_operand" "w")
6441                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6442          VQDMULLTQ_M_S))
6443   ]
6444   "TARGET_HAVE_MVE"
6445   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6446   [(set_attr "type" "mve_move")
6447    (set_attr "length""8")])
6448
6449 ;;
6450 ;; [vqrshrunbq_m_n_s])
6451 ;;
6452 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6453   [
6454    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6455         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6456                        (match_operand:MVE_5 2 "s_register_operand" "w")
6457                        (match_operand:SI 3 "mve_imm_8" "Rb")
6458                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6459          VQRSHRUNBQ_M_N_S))
6460   ]
6461   "TARGET_HAVE_MVE"
6462   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6463   [(set_attr "type" "mve_move")
6464    (set_attr "length""8")])
6465
6466 ;;
6467 ;; [vqrshruntq_m_n_s])
6468 ;;
6469 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6470   [
6471    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6472         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6473                        (match_operand:MVE_5 2 "s_register_operand" "w")
6474                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6475                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6476          VQRSHRUNTQ_M_N_S))
6477   ]
6478   "TARGET_HAVE_MVE"
6479   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6480   [(set_attr "type" "mve_move")
6481    (set_attr "length""8")])
6482
6483 ;;
6484 ;; [vqshrunbq_m_n_s])
6485 ;;
6486 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6487   [
6488    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6489         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6490                        (match_operand:MVE_5 2 "s_register_operand" "w")
6491                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6492                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6493          VQSHRUNBQ_M_N_S))
6494   ]
6495   "TARGET_HAVE_MVE"
6496   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6497   [(set_attr "type" "mve_move")
6498    (set_attr "length""8")])
6499
6500 ;;
6501 ;; [vqshruntq_m_n_s])
6502 ;;
6503 (define_insn "mve_vqshruntq_m_n_s<mode>"
6504   [
6505    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6506         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6507                        (match_operand:MVE_5 2 "s_register_operand" "w")
6508                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6509                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6510          VQSHRUNTQ_M_N_S))
6511   ]
6512   "TARGET_HAVE_MVE"
6513   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6514   [(set_attr "type" "mve_move")
6515    (set_attr "length""8")])
6516
6517 ;;
6518 ;; [vrmlaldavhaq_p_u])
6519 ;;
6520 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6521   [
6522    (set (match_operand:DI 0 "s_register_operand" "=r")
6523         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6524                        (match_operand:V4SI 2 "s_register_operand" "w")
6525                        (match_operand:V4SI 3 "s_register_operand" "w")
6526                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6527          VRMLALDAVHAQ_P_U))
6528   ]
6529   "TARGET_HAVE_MVE"
6530   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6531   [(set_attr "type" "mve_move")
6532    (set_attr "length""8")])
6533
6534 ;;
6535 ;; [vrmlaldavhaxq_p_s])
6536 ;;
6537 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6538   [
6539    (set (match_operand:DI 0 "s_register_operand" "=r")
6540         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6541                        (match_operand:V4SI 2 "s_register_operand" "w")
6542                        (match_operand:V4SI 3 "s_register_operand" "w")
6543                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6544          VRMLALDAVHAXQ_P_S))
6545   ]
6546   "TARGET_HAVE_MVE"
6547   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6548   [(set_attr "type" "mve_move")
6549    (set_attr "length""8")])
6550
6551 ;;
6552 ;; [vrmlsldavhaq_p_s])
6553 ;;
6554 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6555   [
6556    (set (match_operand:DI 0 "s_register_operand" "=r")
6557         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6558                        (match_operand:V4SI 2 "s_register_operand" "w")
6559                        (match_operand:V4SI 3 "s_register_operand" "w")
6560                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6561          VRMLSLDAVHAQ_P_S))
6562   ]
6563   "TARGET_HAVE_MVE"
6564   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6565   [(set_attr "type" "mve_move")
6566    (set_attr "length""8")])
6567
6568 ;;
6569 ;; [vrmlsldavhaxq_p_s])
6570 ;;
6571 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6572   [
6573    (set (match_operand:DI 0 "s_register_operand" "=r")
6574         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6575                        (match_operand:V4SI 2 "s_register_operand" "w")
6576                        (match_operand:V4SI 3 "s_register_operand" "w")
6577                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6578          VRMLSLDAVHAXQ_P_S))
6579   ]
6580   "TARGET_HAVE_MVE"
6581   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6582   [(set_attr "type" "mve_move")
6583    (set_attr "length""8")])
6584 ;;
6585 ;; [vabdq_m_f])
6586 ;;
6587 (define_insn "mve_vabdq_m_f<mode>"
6588   [
6589    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6590         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6591                        (match_operand:MVE_0 2 "s_register_operand" "w")
6592                        (match_operand:MVE_0 3 "s_register_operand" "w")
6593                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6594          VABDQ_M_F))
6595   ]
6596   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6597   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6598   [(set_attr "type" "mve_move")
6599    (set_attr "length""8")])
6600
6601 ;;
6602 ;; [vaddq_m_f])
6603 ;;
6604 (define_insn "mve_vaddq_m_f<mode>"
6605   [
6606    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6607         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6608                        (match_operand:MVE_0 2 "s_register_operand" "w")
6609                        (match_operand:MVE_0 3 "s_register_operand" "w")
6610                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6611          VADDQ_M_F))
6612   ]
6613   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6614   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6615   [(set_attr "type" "mve_move")
6616    (set_attr "length""8")])
6617
6618 ;;
6619 ;; [vaddq_m_n_f])
6620 ;;
6621 (define_insn "mve_vaddq_m_n_f<mode>"
6622   [
6623    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6624         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6625                        (match_operand:MVE_0 2 "s_register_operand" "w")
6626                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6627                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6628          VADDQ_M_N_F))
6629   ]
6630   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6631   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
6632   [(set_attr "type" "mve_move")
6633    (set_attr "length""8")])
6634
6635 ;;
6636 ;; [vandq_m_f])
6637 ;;
6638 (define_insn "mve_vandq_m_f<mode>"
6639   [
6640    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6641         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6642                        (match_operand:MVE_0 2 "s_register_operand" "w")
6643                        (match_operand:MVE_0 3 "s_register_operand" "w")
6644                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6645          VANDQ_M_F))
6646   ]
6647   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6648   "vpst\;vandt %q0, %q2, %q3"
6649   [(set_attr "type" "mve_move")
6650    (set_attr "length""8")])
6651
6652 ;;
6653 ;; [vbicq_m_f])
6654 ;;
6655 (define_insn "mve_vbicq_m_f<mode>"
6656   [
6657    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6658         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6659                        (match_operand:MVE_0 2 "s_register_operand" "w")
6660                        (match_operand:MVE_0 3 "s_register_operand" "w")
6661                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6662          VBICQ_M_F))
6663   ]
6664   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6665   "vpst\;vbict %q0, %q2, %q3"
6666   [(set_attr "type" "mve_move")
6667    (set_attr "length""8")])
6668
6669 ;;
6670 ;; [vbrsrq_m_n_f])
6671 ;;
6672 (define_insn "mve_vbrsrq_m_n_f<mode>"
6673   [
6674    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6675         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6676                        (match_operand:MVE_0 2 "s_register_operand" "w")
6677                        (match_operand:SI 3 "s_register_operand" "r")
6678                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6679          VBRSRQ_M_N_F))
6680   ]
6681   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6682   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6683   [(set_attr "type" "mve_move")
6684    (set_attr "length""8")])
6685
6686 ;;
6687 ;; [vcaddq_rot270_m_f])
6688 ;;
6689 (define_insn "mve_vcaddq_rot270_m_f<mode>"
6690   [
6691    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6692         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6693                        (match_operand:MVE_0 2 "s_register_operand" "w")
6694                        (match_operand:MVE_0 3 "s_register_operand" "w")
6695                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6696          VCADDQ_ROT270_M_F))
6697   ]
6698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6699   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6700   [(set_attr "type" "mve_move")
6701    (set_attr "length""8")])
6702
6703 ;;
6704 ;; [vcaddq_rot90_m_f])
6705 ;;
6706 (define_insn "mve_vcaddq_rot90_m_f<mode>"
6707   [
6708    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6709         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6710                        (match_operand:MVE_0 2 "s_register_operand" "w")
6711                        (match_operand:MVE_0 3 "s_register_operand" "w")
6712                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6713          VCADDQ_ROT90_M_F))
6714   ]
6715   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6716   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6717   [(set_attr "type" "mve_move")
6718    (set_attr "length""8")])
6719
6720 ;;
6721 ;; [vcmlaq_m_f])
6722 ;;
6723 (define_insn "mve_vcmlaq_m_f<mode>"
6724   [
6725    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6726         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6727                        (match_operand:MVE_0 2 "s_register_operand" "w")
6728                        (match_operand:MVE_0 3 "s_register_operand" "w")
6729                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6730          VCMLAQ_M_F))
6731   ]
6732   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6733   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6734   [(set_attr "type" "mve_move")
6735    (set_attr "length""8")])
6736
6737 ;;
6738 ;; [vcmlaq_rot180_m_f])
6739 ;;
6740 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
6741   [
6742    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6743         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6744                        (match_operand:MVE_0 2 "s_register_operand" "w")
6745                        (match_operand:MVE_0 3 "s_register_operand" "w")
6746                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6747          VCMLAQ_ROT180_M_F))
6748   ]
6749   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6750   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6751   [(set_attr "type" "mve_move")
6752    (set_attr "length""8")])
6753
6754 ;;
6755 ;; [vcmlaq_rot270_m_f])
6756 ;;
6757 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
6758   [
6759    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6760         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6761                        (match_operand:MVE_0 2 "s_register_operand" "w")
6762                        (match_operand:MVE_0 3 "s_register_operand" "w")
6763                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6764          VCMLAQ_ROT270_M_F))
6765   ]
6766   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6767   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6768   [(set_attr "type" "mve_move")
6769    (set_attr "length""8")])
6770
6771 ;;
6772 ;; [vcmlaq_rot90_m_f])
6773 ;;
6774 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
6775   [
6776    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6777         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6778                        (match_operand:MVE_0 2 "s_register_operand" "w")
6779                        (match_operand:MVE_0 3 "s_register_operand" "w")
6780                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6781          VCMLAQ_ROT90_M_F))
6782   ]
6783   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6784   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6785   [(set_attr "type" "mve_move")
6786    (set_attr "length""8")])
6787
6788 ;;
6789 ;; [vcmulq_m_f])
6790 ;;
6791 (define_insn "mve_vcmulq_m_f<mode>"
6792   [
6793    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6794         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6795                        (match_operand:MVE_0 2 "s_register_operand" "w")
6796                        (match_operand:MVE_0 3 "s_register_operand" "w")
6797                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6798          VCMULQ_M_F))
6799   ]
6800   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6801   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6802   [(set_attr "type" "mve_move")
6803    (set_attr "length""8")])
6804
6805 ;;
6806 ;; [vcmulq_rot180_m_f])
6807 ;;
6808 (define_insn "mve_vcmulq_rot180_m_f<mode>"
6809   [
6810    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6811         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6812                        (match_operand:MVE_0 2 "s_register_operand" "w")
6813                        (match_operand:MVE_0 3 "s_register_operand" "w")
6814                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6815          VCMULQ_ROT180_M_F))
6816   ]
6817   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6818   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6819   [(set_attr "type" "mve_move")
6820    (set_attr "length""8")])
6821
6822 ;;
6823 ;; [vcmulq_rot270_m_f])
6824 ;;
6825 (define_insn "mve_vcmulq_rot270_m_f<mode>"
6826   [
6827    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6828         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6829                        (match_operand:MVE_0 2 "s_register_operand" "w")
6830                        (match_operand:MVE_0 3 "s_register_operand" "w")
6831                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6832          VCMULQ_ROT270_M_F))
6833   ]
6834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6835   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6836   [(set_attr "type" "mve_move")
6837    (set_attr "length""8")])
6838
6839 ;;
6840 ;; [vcmulq_rot90_m_f])
6841 ;;
6842 (define_insn "mve_vcmulq_rot90_m_f<mode>"
6843   [
6844    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6845         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6846                        (match_operand:MVE_0 2 "s_register_operand" "w")
6847                        (match_operand:MVE_0 3 "s_register_operand" "w")
6848                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6849          VCMULQ_ROT90_M_F))
6850   ]
6851   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6852   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6853   [(set_attr "type" "mve_move")
6854    (set_attr "length""8")])
6855
6856 ;;
6857 ;; [veorq_m_f])
6858 ;;
6859 (define_insn "mve_veorq_m_f<mode>"
6860   [
6861    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6862         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6863                        (match_operand:MVE_0 2 "s_register_operand" "w")
6864                        (match_operand:MVE_0 3 "s_register_operand" "w")
6865                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6866          VEORQ_M_F))
6867   ]
6868   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6869   "vpst\;veort %q0, %q2, %q3"
6870   [(set_attr "type" "mve_move")
6871    (set_attr "length""8")])
6872
6873 ;;
6874 ;; [vfmaq_m_f])
6875 ;;
6876 (define_insn "mve_vfmaq_m_f<mode>"
6877   [
6878    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6879         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6880                        (match_operand:MVE_0 2 "s_register_operand" "w")
6881                        (match_operand:MVE_0 3 "s_register_operand" "w")
6882                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6883          VFMAQ_M_F))
6884   ]
6885   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6886   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
6887   [(set_attr "type" "mve_move")
6888    (set_attr "length""8")])
6889
6890 ;;
6891 ;; [vfmaq_m_n_f])
6892 ;;
6893 (define_insn "mve_vfmaq_m_n_f<mode>"
6894   [
6895    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6896         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6897                        (match_operand:MVE_0 2 "s_register_operand" "w")
6898                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6899                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6900          VFMAQ_M_N_F))
6901   ]
6902   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6903   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
6904   [(set_attr "type" "mve_move")
6905    (set_attr "length""8")])
6906
6907 ;;
6908 ;; [vfmasq_m_n_f])
6909 ;;
6910 (define_insn "mve_vfmasq_m_n_f<mode>"
6911   [
6912    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6913         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6914                        (match_operand:MVE_0 2 "s_register_operand" "w")
6915                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6916                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6917          VFMASQ_M_N_F))
6918   ]
6919   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6920   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
6921   [(set_attr "type" "mve_move")
6922    (set_attr "length""8")])
6923
6924 ;;
6925 ;; [vfmsq_m_f])
6926 ;;
6927 (define_insn "mve_vfmsq_m_f<mode>"
6928   [
6929    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6930         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6931                        (match_operand:MVE_0 2 "s_register_operand" "w")
6932                        (match_operand:MVE_0 3 "s_register_operand" "w")
6933                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6934          VFMSQ_M_F))
6935   ]
6936   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6937   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
6938   [(set_attr "type" "mve_move")
6939    (set_attr "length""8")])
6940
6941 ;;
6942 ;; [vmaxnmq_m_f])
6943 ;;
6944 (define_insn "mve_vmaxnmq_m_f<mode>"
6945   [
6946    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6947         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6948                        (match_operand:MVE_0 2 "s_register_operand" "w")
6949                        (match_operand:MVE_0 3 "s_register_operand" "w")
6950                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6951          VMAXNMQ_M_F))
6952   ]
6953   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6954   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6955   [(set_attr "type" "mve_move")
6956    (set_attr "length""8")])
6957
6958 ;;
6959 ;; [vminnmq_m_f])
6960 ;;
6961 (define_insn "mve_vminnmq_m_f<mode>"
6962   [
6963    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6964         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6965                        (match_operand:MVE_0 2 "s_register_operand" "w")
6966                        (match_operand:MVE_0 3 "s_register_operand" "w")
6967                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6968          VMINNMQ_M_F))
6969   ]
6970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6971   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6972   [(set_attr "type" "mve_move")
6973    (set_attr "length""8")])
6974
6975 ;;
6976 ;; [vmulq_m_f])
6977 ;;
6978 (define_insn "mve_vmulq_m_f<mode>"
6979   [
6980    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6981         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6982                        (match_operand:MVE_0 2 "s_register_operand" "w")
6983                        (match_operand:MVE_0 3 "s_register_operand" "w")
6984                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6985          VMULQ_M_F))
6986   ]
6987   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6988   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
6989   [(set_attr "type" "mve_move")
6990    (set_attr "length""8")])
6991
6992 ;;
6993 ;; [vmulq_m_n_f])
6994 ;;
6995 (define_insn "mve_vmulq_m_n_f<mode>"
6996   [
6997    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6998         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6999                        (match_operand:MVE_0 2 "s_register_operand" "w")
7000                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7001                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7002          VMULQ_M_N_F))
7003   ]
7004   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7005   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7006   [(set_attr "type" "mve_move")
7007    (set_attr "length""8")])
7008
7009 ;;
7010 ;; [vornq_m_f])
7011 ;;
7012 (define_insn "mve_vornq_m_f<mode>"
7013   [
7014    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7015         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7016                        (match_operand:MVE_0 2 "s_register_operand" "w")
7017                        (match_operand:MVE_0 3 "s_register_operand" "w")
7018                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7019          VORNQ_M_F))
7020   ]
7021   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7022   "vpst\;vornt %q0, %q2, %q3"
7023   [(set_attr "type" "mve_move")
7024    (set_attr "length""8")])
7025
7026 ;;
7027 ;; [vorrq_m_f])
7028 ;;
7029 (define_insn "mve_vorrq_m_f<mode>"
7030   [
7031    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7032         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7033                        (match_operand:MVE_0 2 "s_register_operand" "w")
7034                        (match_operand:MVE_0 3 "s_register_operand" "w")
7035                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7036          VORRQ_M_F))
7037   ]
7038   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7039   "vpst\;vorrt %q0, %q2, %q3"
7040   [(set_attr "type" "mve_move")
7041    (set_attr "length""8")])
7042
7043 ;;
7044 ;; [vsubq_m_f])
7045 ;;
7046 (define_insn "mve_vsubq_m_f<mode>"
7047   [
7048    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7049         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7050                        (match_operand:MVE_0 2 "s_register_operand" "w")
7051                        (match_operand:MVE_0 3 "s_register_operand" "w")
7052                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7053          VSUBQ_M_F))
7054   ]
7055   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7056   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7057   [(set_attr "type" "mve_move")
7058    (set_attr "length""8")])
7059
7060 ;;
7061 ;; [vsubq_m_n_f])
7062 ;;
7063 (define_insn "mve_vsubq_m_n_f<mode>"
7064   [
7065    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7066         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7067                        (match_operand:MVE_0 2 "s_register_operand" "w")
7068                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7069                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7070          VSUBQ_M_N_F))
7071   ]
7072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7073   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7074   [(set_attr "type" "mve_move")
7075    (set_attr "length""8")])
7076
7077 ;;
7078 ;; [vstrbq_s vstrbq_u]
7079 ;;
7080 (define_insn "mve_vstrbq_<supf><mode>"
7081   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7082         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7083          VSTRBQ))
7084   ]
7085   "TARGET_HAVE_MVE"
7086 {
7087    rtx ops[2];
7088    int regno = REGNO (operands[1]);
7089    ops[1] = gen_rtx_REG (TImode, regno);
7090    ops[0]  = operands[0];
7091    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7092    return "";
7093 }
7094   [(set_attr "length" "4")])
7095
7096 ;;
7097 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7098 ;;
7099 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7100   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7101    (match_operand:MVE_2 1 "s_register_operand")
7102    (match_operand:MVE_2 2 "s_register_operand")
7103    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7104   "TARGET_HAVE_MVE"
7105 {
7106   rtx ind = XEXP (operands[0], 0);
7107   gcc_assert (REG_P (ind));
7108   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7109                                                               operands[2]));
7110   DONE;
7111 })
7112
7113 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7114   [(set (mem:BLK (scratch))
7115         (unspec:BLK
7116           [(match_operand:SI 0 "register_operand" "r")
7117            (match_operand:MVE_2 1 "s_register_operand" "w")
7118            (match_operand:MVE_2 2 "s_register_operand" "w")]
7119           VSTRBSOQ))]
7120   "TARGET_HAVE_MVE"
7121   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7122   [(set_attr "length" "4")])
7123
7124 ;;
7125 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7126 ;;
7127 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7128   [(set (mem:BLK (scratch))
7129         (unspec:BLK
7130                 [(match_operand:V4SI 0 "s_register_operand" "w")
7131                  (match_operand:SI 1 "immediate_operand" "i")
7132                  (match_operand:V4SI 2 "s_register_operand" "w")]
7133          VSTRWSBQ))
7134   ]
7135   "TARGET_HAVE_MVE"
7136 {
7137    rtx ops[3];
7138    ops[0] = operands[0];
7139    ops[1] = operands[1];
7140    ops[2] = operands[2];
7141    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7142    return "";
7143 }
7144   [(set_attr "length" "4")])
7145
7146 ;;
7147 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7148 ;;
7149 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7150   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7151         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7152                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7153          VLDRBGOQ))
7154   ]
7155   "TARGET_HAVE_MVE"
7156 {
7157    rtx ops[3];
7158    ops[0] = operands[0];
7159    ops[1] = operands[1];
7160    ops[2] = operands[2];
7161    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7162      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7163    else
7164      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7165    return "";
7166 }
7167   [(set_attr "length" "4")])
7168
7169 ;;
7170 ;; [vldrbq_s vldrbq_u]
7171 ;;
7172 (define_insn "mve_vldrbq_<supf><mode>"
7173   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7174         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7175          VLDRBQ))
7176   ]
7177   "TARGET_HAVE_MVE"
7178 {
7179    rtx ops[2];
7180    int regno = REGNO (operands[0]);
7181    ops[0] = gen_rtx_REG (TImode, regno);
7182    ops[1]  = operands[1];
7183    if (<V_sz_elem> == 8)
7184      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7185    else
7186      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7187    return "";
7188 }
7189   [(set_attr "length" "4")])
7190
7191 ;;
7192 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7193 ;;
7194 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7195   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7196         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7197                       (match_operand:SI 2 "immediate_operand" "i")]
7198          VLDRWGBQ))
7199   ]
7200   "TARGET_HAVE_MVE"
7201 {
7202    rtx ops[3];
7203    ops[0] = operands[0];
7204    ops[1] = operands[1];
7205    ops[2] = operands[2];
7206    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7207    return "";
7208 }
7209   [(set_attr "length" "4")])
7210
7211 ;;
7212 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7213 ;;
7214 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7215   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7216    (match_operand:MVE_2 1 "s_register_operand")
7217    (match_operand:MVE_2 2 "s_register_operand")
7218    (match_operand:HI 3 "vpr_register_operand" "Up")
7219    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7220   "TARGET_HAVE_MVE"
7221 {
7222   rtx ind = XEXP (operands[0], 0);
7223   gcc_assert (REG_P (ind));
7224   emit_insn (
7225     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7226                                                        operands[2],
7227                                                        operands[3]));
7228   DONE;
7229 })
7230
7231 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7232   [(set (mem:BLK (scratch))
7233         (unspec:BLK
7234           [(match_operand:SI 0 "register_operand" "r")
7235            (match_operand:MVE_2 1 "s_register_operand" "w")
7236            (match_operand:MVE_2 2 "s_register_operand" "w")
7237            (match_operand:HI 3 "vpr_register_operand" "Up")]
7238           VSTRBSOQ))]
7239   "TARGET_HAVE_MVE"
7240   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7241   [(set_attr "length" "8")])
7242
7243 ;;
7244 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7245 ;;
7246 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7247   [(set (mem:BLK (scratch))
7248         (unspec:BLK
7249                 [(match_operand:V4SI 0 "s_register_operand" "w")
7250                  (match_operand:SI 1 "immediate_operand" "i")
7251                  (match_operand:V4SI 2 "s_register_operand" "w")
7252                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7253          VSTRWSBQ))
7254   ]
7255   "TARGET_HAVE_MVE"
7256 {
7257    rtx ops[3];
7258    ops[0] = operands[0];
7259    ops[1] = operands[1];
7260    ops[2] = operands[2];
7261    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7262    return "";
7263 }
7264   [(set_attr "length" "8")])
7265
7266 ;;
7267 ;; [vstrbq_p_s vstrbq_p_u]
7268 ;;
7269 (define_insn "mve_vstrbq_p_<supf><mode>"
7270   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7271         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7272                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7273          VSTRBQ))
7274   ]
7275   "TARGET_HAVE_MVE"
7276 {
7277    rtx ops[2];
7278    int regno = REGNO (operands[1]);
7279    ops[1] = gen_rtx_REG (TImode, regno);
7280    ops[0]  = operands[0];
7281    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7282    return "";
7283 }
7284   [(set_attr "length" "8")])
7285
7286 ;;
7287 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7288 ;;
7289 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7290   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7291         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7292                        (match_operand:MVE_2 2 "s_register_operand" "w")
7293                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7294          VLDRBGOQ))
7295   ]
7296   "TARGET_HAVE_MVE"
7297 {
7298    rtx ops[4];
7299    ops[0] = operands[0];
7300    ops[1] = operands[1];
7301    ops[2] = operands[2];
7302    ops[3] = operands[3];
7303    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7304      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7305    else
7306      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7307    return "";
7308 }
7309   [(set_attr "length" "8")])
7310
7311 ;;
7312 ;; [vldrbq_z_s vldrbq_z_u]
7313 ;;
7314 (define_insn "mve_vldrbq_z_<supf><mode>"
7315   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7316         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7317                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7318          VLDRBQ))
7319   ]
7320   "TARGET_HAVE_MVE"
7321 {
7322    rtx ops[2];
7323    int regno = REGNO (operands[0]);
7324    ops[0] = gen_rtx_REG (TImode, regno);
7325    ops[1]  = operands[1];
7326    if (<V_sz_elem> == 8)
7327      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7328    else
7329      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7330    return "";
7331 }
7332   [(set_attr "length" "8")])
7333
7334 ;;
7335 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7336 ;;
7337 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7338   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7339         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7340                       (match_operand:SI 2 "immediate_operand" "i")
7341                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7342          VLDRWGBQ))
7343   ]
7344   "TARGET_HAVE_MVE"
7345 {
7346    rtx ops[3];
7347    ops[0] = operands[0];
7348    ops[1] = operands[1];
7349    ops[2] = operands[2];
7350    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7351    return "";
7352 }
7353   [(set_attr "length" "8")])
7354
7355 ;;
7356 ;; [vldrhq_f]
7357 ;;
7358 (define_insn "mve_vldrhq_fv8hf"
7359   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7360         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7361          VLDRHQ_F))
7362   ]
7363   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7364 {
7365    rtx ops[2];
7366    int regno = REGNO (operands[0]);
7367    ops[0] = gen_rtx_REG (TImode, regno);
7368    ops[1]  = operands[1];
7369    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7370    return "";
7371 }
7372   [(set_attr "length" "4")])
7373
7374 ;;
7375 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7376 ;;
7377 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7378   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7379         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7380                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7381         VLDRHGOQ))
7382   ]
7383   "TARGET_HAVE_MVE"
7384 {
7385    rtx ops[3];
7386    ops[0] = operands[0];
7387    ops[1] = operands[1];
7388    ops[2] = operands[2];
7389    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7390      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7391    else
7392      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7393    return "";
7394 }
7395   [(set_attr "length" "4")])
7396
7397 ;;
7398 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7399 ;;
7400 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7401   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7402         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7403                        (match_operand:MVE_6 2 "s_register_operand" "w")
7404                        (match_operand:HI 3 "vpr_register_operand" "Up")
7405         ]VLDRHGOQ))
7406   ]
7407   "TARGET_HAVE_MVE"
7408 {
7409    rtx ops[4];
7410    ops[0] = operands[0];
7411    ops[1] = operands[1];
7412    ops[2] = operands[2];
7413    ops[3] = operands[3];
7414    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7415      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7416    else
7417      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7418    return "";
7419 }
7420  [(set_attr "length" "8")])
7421
7422 ;;
7423 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7424 ;;
7425 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7426   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7427         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7428                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7429         VLDRHGSOQ))
7430   ]
7431   "TARGET_HAVE_MVE"
7432 {
7433    rtx ops[3];
7434    ops[0] = operands[0];
7435    ops[1] = operands[1];
7436    ops[2] = operands[2];
7437       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7438      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7439    else
7440      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7441    return "";
7442 }
7443   [(set_attr "length" "4")])
7444
7445 ;;
7446 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7447 ;;
7448 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7449   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7450         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7451                        (match_operand:MVE_6 2 "s_register_operand" "w")
7452                        (match_operand:HI 3 "vpr_register_operand" "Up")
7453         ]VLDRHGSOQ))
7454   ]
7455   "TARGET_HAVE_MVE"
7456 {
7457    rtx ops[4];
7458    ops[0] = operands[0];
7459    ops[1] = operands[1];
7460    ops[2] = operands[2];
7461    ops[3] = operands[3];
7462    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7463      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7464    else
7465      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7466    return "";
7467 }
7468   [(set_attr "length" "8")])
7469
7470 ;;
7471 ;; [vldrhq_s, vldrhq_u]
7472 ;;
7473 (define_insn "mve_vldrhq_<supf><mode>"
7474   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7475         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7476          VLDRHQ))
7477   ]
7478   "TARGET_HAVE_MVE"
7479 {
7480    rtx ops[2];
7481    int regno = REGNO (operands[0]);
7482    ops[0] = gen_rtx_REG (TImode, regno);
7483    ops[1]  = operands[1];
7484    if (<V_sz_elem> == 16)
7485      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7486    else
7487      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7488    return "";
7489 }
7490   [(set_attr "length" "4")])
7491
7492 ;;
7493 ;; [vldrhq_z_f]
7494 ;;
7495 (define_insn "mve_vldrhq_z_fv8hf"
7496   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7497         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7498         (match_operand:HI 2 "vpr_register_operand" "Up")]
7499          VLDRHQ_F))
7500   ]
7501   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7502 {
7503    rtx ops[2];
7504    int regno = REGNO (operands[0]);
7505    ops[0] = gen_rtx_REG (TImode, regno);
7506    ops[1]  = operands[1];
7507    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7508    return "";
7509 }
7510   [(set_attr "length" "8")])
7511
7512 ;;
7513 ;; [vldrhq_z_s vldrhq_z_u]
7514 ;;
7515 (define_insn "mve_vldrhq_z_<supf><mode>"
7516   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7517         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7518         (match_operand:HI 2 "vpr_register_operand" "Up")]
7519          VLDRHQ))
7520   ]
7521   "TARGET_HAVE_MVE"
7522 {
7523    rtx ops[2];
7524    int regno = REGNO (operands[0]);
7525    ops[0] = gen_rtx_REG (TImode, regno);
7526    ops[1]  = operands[1];
7527    if (<V_sz_elem> == 16)
7528      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7529    else
7530      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7531    return "";
7532 }
7533   [(set_attr "length" "8")])
7534
7535 ;;
7536 ;; [vldrwq_f]
7537 ;;
7538 (define_insn "mve_vldrwq_fv4sf"
7539   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7540         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7541          VLDRWQ_F))
7542   ]
7543   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7544 {
7545    rtx ops[2];
7546    int regno = REGNO (operands[0]);
7547    ops[0] = gen_rtx_REG (TImode, regno);
7548    ops[1]  = operands[1];
7549    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7550    return "";
7551 }
7552   [(set_attr "length" "4")])
7553
7554 ;;
7555 ;; [vldrwq_s vldrwq_u]
7556 ;;
7557 (define_insn "mve_vldrwq_<supf>v4si"
7558   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7559         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
7560          VLDRWQ))
7561   ]
7562   "TARGET_HAVE_MVE"
7563 {
7564    rtx ops[2];
7565    int regno = REGNO (operands[0]);
7566    ops[0] = gen_rtx_REG (TImode, regno);
7567    ops[1]  = operands[1];
7568    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7569    return "";
7570 }
7571   [(set_attr "length" "4")])
7572
7573 ;;
7574 ;; [vldrwq_z_f]
7575 ;;
7576 (define_insn "mve_vldrwq_z_fv4sf"
7577   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7578         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
7579         (match_operand:HI 2 "vpr_register_operand" "Up")]
7580          VLDRWQ_F))
7581   ]
7582   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7583 {
7584    rtx ops[2];
7585    int regno = REGNO (operands[0]);
7586    ops[0] = gen_rtx_REG (TImode, regno);
7587    ops[1]  = operands[1];
7588    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7589    return "";
7590 }
7591   [(set_attr "length" "8")])
7592
7593 ;;
7594 ;; [vldrwq_z_s vldrwq_z_u]
7595 ;;
7596 (define_insn "mve_vldrwq_z_<supf>v4si"
7597   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7598         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
7599         (match_operand:HI 2 "vpr_register_operand" "Up")]
7600          VLDRWQ))
7601   ]
7602   "TARGET_HAVE_MVE"
7603 {
7604    rtx ops[2];
7605    int regno = REGNO (operands[0]);
7606    ops[0] = gen_rtx_REG (TImode, regno);
7607    ops[1]  = operands[1];
7608    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7609    return "";
7610 }
7611   [(set_attr "length" "8")])
7612
7613 (define_expand "mve_vld1q_f<mode>"
7614   [(match_operand:MVE_0 0 "s_register_operand")
7615    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7616   ]
7617   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
7618 {
7619   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
7620   DONE;
7621 })
7622
7623 (define_expand "mve_vld1q_<supf><mode>"
7624   [(match_operand:MVE_2 0 "s_register_operand")
7625    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
7626   ]
7627   "TARGET_HAVE_MVE"
7628 {
7629   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
7630   DONE;
7631 })
7632
7633 ;;
7634 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
7635 ;;
7636 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
7637   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7638         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7639                       (match_operand:SI 2 "immediate_operand" "i")]
7640          VLDRDGBQ))
7641   ]
7642   "TARGET_HAVE_MVE"
7643 {
7644    rtx ops[3];
7645    ops[0] = operands[0];
7646    ops[1] = operands[1];
7647    ops[2] = operands[2];
7648    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
7649    return "";
7650 }
7651   [(set_attr "length" "4")])
7652
7653 ;;
7654 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
7655 ;;
7656 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
7657   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7658         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7659                       (match_operand:SI 2 "immediate_operand" "i")
7660                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7661          VLDRDGBQ))
7662   ]
7663   "TARGET_HAVE_MVE"
7664 {
7665    rtx ops[3];
7666    ops[0] = operands[0];
7667    ops[1] = operands[1];
7668    ops[2] = operands[2];
7669    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
7670    return "";
7671 }
7672   [(set_attr "length" "8")])
7673
7674 ;;
7675 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
7676 ;;
7677 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
7678  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7679        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7680                      (match_operand:V2DI 2 "s_register_operand" "w")]
7681         VLDRDGOQ))
7682  ]
7683  "TARGET_HAVE_MVE"
7684 {
7685   rtx ops[3];
7686   ops[0] = operands[0];
7687   ops[1] = operands[1];
7688   ops[2] = operands[2];
7689   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
7690   return "";
7691 }
7692  [(set_attr "length" "4")])
7693
7694 ;;
7695 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
7696 ;;
7697 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
7698  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7699        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7700                      (match_operand:V2DI 2 "s_register_operand" "w")
7701                      (match_operand:HI 3 "vpr_register_operand" "Up")]
7702         VLDRDGOQ))
7703  ]
7704  "TARGET_HAVE_MVE"
7705 {
7706   rtx ops[3];
7707   ops[0] = operands[0];
7708   ops[1] = operands[1];
7709   ops[2] = operands[2];
7710   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
7711   return "";
7712 }
7713  [(set_attr "length" "8")])
7714
7715 ;;
7716 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
7717 ;;
7718 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
7719   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7720         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7721                       (match_operand:V2DI 2 "s_register_operand" "w")]
7722          VLDRDGSOQ))
7723   ]
7724   "TARGET_HAVE_MVE"
7725 {
7726    rtx ops[3];
7727    ops[0] = operands[0];
7728    ops[1] = operands[1];
7729    ops[2] = operands[2];
7730    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7731    return "";
7732 }
7733   [(set_attr "length" "4")])
7734
7735 ;;
7736 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
7737 ;;
7738 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
7739   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7740         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7741                       (match_operand:V2DI 2 "s_register_operand" "w")
7742                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7743          VLDRDGSOQ))
7744   ]
7745   "TARGET_HAVE_MVE"
7746 {
7747    rtx ops[3];
7748    ops[0] = operands[0];
7749    ops[1] = operands[1];
7750    ops[2] = operands[2];
7751    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7752    return "";
7753 }
7754   [(set_attr "length" "8")])
7755
7756 ;;
7757 ;; [vldrhq_gather_offset_f]
7758 ;;
7759 (define_insn "mve_vldrhq_gather_offset_fv8hf"
7760   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7761         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7762                       (match_operand:V8HI 2 "s_register_operand" "w")]
7763          VLDRHQGO_F))
7764   ]
7765   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7766 {
7767    rtx ops[3];
7768    ops[0] = operands[0];
7769    ops[1] = operands[1];
7770    ops[2] = operands[2];
7771    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
7772    return "";
7773 }
7774   [(set_attr "length" "4")])
7775
7776 ;;
7777 ;; [vldrhq_gather_offset_z_f]
7778 ;;
7779 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
7780   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7781         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7782                       (match_operand:V8HI 2 "s_register_operand" "w")
7783                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7784          VLDRHQGO_F))
7785   ]
7786   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7787 {
7788    rtx ops[4];
7789    ops[0] = operands[0];
7790    ops[1] = operands[1];
7791    ops[2] = operands[2];
7792    ops[3] = operands[3];
7793    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
7794    return "";
7795 }
7796   [(set_attr "length" "8")])
7797
7798 ;;
7799 ;; [vldrhq_gather_shifted_offset_f]
7800 ;;
7801 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
7802   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7803         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7804                       (match_operand:V8HI 2 "s_register_operand" "w")]
7805          VLDRHQGSO_F))
7806   ]
7807   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7808 {
7809    rtx ops[3];
7810    ops[0] = operands[0];
7811    ops[1] = operands[1];
7812    ops[2] = operands[2];
7813    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7814    return "";
7815 }
7816   [(set_attr "length" "4")])
7817
7818 ;;
7819 ;; [vldrhq_gather_shifted_offset_z_f]
7820 ;;
7821 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
7822   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7823         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7824                       (match_operand:V8HI 2 "s_register_operand" "w")
7825                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7826          VLDRHQGSO_F))
7827   ]
7828   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7829 {
7830    rtx ops[4];
7831    ops[0] = operands[0];
7832    ops[1] = operands[1];
7833    ops[2] = operands[2];
7834    ops[3] = operands[3];
7835    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7836    return "";
7837 }
7838   [(set_attr "length" "8")])
7839
7840 ;;
7841 ;; [vldrwq_gather_base_f]
7842 ;;
7843 (define_insn "mve_vldrwq_gather_base_fv4sf"
7844   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7845         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7846                       (match_operand:SI 2 "immediate_operand" "i")]
7847          VLDRWQGB_F))
7848   ]
7849   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7850 {
7851    rtx ops[3];
7852    ops[0] = operands[0];
7853    ops[1] = operands[1];
7854    ops[2] = operands[2];
7855    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7856    return "";
7857 }
7858   [(set_attr "length" "4")])
7859
7860 ;;
7861 ;; [vldrwq_gather_base_z_f]
7862 ;;
7863 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
7864   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7865         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7866                       (match_operand:SI 2 "immediate_operand" "i")
7867                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7868          VLDRWQGB_F))
7869   ]
7870   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7871 {
7872    rtx ops[3];
7873    ops[0] = operands[0];
7874    ops[1] = operands[1];
7875    ops[2] = operands[2];
7876    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7877    return "";
7878 }
7879   [(set_attr "length" "8")])
7880
7881 ;;
7882 ;; [vldrwq_gather_offset_f]
7883 ;;
7884 (define_insn "mve_vldrwq_gather_offset_fv4sf"
7885   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7886         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7887                        (match_operand:V4SI 2 "s_register_operand" "w")]
7888          VLDRWQGO_F))
7889   ]
7890   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7891 {
7892    rtx ops[3];
7893    ops[0] = operands[0];
7894    ops[1] = operands[1];
7895    ops[2] = operands[2];
7896    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7897    return "";
7898 }
7899   [(set_attr "length" "4")])
7900
7901 ;;
7902 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
7903 ;;
7904 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
7905   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7906         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7907                        (match_operand:V4SI 2 "s_register_operand" "w")]
7908          VLDRWGOQ))
7909   ]
7910   "TARGET_HAVE_MVE"
7911 {
7912    rtx ops[3];
7913    ops[0] = operands[0];
7914    ops[1] = operands[1];
7915    ops[2] = operands[2];
7916    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7917    return "";
7918 }
7919   [(set_attr "length" "4")])
7920
7921 ;;
7922 ;; [vldrwq_gather_offset_z_f]
7923 ;;
7924 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
7925   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7926         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7927                       (match_operand:V4SI 2 "s_register_operand" "w")
7928                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7929          VLDRWQGO_F))
7930   ]
7931   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7932 {
7933    rtx ops[4];
7934    ops[0] = operands[0];
7935    ops[1] = operands[1];
7936    ops[2] = operands[2];
7937    ops[3] = operands[3];
7938    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7939    return "";
7940 }
7941   [(set_attr "length" "8")])
7942
7943 ;;
7944 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
7945 ;;
7946 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
7947   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7948         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7949                       (match_operand:V4SI 2 "s_register_operand" "w")
7950                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7951          VLDRWGOQ))
7952   ]
7953   "TARGET_HAVE_MVE"
7954 {
7955    rtx ops[4];
7956    ops[0] = operands[0];
7957    ops[1] = operands[1];
7958    ops[2] = operands[2];
7959    ops[3] = operands[3];
7960    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7961    return "";
7962 }
7963   [(set_attr "length" "8")])
7964
7965 ;;
7966 ;; [vldrwq_gather_shifted_offset_f]
7967 ;;
7968 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
7969   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7970         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7971                       (match_operand:V4SI 2 "s_register_operand" "w")]
7972          VLDRWQGSO_F))
7973   ]
7974   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7975 {
7976    rtx ops[3];
7977    ops[0] = operands[0];
7978    ops[1] = operands[1];
7979    ops[2] = operands[2];
7980    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
7981    return "";
7982 }
7983   [(set_attr "length" "4")])
7984
7985 ;;
7986 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
7987 ;;
7988 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
7989   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7990         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7991                       (match_operand:V4SI 2 "s_register_operand" "w")]
7992          VLDRWGSOQ))
7993   ]
7994   "TARGET_HAVE_MVE"
7995 {
7996    rtx ops[3];
7997    ops[0] = operands[0];
7998    ops[1] = operands[1];
7999    ops[2] = operands[2];
8000    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8001    return "";
8002 }
8003   [(set_attr "length" "4")])
8004
8005 ;;
8006 ;; [vldrwq_gather_shifted_offset_z_f]
8007 ;;
8008 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8009   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8010         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8011                       (match_operand:V4SI 2 "s_register_operand" "w")
8012                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8013          VLDRWQGSO_F))
8014   ]
8015   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8016 {
8017    rtx ops[4];
8018    ops[0] = operands[0];
8019    ops[1] = operands[1];
8020    ops[2] = operands[2];
8021    ops[3] = operands[3];
8022    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8023    return "";
8024 }
8025   [(set_attr "length" "8")])
8026
8027 ;;
8028 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8029 ;;
8030 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8031   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8032         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8033                       (match_operand:V4SI 2 "s_register_operand" "w")
8034                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8035          VLDRWGSOQ))
8036   ]
8037   "TARGET_HAVE_MVE"
8038 {
8039    rtx ops[4];
8040    ops[0] = operands[0];
8041    ops[1] = operands[1];
8042    ops[2] = operands[2];
8043    ops[3] = operands[3];
8044    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8045    return "";
8046 }
8047   [(set_attr "length" "8")])
8048
8049 ;;
8050 ;; [vstrhq_f]
8051 ;;
8052 (define_insn "mve_vstrhq_fv8hf"
8053   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8054         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8055          VSTRHQ_F))
8056   ]
8057   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8058 {
8059    rtx ops[2];
8060    int regno = REGNO (operands[1]);
8061    ops[1] = gen_rtx_REG (TImode, regno);
8062    ops[0]  = operands[0];
8063    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8064    return "";
8065 }
8066   [(set_attr "length" "4")])
8067
8068 ;;
8069 ;; [vstrhq_p_f]
8070 ;;
8071 (define_insn "mve_vstrhq_p_fv8hf"
8072   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8073         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8074                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8075          VSTRHQ_F))
8076   ]
8077   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8078 {
8079    rtx ops[2];
8080    int regno = REGNO (operands[1]);
8081    ops[1] = gen_rtx_REG (TImode, regno);
8082    ops[0]  = operands[0];
8083    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8084    return "";
8085 }
8086   [(set_attr "length" "8")])
8087
8088 ;;
8089 ;; [vstrhq_p_s vstrhq_p_u]
8090 ;;
8091 (define_insn "mve_vstrhq_p_<supf><mode>"
8092   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8093         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8094                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8095          VSTRHQ))
8096   ]
8097   "TARGET_HAVE_MVE"
8098 {
8099    rtx ops[2];
8100    int regno = REGNO (operands[1]);
8101    ops[1] = gen_rtx_REG (TImode, regno);
8102    ops[0]  = operands[0];
8103    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8104    return "";
8105 }
8106   [(set_attr "length" "8")])
8107
8108 ;;
8109 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8110 ;;
8111 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8112   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8113    (match_operand:MVE_6 1 "s_register_operand")
8114    (match_operand:MVE_6 2 "s_register_operand")
8115    (match_operand:HI 3 "vpr_register_operand")
8116    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8117   "TARGET_HAVE_MVE"
8118 {
8119   rtx ind = XEXP (operands[0], 0);
8120   gcc_assert (REG_P (ind));
8121   emit_insn (
8122     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8123                                                        operands[2],
8124                                                        operands[3]));
8125   DONE;
8126 })
8127
8128 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8129   [(set (mem:BLK (scratch))
8130         (unspec:BLK
8131           [(match_operand:SI 0 "register_operand" "r")
8132            (match_operand:MVE_6 1 "s_register_operand" "w")
8133            (match_operand:MVE_6 2 "s_register_operand" "w")
8134            (match_operand:HI 3 "vpr_register_operand" "Up")]
8135           VSTRHSOQ))]
8136   "TARGET_HAVE_MVE"
8137   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8138   [(set_attr "length" "8")])
8139
8140 ;;
8141 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8142 ;;
8143 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8144   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8145    (match_operand:MVE_6 1 "s_register_operand")
8146    (match_operand:MVE_6 2 "s_register_operand")
8147    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8148   "TARGET_HAVE_MVE"
8149 {
8150   rtx ind = XEXP (operands[0], 0);
8151   gcc_assert (REG_P (ind));
8152   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8153                                                               operands[2]));
8154   DONE;
8155 })
8156
8157 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8158   [(set (mem:BLK (scratch))
8159         (unspec:BLK
8160           [(match_operand:SI 0 "register_operand" "r")
8161            (match_operand:MVE_6 1 "s_register_operand" "w")
8162            (match_operand:MVE_6 2 "s_register_operand" "w")]
8163           VSTRHSOQ))]
8164   "TARGET_HAVE_MVE"
8165   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8166   [(set_attr "length" "4")])
8167
8168 ;;
8169 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8170 ;;
8171 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8172   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8173    (match_operand:MVE_6 1 "s_register_operand")
8174    (match_operand:MVE_6 2 "s_register_operand")
8175    (match_operand:HI 3 "vpr_register_operand")
8176    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8177   "TARGET_HAVE_MVE"
8178 {
8179   rtx ind = XEXP (operands[0], 0);
8180   gcc_assert (REG_P (ind));
8181   emit_insn (
8182     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8183                                                                operands[2],
8184                                                                operands[3]));
8185   DONE;
8186 })
8187
8188 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8189   [(set (mem:BLK (scratch))
8190         (unspec:BLK
8191           [(match_operand:SI 0 "register_operand" "r")
8192            (match_operand:MVE_6 1 "s_register_operand" "w")
8193            (match_operand:MVE_6 2 "s_register_operand" "w")
8194            (match_operand:HI 3 "vpr_register_operand" "Up")]
8195           VSTRHSSOQ))]
8196   "TARGET_HAVE_MVE"
8197   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8198   [(set_attr "length" "8")])
8199
8200 ;;
8201 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8202 ;;
8203 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8204   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8205    (match_operand:MVE_6 1 "s_register_operand")
8206    (match_operand:MVE_6 2 "s_register_operand")
8207    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8208   "TARGET_HAVE_MVE"
8209 {
8210   rtx ind = XEXP (operands[0], 0);
8211   gcc_assert (REG_P (ind));
8212   emit_insn (
8213     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8214                                                              operands[2]));
8215   DONE;
8216 })
8217
8218 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8219   [(set (mem:BLK (scratch))
8220         (unspec:BLK
8221           [(match_operand:SI 0 "register_operand" "r")
8222            (match_operand:MVE_6 1 "s_register_operand" "w")
8223            (match_operand:MVE_6 2 "s_register_operand" "w")]
8224           VSTRHSSOQ))]
8225   "TARGET_HAVE_MVE"
8226   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8227   [(set_attr "length" "4")])
8228
8229 ;;
8230 ;; [vstrhq_s, vstrhq_u]
8231 ;;
8232 (define_insn "mve_vstrhq_<supf><mode>"
8233   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8234         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8235          VSTRHQ))
8236   ]
8237   "TARGET_HAVE_MVE"
8238 {
8239    rtx ops[2];
8240    int regno = REGNO (operands[1]);
8241    ops[1] = gen_rtx_REG (TImode, regno);
8242    ops[0]  = operands[0];
8243    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8244    return "";
8245 }
8246   [(set_attr "length" "4")])
8247
8248 ;;
8249 ;; [vstrwq_f]
8250 ;;
8251 (define_insn "mve_vstrwq_fv4sf"
8252   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8253         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8254          VSTRWQ_F))
8255   ]
8256   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8257 {
8258    rtx ops[2];
8259    int regno = REGNO (operands[1]);
8260    ops[1] = gen_rtx_REG (TImode, regno);
8261    ops[0]  = operands[0];
8262    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8263    return "";
8264 }
8265   [(set_attr "length" "4")])
8266
8267 ;;
8268 ;; [vstrwq_p_f]
8269 ;;
8270 (define_insn "mve_vstrwq_p_fv4sf"
8271   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8272         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8273                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8274          VSTRWQ_F))
8275   ]
8276   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8277 {
8278    rtx ops[2];
8279    int regno = REGNO (operands[1]);
8280    ops[1] = gen_rtx_REG (TImode, regno);
8281    ops[0]  = operands[0];
8282    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8283    return "";
8284 }
8285   [(set_attr "length" "8")])
8286
8287 ;;
8288 ;; [vstrwq_p_s vstrwq_p_u]
8289 ;;
8290 (define_insn "mve_vstrwq_p_<supf>v4si"
8291   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8292         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8293                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8294          VSTRWQ))
8295   ]
8296   "TARGET_HAVE_MVE"
8297 {
8298    rtx ops[2];
8299    int regno = REGNO (operands[1]);
8300    ops[1] = gen_rtx_REG (TImode, regno);
8301    ops[0]  = operands[0];
8302    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8303    return "";
8304 }
8305   [(set_attr "length" "8")])
8306
8307 ;;
8308 ;; [vstrwq_s vstrwq_u]
8309 ;;
8310 (define_insn "mve_vstrwq_<supf>v4si"
8311   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8312         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8313          VSTRWQ))
8314   ]
8315   "TARGET_HAVE_MVE"
8316 {
8317    rtx ops[2];
8318    int regno = REGNO (operands[1]);
8319    ops[1] = gen_rtx_REG (TImode, regno);
8320    ops[0]  = operands[0];
8321    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8322    return "";
8323 }
8324   [(set_attr "length" "4")])
8325
8326 (define_expand "mve_vst1q_f<mode>"
8327   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8328    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8329   ]
8330   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8331 {
8332   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8333   DONE;
8334 })
8335
8336 (define_expand "mve_vst1q_<supf><mode>"
8337   [(match_operand:MVE_2 0 "mve_memory_operand")
8338    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8339   ]
8340   "TARGET_HAVE_MVE"
8341 {
8342   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8343   DONE;
8344 })
8345
8346 ;;
8347 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8348 ;;
8349 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8350   [(set (mem:BLK (scratch))
8351         (unspec:BLK
8352                 [(match_operand:V2DI 0 "s_register_operand" "w")
8353                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8354                  (match_operand:V2DI 2 "s_register_operand" "w")
8355                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8356          VSTRDSBQ))
8357   ]
8358   "TARGET_HAVE_MVE"
8359 {
8360    rtx ops[3];
8361    ops[0] = operands[0];
8362    ops[1] = operands[1];
8363    ops[2] = operands[2];
8364    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8365    return "";
8366 }
8367   [(set_attr "length" "8")])
8368
8369 ;;
8370 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8371 ;;
8372 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8373   [(set (mem:BLK (scratch))
8374         (unspec:BLK
8375                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8376                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8377                  (match_operand:V2DI 2 "s_register_operand" "w")]
8378          VSTRDSBQ))
8379   ]
8380   "TARGET_HAVE_MVE"
8381 {
8382    rtx ops[3];
8383    ops[0] = operands[0];
8384    ops[1] = operands[1];
8385    ops[2] = operands[2];
8386    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8387    return "";
8388 }
8389   [(set_attr "length" "4")])
8390
8391 ;;
8392 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8393 ;;
8394 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8395   [(match_operand:V2DI 0 "mve_scatter_memory")
8396    (match_operand:V2DI 1 "s_register_operand")
8397    (match_operand:V2DI 2 "s_register_operand")
8398    (match_operand:HI 3 "vpr_register_operand")
8399    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8400   "TARGET_HAVE_MVE"
8401 {
8402   rtx ind = XEXP (operands[0], 0);
8403   gcc_assert (REG_P (ind));
8404   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8405                                                               operands[2],
8406                                                               operands[3]));
8407   DONE;
8408 })
8409
8410 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8411   [(set (mem:BLK (scratch))
8412         (unspec:BLK
8413           [(match_operand:SI 0 "register_operand" "r")
8414            (match_operand:V2DI 1 "s_register_operand" "w")
8415            (match_operand:V2DI 2 "s_register_operand" "w")
8416            (match_operand:HI 3 "vpr_register_operand" "Up")]
8417           VSTRDSOQ))]
8418   "TARGET_HAVE_MVE"
8419   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8420   [(set_attr "length" "8")])
8421
8422 ;;
8423 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8424 ;;
8425 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8426   [(match_operand:V2DI 0 "mve_scatter_memory")
8427    (match_operand:V2DI 1 "s_register_operand")
8428    (match_operand:V2DI 2 "s_register_operand")
8429    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8430   "TARGET_HAVE_MVE"
8431 {
8432   rtx ind = XEXP (operands[0], 0);
8433   gcc_assert (REG_P (ind));
8434   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8435                                                             operands[2]));
8436   DONE;
8437 })
8438
8439 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8440   [(set (mem:BLK (scratch))
8441         (unspec:BLK
8442           [(match_operand:SI 0 "register_operand" "r")
8443            (match_operand:V2DI 1 "s_register_operand" "w")
8444            (match_operand:V2DI 2 "s_register_operand" "w")]
8445           VSTRDSOQ))]
8446   "TARGET_HAVE_MVE"
8447   "vstrd.64\t%q2, [%0, %q1]"
8448   [(set_attr "length" "4")])
8449
8450 ;;
8451 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8452 ;;
8453 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8454   [(match_operand:V2DI 0 "mve_scatter_memory")
8455    (match_operand:V2DI 1 "s_register_operand")
8456    (match_operand:V2DI 2 "s_register_operand")
8457    (match_operand:HI 3 "vpr_register_operand")
8458    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8459   "TARGET_HAVE_MVE"
8460 {
8461   rtx ind = XEXP (operands[0], 0);
8462   gcc_assert (REG_P (ind));
8463   emit_insn (
8464     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8465                                                              operands[2],
8466                                                              operands[3]));
8467   DONE;
8468 })
8469
8470 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8471   [(set (mem:BLK (scratch))
8472         (unspec:BLK
8473           [(match_operand:SI 0 "register_operand" "r")
8474            (match_operand:V2DI 1 "s_register_operand" "w")
8475            (match_operand:V2DI 2 "s_register_operand" "w")
8476            (match_operand:HI 3 "vpr_register_operand" "Up")]
8477           VSTRDSSOQ))]
8478   "TARGET_HAVE_MVE"
8479   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8480   [(set_attr "length" "8")])
8481
8482 ;;
8483 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8484 ;;
8485 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8486   [(match_operand:V2DI 0 "mve_scatter_memory")
8487    (match_operand:V2DI 1 "s_register_operand")
8488    (match_operand:V2DI 2 "s_register_operand")
8489    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8490   "TARGET_HAVE_MVE"
8491 {
8492   rtx ind = XEXP (operands[0], 0);
8493   gcc_assert (REG_P (ind));
8494   emit_insn (
8495     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8496                                                            operands[2]));
8497   DONE;
8498 })
8499
8500 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8501   [(set (mem:BLK (scratch))
8502         (unspec:BLK
8503           [(match_operand:SI 0 "register_operand" "r")
8504            (match_operand:V2DI 1 "s_register_operand" "w")
8505            (match_operand:V2DI 2 "s_register_operand" "w")]
8506           VSTRDSSOQ))]
8507   "TARGET_HAVE_MVE"
8508   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8509   [(set_attr "length" "4")])
8510
8511 ;;
8512 ;; [vstrhq_scatter_offset_f]
8513 ;;
8514 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8515   [(match_operand:V8HI 0 "mve_scatter_memory")
8516    (match_operand:V8HI 1 "s_register_operand")
8517    (match_operand:V8HF 2 "s_register_operand")
8518    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8519   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8520 {
8521   rtx ind = XEXP (operands[0], 0);
8522   gcc_assert (REG_P (ind));
8523   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8524                                                        operands[2]));
8525   DONE;
8526 })
8527
8528 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8529   [(set (mem:BLK (scratch))
8530         (unspec:BLK
8531           [(match_operand:SI 0 "register_operand" "r")
8532            (match_operand:V8HI 1 "s_register_operand" "w")
8533            (match_operand:V8HF 2 "s_register_operand" "w")]
8534           VSTRHQSO_F))]
8535   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8536   "vstrh.16\t%q2, [%0, %q1]"
8537   [(set_attr "length" "4")])
8538
8539 ;;
8540 ;; [vstrhq_scatter_offset_p_f]
8541 ;;
8542 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8543   [(match_operand:V8HI 0 "mve_scatter_memory")
8544    (match_operand:V8HI 1 "s_register_operand")
8545    (match_operand:V8HF 2 "s_register_operand")
8546    (match_operand:HI 3 "vpr_register_operand")
8547    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8548   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8549 {
8550   rtx ind = XEXP (operands[0], 0);
8551   gcc_assert (REG_P (ind));
8552   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8553                                                          operands[2],
8554                                                          operands[3]));
8555   DONE;
8556 })
8557
8558 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8559   [(set (mem:BLK (scratch))
8560         (unspec:BLK
8561           [(match_operand:SI 0 "register_operand" "r")
8562            (match_operand:V8HI 1 "s_register_operand" "w")
8563            (match_operand:V8HF 2 "s_register_operand" "w")
8564            (match_operand:HI 3 "vpr_register_operand" "Up")]
8565           VSTRHQSO_F))]
8566   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8567   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8568   [(set_attr "length" "8")])
8569
8570 ;;
8571 ;; [vstrhq_scatter_shifted_offset_f]
8572 ;;
8573 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8574   [(match_operand:V8HI 0 "memory_operand" "=Us")
8575    (match_operand:V8HI 1 "s_register_operand" "w")
8576    (match_operand:V8HF 2 "s_register_operand" "w")
8577    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8579 {
8580   rtx ind = XEXP (operands[0], 0);
8581   gcc_assert (REG_P (ind));
8582   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8583                                                                operands[2]));
8584   DONE;
8585 })
8586
8587 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8588   [(set (mem:BLK (scratch))
8589         (unspec:BLK
8590           [(match_operand:SI 0 "register_operand" "r")
8591            (match_operand:V8HI 1 "s_register_operand" "w")
8592            (match_operand:V8HF 2 "s_register_operand" "w")]
8593           VSTRHQSSO_F))]
8594   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8595   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8596   [(set_attr "length" "4")])
8597
8598 ;;
8599 ;; [vstrhq_scatter_shifted_offset_p_f]
8600 ;;
8601 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8602   [(match_operand:V8HI 0 "memory_operand" "=Us")
8603    (match_operand:V8HI 1 "s_register_operand" "w")
8604    (match_operand:V8HF 2 "s_register_operand" "w")
8605    (match_operand:HI 3 "vpr_register_operand" "Up")
8606    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8607   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8608 {
8609   rtx ind = XEXP (operands[0], 0);
8610   gcc_assert (REG_P (ind));
8611   emit_insn (
8612     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8613                                                         operands[2],
8614                                                         operands[3]));
8615   DONE;
8616 })
8617
8618 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
8619   [(set (mem:BLK (scratch))
8620         (unspec:BLK
8621           [(match_operand:SI 0 "register_operand" "r")
8622            (match_operand:V8HI 1 "s_register_operand" "w")
8623            (match_operand:V8HF 2 "s_register_operand" "w")
8624            (match_operand:HI 3 "vpr_register_operand" "Up")]
8625           VSTRHQSSO_F))]
8626   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8627   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
8628   [(set_attr "length" "8")])
8629
8630 ;;
8631 ;; [vstrwq_scatter_base_f]
8632 ;;
8633 (define_insn "mve_vstrwq_scatter_base_fv4sf"
8634   [(set (mem:BLK (scratch))
8635         (unspec:BLK
8636                 [(match_operand:V4SI 0 "s_register_operand" "w")
8637                  (match_operand:SI 1 "immediate_operand" "i")
8638                  (match_operand:V4SF 2 "s_register_operand" "w")]
8639          VSTRWQSB_F))
8640   ]
8641   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8642 {
8643    rtx ops[3];
8644    ops[0] = operands[0];
8645    ops[1] = operands[1];
8646    ops[2] = operands[2];
8647    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
8648    return "";
8649 }
8650   [(set_attr "length" "4")])
8651
8652 ;;
8653 ;; [vstrwq_scatter_base_p_f]
8654 ;;
8655 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
8656   [(set (mem:BLK (scratch))
8657         (unspec:BLK
8658                 [(match_operand:V4SI 0 "s_register_operand" "w")
8659                  (match_operand:SI 1 "immediate_operand" "i")
8660                  (match_operand:V4SF 2 "s_register_operand" "w")
8661                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8662          VSTRWQSB_F))
8663   ]
8664   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8665 {
8666    rtx ops[3];
8667    ops[0] = operands[0];
8668    ops[1] = operands[1];
8669    ops[2] = operands[2];
8670    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8671    return "";
8672 }
8673   [(set_attr "length" "8")])
8674
8675 ;;
8676 ;; [vstrwq_scatter_offset_f]
8677 ;;
8678 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
8679   [(match_operand:V4SI 0 "mve_scatter_memory")
8680    (match_operand:V4SI 1 "s_register_operand")
8681    (match_operand:V4SF 2 "s_register_operand")
8682    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8683   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8684 {
8685   rtx ind = XEXP (operands[0], 0);
8686   gcc_assert (REG_P (ind));
8687   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
8688                                                        operands[2]));
8689   DONE;
8690 })
8691
8692 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
8693   [(set (mem:BLK (scratch))
8694         (unspec:BLK
8695           [(match_operand:SI 0 "register_operand" "r")
8696            (match_operand:V4SI 1 "s_register_operand" "w")
8697            (match_operand:V4SF 2 "s_register_operand" "w")]
8698           VSTRWQSO_F))]
8699   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8700   "vstrw.32\t%q2, [%0, %q1]"
8701   [(set_attr "length" "4")])
8702
8703 ;;
8704 ;; [vstrwq_scatter_offset_p_f]
8705 ;;
8706 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
8707   [(match_operand:V4SI 0 "mve_scatter_memory")
8708    (match_operand:V4SI 1 "s_register_operand")
8709    (match_operand:V4SF 2 "s_register_operand")
8710    (match_operand:HI 3 "vpr_register_operand")
8711    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8712   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8713 {
8714   rtx ind = XEXP (operands[0], 0);
8715   gcc_assert (REG_P (ind));
8716   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
8717                                                          operands[2],
8718                                                          operands[3]));
8719   DONE;
8720 })
8721
8722 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
8723   [(set (mem:BLK (scratch))
8724         (unspec:BLK
8725           [(match_operand:SI 0 "register_operand" "r")
8726            (match_operand:V4SI 1 "s_register_operand" "w")
8727            (match_operand:V4SF 2 "s_register_operand" "w")
8728            (match_operand:HI 3 "vpr_register_operand" "Up")]
8729           VSTRWQSO_F))]
8730   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8731   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8732   [(set_attr "length" "8")])
8733
8734 ;;
8735 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8736 ;;
8737 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
8738   [(match_operand:V4SI 0 "mve_scatter_memory")
8739    (match_operand:V4SI 1 "s_register_operand")
8740    (match_operand:V4SI 2 "s_register_operand")
8741    (match_operand:HI 3 "vpr_register_operand")
8742    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8743   "TARGET_HAVE_MVE"
8744 {
8745   rtx ind = XEXP (operands[0], 0);
8746   gcc_assert (REG_P (ind));
8747   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
8748                                                               operands[2],
8749                                                               operands[3]));
8750   DONE;
8751 })
8752
8753 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
8754   [(set (mem:BLK (scratch))
8755         (unspec:BLK
8756           [(match_operand:SI 0 "register_operand" "r")
8757            (match_operand:V4SI 1 "s_register_operand" "w")
8758            (match_operand:V4SI 2 "s_register_operand" "w")
8759            (match_operand:HI 3 "vpr_register_operand" "Up")]
8760           VSTRWSOQ))]
8761   "TARGET_HAVE_MVE"
8762   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8763   [(set_attr "length" "8")])
8764
8765 ;;
8766 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8767 ;;
8768 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
8769   [(match_operand:V4SI 0 "mve_scatter_memory")
8770    (match_operand:V4SI 1 "s_register_operand")
8771    (match_operand:V4SI 2 "s_register_operand")
8772    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8773   "TARGET_HAVE_MVE"
8774 {
8775   rtx ind = XEXP (operands[0], 0);
8776   gcc_assert (REG_P (ind));
8777   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
8778                                                             operands[2]));
8779   DONE;
8780 })
8781
8782 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
8783   [(set (mem:BLK (scratch))
8784         (unspec:BLK
8785           [(match_operand:SI 0 "register_operand" "r")
8786            (match_operand:V4SI 1 "s_register_operand" "w")
8787            (match_operand:V4SI 2 "s_register_operand" "w")]
8788           VSTRWSOQ))]
8789   "TARGET_HAVE_MVE"
8790   "vstrw.32\t%q2, [%0, %q1]"
8791   [(set_attr "length" "4")])
8792
8793 ;;
8794 ;; [vstrwq_scatter_shifted_offset_f]
8795 ;;
8796 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
8797   [(match_operand:V4SI 0 "mve_scatter_memory")
8798    (match_operand:V4SI 1 "s_register_operand")
8799    (match_operand:V4SF 2 "s_register_operand")
8800    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8801   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8802 {
8803   rtx ind = XEXP (operands[0], 0);
8804   gcc_assert (REG_P (ind));
8805   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
8806                                                                operands[2]));
8807   DONE;
8808 })
8809
8810 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
8811   [(set (mem:BLK (scratch))
8812         (unspec:BLK
8813           [(match_operand:SI 0 "register_operand" "r")
8814            (match_operand:V4SI 1 "s_register_operand" "w")
8815            (match_operand:V4SF 2 "s_register_operand" "w")]
8816          VSTRWQSSO_F))]
8817   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8818   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8819   [(set_attr "length" "8")])
8820
8821 ;;
8822 ;; [vstrwq_scatter_shifted_offset_p_f]
8823 ;;
8824 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
8825   [(match_operand:V4SI 0 "mve_scatter_memory")
8826    (match_operand:V4SI 1 "s_register_operand")
8827    (match_operand:V4SF 2 "s_register_operand")
8828    (match_operand:HI 3 "vpr_register_operand")
8829    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8830   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8831 {
8832   rtx ind = XEXP (operands[0], 0);
8833   gcc_assert (REG_P (ind));
8834   emit_insn (
8835     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
8836                                                         operands[2],
8837                                                         operands[3]));
8838   DONE;
8839 })
8840
8841 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
8842   [(set (mem:BLK (scratch))
8843         (unspec:BLK
8844           [(match_operand:SI 0 "register_operand" "r")
8845            (match_operand:V4SI 1 "s_register_operand" "w")
8846            (match_operand:V4SF 2 "s_register_operand" "w")
8847            (match_operand:HI 3 "vpr_register_operand" "Up")]
8848           VSTRWQSSO_F))]
8849   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8850   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8851   [(set_attr "length" "8")])
8852
8853 ;;
8854 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
8855 ;;
8856 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
8857   [(match_operand:V4SI 0 "mve_scatter_memory")
8858    (match_operand:V4SI 1 "s_register_operand")
8859    (match_operand:V4SI 2 "s_register_operand")
8860    (match_operand:HI 3 "vpr_register_operand")
8861    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8862   "TARGET_HAVE_MVE"
8863 {
8864   rtx ind = XEXP (operands[0], 0);
8865   gcc_assert (REG_P (ind));
8866   emit_insn (
8867     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
8868                                                              operands[2],
8869                                                              operands[3]));
8870   DONE;
8871 })
8872
8873 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
8874   [(set (mem:BLK (scratch))
8875         (unspec:BLK
8876           [(match_operand:SI 0 "register_operand" "r")
8877            (match_operand:V4SI 1 "s_register_operand" "w")
8878            (match_operand:V4SI 2 "s_register_operand" "w")
8879            (match_operand:HI 3 "vpr_register_operand" "Up")]
8880           VSTRWSSOQ))]
8881   "TARGET_HAVE_MVE"
8882   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8883   [(set_attr "length" "8")])
8884
8885 ;;
8886 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
8887 ;;
8888 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
8889   [(match_operand:V4SI 0 "mve_scatter_memory")
8890    (match_operand:V4SI 1 "s_register_operand")
8891    (match_operand:V4SI 2 "s_register_operand")
8892    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8893   "TARGET_HAVE_MVE"
8894 {
8895   rtx ind = XEXP (operands[0], 0);
8896   gcc_assert (REG_P (ind));
8897   emit_insn (
8898     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
8899                                                            operands[2]));
8900   DONE;
8901 })
8902
8903 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
8904   [(set (mem:BLK (scratch))
8905         (unspec:BLK
8906           [(match_operand:SI 0 "register_operand" "r")
8907            (match_operand:V4SI 1 "s_register_operand" "w")
8908            (match_operand:V4SI 2 "s_register_operand" "w")]
8909           VSTRWSSOQ))]
8910   "TARGET_HAVE_MVE"
8911   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8912   [(set_attr "length" "4")])
8913
8914 ;;
8915 ;; [vaddq_s, vaddq_u])
8916 ;;
8917 (define_insn "mve_vaddq<mode>"
8918   [
8919    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
8920         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
8921                     (match_operand:MVE_2 2 "s_register_operand" "w")))
8922   ]
8923   "TARGET_HAVE_MVE"
8924   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
8925   [(set_attr "type" "mve_move")
8926 ])
8927
8928 ;;
8929 ;; [vaddq_f])
8930 ;;
8931 (define_insn "mve_vaddq_f<mode>"
8932   [
8933    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8934         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
8935                     (match_operand:MVE_0 2 "s_register_operand" "w")))
8936   ]
8937   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8938   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
8939   [(set_attr "type" "mve_move")
8940 ])
8941
8942 ;;
8943 ;; [vidupq_n_u])
8944 ;;
8945 (define_expand "mve_vidupq_n_u<mode>"
8946  [(match_operand:MVE_2 0 "s_register_operand")
8947   (match_operand:SI 1 "s_register_operand")
8948   (match_operand:SI 2 "mve_imm_selective_upto_8")]
8949  "TARGET_HAVE_MVE"
8950 {
8951   rtx temp = gen_reg_rtx (SImode);
8952   emit_move_insn (temp, operands[1]);
8953   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
8954   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
8955                                           operands[2], inc));
8956   DONE;
8957 })
8958
8959 ;;
8960 ;; [vidupq_u_insn])
8961 ;;
8962 (define_insn "mve_vidupq_u<mode>_insn"
8963  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8964        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
8965                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
8966          VIDUPQ))
8967   (set (match_operand:SI 1 "s_register_operand" "=Te")
8968        (plus:SI (match_dup 2)
8969                 (match_operand:SI 4 "immediate_operand" "i")))]
8970  "TARGET_HAVE_MVE"
8971  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
8972
8973 ;;
8974 ;; [vidupq_m_n_u])
8975 ;;
8976 (define_expand "mve_vidupq_m_n_u<mode>"
8977   [(match_operand:MVE_2 0 "s_register_operand")
8978    (match_operand:MVE_2 1 "s_register_operand")
8979    (match_operand:SI 2 "s_register_operand")
8980    (match_operand:SI 3 "mve_imm_selective_upto_8")
8981    (match_operand:HI 4 "vpr_register_operand")]
8982   "TARGET_HAVE_MVE"
8983 {
8984   rtx temp = gen_reg_rtx (SImode);
8985   emit_move_insn (temp, operands[2]);
8986   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
8987   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
8988                                              operands[2], operands[3],
8989                                              operands[4], inc));
8990   DONE;
8991 })
8992
8993 ;;
8994 ;; [vidupq_m_wb_u_insn])
8995 ;;
8996 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
8997  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8998        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
8999                       (match_operand:SI 3 "s_register_operand" "2")
9000                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9001                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9002         VIDUPQ_M))
9003   (set (match_operand:SI 2 "s_register_operand" "=Te")
9004        (plus:SI (match_dup 3)
9005                 (match_operand:SI 6 "immediate_operand" "i")))]
9006  "TARGET_HAVE_MVE"
9007  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9008  [(set_attr "length""8")])
9009
9010 ;;
9011 ;; [vddupq_n_u])
9012 ;;
9013 (define_expand "mve_vddupq_n_u<mode>"
9014  [(match_operand:MVE_2 0 "s_register_operand")
9015   (match_operand:SI 1 "s_register_operand")
9016   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9017  "TARGET_HAVE_MVE"
9018 {
9019   rtx temp = gen_reg_rtx (SImode);
9020   emit_move_insn (temp, operands[1]);
9021   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9022   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9023                                           operands[2], inc));
9024   DONE;
9025 })
9026
9027 ;;
9028 ;; [vddupq_u_insn])
9029 ;;
9030 (define_insn "mve_vddupq_u<mode>_insn"
9031  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9032        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9033                       (match_operand:SI 3 "immediate_operand" "i")]
9034         VDDUPQ))
9035   (set (match_operand:SI 1 "s_register_operand" "=Te")
9036        (minus:SI (match_dup 2)
9037                  (match_operand:SI 4 "immediate_operand" "i")))]
9038  "TARGET_HAVE_MVE"
9039  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9040
9041 ;;
9042 ;; [vddupq_m_n_u])
9043 ;;
9044 (define_expand "mve_vddupq_m_n_u<mode>"
9045   [(match_operand:MVE_2 0 "s_register_operand")
9046    (match_operand:MVE_2 1 "s_register_operand")
9047    (match_operand:SI 2 "s_register_operand")
9048    (match_operand:SI 3 "mve_imm_selective_upto_8")
9049    (match_operand:HI 4 "vpr_register_operand")]
9050   "TARGET_HAVE_MVE"
9051 {
9052   rtx temp = gen_reg_rtx (SImode);
9053   emit_move_insn (temp, operands[2]);
9054   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9055   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9056                                              operands[2], operands[3],
9057                                              operands[4], inc));
9058   DONE;
9059 })
9060
9061 ;;
9062 ;; [vddupq_m_wb_u_insn])
9063 ;;
9064 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9065  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9066        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9067                       (match_operand:SI 3 "s_register_operand" "2")
9068                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9069                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9070         VDDUPQ_M))
9071   (set (match_operand:SI 2 "s_register_operand" "=Te")
9072        (minus:SI (match_dup 3)
9073                  (match_operand:SI 6 "immediate_operand" "i")))]
9074  "TARGET_HAVE_MVE"
9075  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9076  [(set_attr "length""8")])
9077
9078 ;;
9079 ;; [vdwdupq_n_u])
9080 ;;
9081 (define_expand "mve_vdwdupq_n_u<mode>"
9082  [(match_operand:MVE_2 0 "s_register_operand")
9083   (match_operand:SI 1 "s_register_operand")
9084   (match_operand:DI 2 "s_register_operand")
9085   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9086  "TARGET_HAVE_MVE"
9087 {
9088   rtx ignore_wb = gen_reg_rtx (SImode);
9089   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9090                                               operands[1], operands[2],
9091                                               operands[3]));
9092   DONE;
9093 })
9094
9095 ;;
9096 ;; [vdwdupq_wb_u])
9097 ;;
9098 (define_expand "mve_vdwdupq_wb_u<mode>"
9099  [(match_operand:SI 0 "s_register_operand")
9100   (match_operand:SI 1 "s_register_operand")
9101   (match_operand:DI 2 "s_register_operand")
9102   (match_operand:SI 3 "mve_imm_selective_upto_8")
9103   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9104  "TARGET_HAVE_MVE"
9105 {
9106   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9107   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9108                                               operands[1], operands[2],
9109                                               operands[3]));
9110   DONE;
9111 })
9112
9113 ;;
9114 ;; [vdwdupq_wb_u_insn])
9115 ;;
9116 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9117   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9118         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9119                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9120                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9121          VDWDUPQ))
9122    (set (match_operand:SI 1 "s_register_operand" "=Te")
9123         (unspec:SI [(match_dup 2)
9124                     (subreg:SI (match_dup 3) 4)
9125                     (match_dup 4)]
9126          VDWDUPQ))]
9127   "TARGET_HAVE_MVE"
9128   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9129 )
9130
9131 ;;
9132 ;; [vdwdupq_m_n_u])
9133 ;;
9134 (define_expand "mve_vdwdupq_m_n_u<mode>"
9135  [(match_operand:MVE_2 0 "s_register_operand")
9136   (match_operand:MVE_2 1 "s_register_operand")
9137   (match_operand:SI 2 "s_register_operand")
9138   (match_operand:DI 3 "s_register_operand")
9139   (match_operand:SI 4 "mve_imm_selective_upto_8")
9140   (match_operand:HI 5 "vpr_register_operand")]
9141  "TARGET_HAVE_MVE"
9142 {
9143   rtx ignore_wb = gen_reg_rtx (SImode);
9144   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9145                                                 operands[1], operands[2],
9146                                                 operands[3], operands[4],
9147                                                 operands[5]));
9148   DONE;
9149 })
9150
9151 ;;
9152 ;; [vdwdupq_m_wb_u])
9153 ;;
9154 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9155  [(match_operand:SI 0 "s_register_operand")
9156   (match_operand:MVE_2 1 "s_register_operand")
9157   (match_operand:SI 2 "s_register_operand")
9158   (match_operand:DI 3 "s_register_operand")
9159   (match_operand:SI 4 "mve_imm_selective_upto_8")
9160   (match_operand:HI 5 "vpr_register_operand")]
9161  "TARGET_HAVE_MVE"
9162 {
9163   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9164   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9165                                                 operands[1], operands[2],
9166                                                 operands[3], operands[4],
9167                                                 operands[5]));
9168   DONE;
9169 })
9170
9171 ;;
9172 ;; [vdwdupq_m_wb_u_insn])
9173 ;;
9174 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9175   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9176         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9177                        (match_operand:SI 3 "s_register_operand" "1")
9178                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9179                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9180                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9181          VDWDUPQ_M))
9182    (set (match_operand:SI 1 "s_register_operand" "=Te")
9183         (unspec:SI [(match_dup 2)
9184                     (match_dup 3)
9185                     (subreg:SI (match_dup 4) 4)
9186                     (match_dup 5)
9187                     (match_dup 6)]
9188          VDWDUPQ_M))
9189   ]
9190   "TARGET_HAVE_MVE"
9191   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9192   [(set_attr "type" "mve_move")
9193    (set_attr "length""8")])
9194
9195 ;;
9196 ;; [viwdupq_n_u])
9197 ;;
9198 (define_expand "mve_viwdupq_n_u<mode>"
9199  [(match_operand:MVE_2 0 "s_register_operand")
9200   (match_operand:SI 1 "s_register_operand")
9201   (match_operand:DI 2 "s_register_operand")
9202   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9203  "TARGET_HAVE_MVE"
9204 {
9205   rtx ignore_wb = gen_reg_rtx (SImode);
9206   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9207                                               operands[1], operands[2],
9208                                               operands[3]));
9209   DONE;
9210 })
9211
9212 ;;
9213 ;; [viwdupq_wb_u])
9214 ;;
9215 (define_expand "mve_viwdupq_wb_u<mode>"
9216  [(match_operand:SI 0 "s_register_operand")
9217   (match_operand:SI 1 "s_register_operand")
9218   (match_operand:DI 2 "s_register_operand")
9219   (match_operand:SI 3 "mve_imm_selective_upto_8")
9220   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9221  "TARGET_HAVE_MVE"
9222 {
9223   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9224   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9225                                               operands[1], operands[2],
9226                                               operands[3]));
9227   DONE;
9228 })
9229
9230 ;;
9231 ;; [viwdupq_wb_u_insn])
9232 ;;
9233 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9234   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9235         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9236                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9237                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9238          VIWDUPQ))
9239    (set (match_operand:SI 1 "s_register_operand" "=Te")
9240         (unspec:SI [(match_dup 2)
9241                     (subreg:SI (match_dup 3) 4)
9242                     (match_dup 4)]
9243          VIWDUPQ))]
9244   "TARGET_HAVE_MVE"
9245   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9246 )
9247
9248 ;;
9249 ;; [viwdupq_m_n_u])
9250 ;;
9251 (define_expand "mve_viwdupq_m_n_u<mode>"
9252  [(match_operand:MVE_2 0 "s_register_operand")
9253   (match_operand:MVE_2 1 "s_register_operand")
9254   (match_operand:SI 2 "s_register_operand")
9255   (match_operand:DI 3 "s_register_operand")
9256   (match_operand:SI 4 "mve_imm_selective_upto_8")
9257   (match_operand:HI 5 "vpr_register_operand")]
9258  "TARGET_HAVE_MVE"
9259 {
9260   rtx ignore_wb = gen_reg_rtx (SImode);
9261   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9262                                                 operands[1], operands[2],
9263                                                 operands[3], operands[4],
9264                                                 operands[5]));
9265   DONE;
9266 })
9267
9268 ;;
9269 ;; [viwdupq_m_wb_u])
9270 ;;
9271 (define_expand "mve_viwdupq_m_wb_u<mode>"
9272  [(match_operand:SI 0 "s_register_operand")
9273   (match_operand:MVE_2 1 "s_register_operand")
9274   (match_operand:SI 2 "s_register_operand")
9275   (match_operand:DI 3 "s_register_operand")
9276   (match_operand:SI 4 "mve_imm_selective_upto_8")
9277   (match_operand:HI 5 "vpr_register_operand")]
9278  "TARGET_HAVE_MVE"
9279 {
9280   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9281   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9282                                                 operands[1], operands[2],
9283                                                 operands[3], operands[4],
9284                                                 operands[5]));
9285   DONE;
9286 })
9287
9288 ;;
9289 ;; [viwdupq_m_wb_u_insn])
9290 ;;
9291 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9292   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9293         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9294                        (match_operand:SI 3 "s_register_operand" "1")
9295                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9296                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9297                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9298          VIWDUPQ_M))
9299    (set (match_operand:SI 1 "s_register_operand" "=Te")
9300         (unspec:SI [(match_dup 2)
9301                     (match_dup 3)
9302                     (subreg:SI (match_dup 4) 4)
9303                     (match_dup 5)
9304                     (match_dup 6)]
9305          VIWDUPQ_M))
9306   ]
9307   "TARGET_HAVE_MVE"
9308   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9309   [(set_attr "type" "mve_move")
9310    (set_attr "length""8")])
9311
9312 ;;
9313 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9314 ;;
9315 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9316   [(set (mem:BLK (scratch))
9317         (unspec:BLK
9318                 [(match_operand:V4SI 1 "s_register_operand" "0")
9319                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9320                  (match_operand:V4SI 3 "s_register_operand" "w")]
9321          VSTRWSBWBQ))
9322    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9323         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9324          VSTRWSBWBQ))
9325   ]
9326   "TARGET_HAVE_MVE"
9327 {
9328    rtx ops[3];
9329    ops[0] = operands[1];
9330    ops[1] = operands[2];
9331    ops[2] = operands[3];
9332    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9333    return "";
9334 }
9335   [(set_attr "length" "4")])
9336
9337 ;;
9338 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9339 ;;
9340 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9341  [(set (mem:BLK (scratch))
9342        (unspec:BLK
9343                 [(match_operand:V4SI 1 "s_register_operand" "0")
9344                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9345                  (match_operand:V4SI 3 "s_register_operand" "w")
9346                  (match_operand:HI 4 "vpr_register_operand")]
9347         VSTRWSBWBQ))
9348    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9349         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9350          VSTRWSBWBQ))
9351   ]
9352   "TARGET_HAVE_MVE"
9353 {
9354    rtx ops[3];
9355    ops[0] = operands[1];
9356    ops[1] = operands[2];
9357    ops[2] = operands[3];
9358    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9359    return "";
9360 }
9361   [(set_attr "length" "8")])
9362
9363 ;;
9364 ;; [vstrwq_scatter_base_wb_f]
9365 ;;
9366 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9367  [(set (mem:BLK (scratch))
9368        (unspec:BLK
9369                 [(match_operand:V4SI 1 "s_register_operand" "0")
9370                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9371                  (match_operand:V4SF 3 "s_register_operand" "w")]
9372          VSTRWQSBWB_F))
9373    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9374         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9375          VSTRWQSBWB_F))
9376   ]
9377   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9378 {
9379    rtx ops[3];
9380    ops[0] = operands[1];
9381    ops[1] = operands[2];
9382    ops[2] = operands[3];
9383    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9384    return "";
9385 }
9386   [(set_attr "length" "4")])
9387
9388 ;;
9389 ;; [vstrwq_scatter_base_wb_p_f]
9390 ;;
9391 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9392  [(set (mem:BLK (scratch))
9393        (unspec:BLK
9394                 [(match_operand:V4SI 1 "s_register_operand" "0")
9395                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9396                  (match_operand:V4SF 3 "s_register_operand" "w")
9397                  (match_operand:HI 4 "vpr_register_operand")]
9398         VSTRWQSBWB_F))
9399    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9400         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9401          VSTRWQSBWB_F))
9402   ]
9403   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9404 {
9405    rtx ops[3];
9406    ops[0] = operands[1];
9407    ops[1] = operands[2];
9408    ops[2] = operands[3];
9409    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9410    return "";
9411 }
9412   [(set_attr "length" "8")])
9413
9414 ;;
9415 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9416 ;;
9417 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9418   [(set (mem:BLK (scratch))
9419         (unspec:BLK
9420                 [(match_operand:V2DI 1 "s_register_operand" "0")
9421                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9422                  (match_operand:V2DI 3 "s_register_operand" "w")]
9423          VSTRDSBWBQ))
9424    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9425         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9426          VSTRDSBWBQ))
9427   ]
9428   "TARGET_HAVE_MVE"
9429 {
9430    rtx ops[3];
9431    ops[0] = operands[1];
9432    ops[1] = operands[2];
9433    ops[2] = operands[3];
9434    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9435    return "";
9436 }
9437   [(set_attr "length" "4")])
9438
9439 ;;
9440 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9441 ;;
9442 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9443   [(set (mem:BLK (scratch))
9444         (unspec:BLK
9445                 [(match_operand:V2DI 1 "s_register_operand" "0")
9446                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9447                  (match_operand:V2DI 3 "s_register_operand" "w")
9448                  (match_operand:HI 4 "vpr_register_operand")]
9449          VSTRDSBWBQ))
9450    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9451         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9452          VSTRDSBWBQ))
9453   ]
9454   "TARGET_HAVE_MVE"
9455 {
9456    rtx ops[3];
9457    ops[0] = operands[1];
9458    ops[1] = operands[2];
9459    ops[2] = operands[3];
9460    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9461    return "";
9462 }
9463   [(set_attr "length" "8")])
9464
9465 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9466   [(match_operand:V4SI 0 "s_register_operand")
9467    (match_operand:V4SI 1 "s_register_operand")
9468    (match_operand:SI 2 "mve_vldrd_immediate")
9469    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9470   "TARGET_HAVE_MVE"
9471 {
9472   rtx ignore_result = gen_reg_rtx (V4SImode);
9473   emit_insn (
9474   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9475                                                  operands[1], operands[2]));
9476   DONE;
9477 })
9478
9479 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9480   [(match_operand:V4SI 0 "s_register_operand")
9481    (match_operand:V4SI 1 "s_register_operand")
9482    (match_operand:SI 2 "mve_vldrd_immediate")
9483    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9484   "TARGET_HAVE_MVE"
9485 {
9486   rtx ignore_wb = gen_reg_rtx (V4SImode);
9487   emit_insn (
9488   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9489                                                  operands[1], operands[2]));
9490   DONE;
9491 })
9492
9493 ;;
9494 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9495 ;;
9496 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9497   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9498         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9499                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9500                       (mem:BLK (scratch))]
9501          VLDRWGBWBQ))
9502    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9503         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9504          VLDRWGBWBQ))
9505   ]
9506   "TARGET_HAVE_MVE"
9507 {
9508    rtx ops[3];
9509    ops[0] = operands[0];
9510    ops[1] = operands[2];
9511    ops[2] = operands[3];
9512    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9513    return "";
9514 }
9515   [(set_attr "length" "4")])
9516
9517 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9518   [(match_operand:V4SI 0 "s_register_operand")
9519    (match_operand:V4SI 1 "s_register_operand")
9520    (match_operand:SI 2 "mve_vldrd_immediate")
9521    (match_operand:HI 3 "vpr_register_operand")
9522    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9523   "TARGET_HAVE_MVE"
9524 {
9525   rtx ignore_result = gen_reg_rtx (V4SImode);
9526   emit_insn (
9527   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9528                                                    operands[1], operands[2],
9529                                                    operands[3]));
9530   DONE;
9531 })
9532 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9533   [(match_operand:V4SI 0 "s_register_operand")
9534    (match_operand:V4SI 1 "s_register_operand")
9535    (match_operand:SI 2 "mve_vldrd_immediate")
9536    (match_operand:HI 3 "vpr_register_operand")
9537    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9538   "TARGET_HAVE_MVE"
9539 {
9540   rtx ignore_wb = gen_reg_rtx (V4SImode);
9541   emit_insn (
9542   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9543                                                    operands[1], operands[2],
9544                                                    operands[3]));
9545   DONE;
9546 })
9547
9548 ;;
9549 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9550 ;;
9551 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9552   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9553         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9554                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9555                       (match_operand:HI 4 "vpr_register_operand" "Up")
9556                       (mem:BLK (scratch))]
9557          VLDRWGBWBQ))
9558    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9559         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9560          VLDRWGBWBQ))
9561   ]
9562   "TARGET_HAVE_MVE"
9563 {
9564    rtx ops[3];
9565    ops[0] = operands[0];
9566    ops[1] = operands[2];
9567    ops[2] = operands[3];
9568    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9569    return "";
9570 }
9571   [(set_attr "length" "8")])
9572
9573 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9574   [(match_operand:V4SI 0 "s_register_operand")
9575    (match_operand:V4SI 1 "s_register_operand")
9576    (match_operand:SI 2 "mve_vldrd_immediate")
9577    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9579 {
9580   rtx ignore_result = gen_reg_rtx (V4SFmode);
9581   emit_insn (
9582   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9583                                             operands[1], operands[2]));
9584   DONE;
9585 })
9586
9587 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9588   [(match_operand:V4SF 0 "s_register_operand")
9589    (match_operand:V4SI 1 "s_register_operand")
9590    (match_operand:SI 2 "mve_vldrd_immediate")
9591    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9592   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9593 {
9594   rtx ignore_wb = gen_reg_rtx (V4SImode);
9595   emit_insn (
9596   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9597                                             operands[1], operands[2]));
9598   DONE;
9599 })
9600
9601 ;;
9602 ;; [vldrwq_gather_base_wb_f]
9603 ;;
9604 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9605   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9606         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9607                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9608                       (mem:BLK (scratch))]
9609          VLDRWQGBWB_F))
9610    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9611         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9612          VLDRWQGBWB_F))
9613   ]
9614   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9615 {
9616    rtx ops[3];
9617    ops[0] = operands[0];
9618    ops[1] = operands[2];
9619    ops[2] = operands[3];
9620    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9621    return "";
9622 }
9623   [(set_attr "length" "4")])
9624
9625 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
9626   [(match_operand:V4SI 0 "s_register_operand")
9627    (match_operand:V4SI 1 "s_register_operand")
9628    (match_operand:SI 2 "mve_vldrd_immediate")
9629    (match_operand:HI 3 "vpr_register_operand")
9630    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9631   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9632 {
9633   rtx ignore_result = gen_reg_rtx (V4SFmode);
9634   emit_insn (
9635   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
9636                                               operands[1], operands[2],
9637                                               operands[3]));
9638   DONE;
9639 })
9640
9641 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
9642   [(match_operand:V4SF 0 "s_register_operand")
9643    (match_operand:V4SI 1 "s_register_operand")
9644    (match_operand:SI 2 "mve_vldrd_immediate")
9645    (match_operand:HI 3 "vpr_register_operand")
9646    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9647   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9648 {
9649   rtx ignore_wb = gen_reg_rtx (V4SImode);
9650   emit_insn (
9651   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
9652                                               operands[1], operands[2],
9653                                               operands[3]));
9654   DONE;
9655 })
9656
9657 ;;
9658 ;; [vldrwq_gather_base_wb_z_f]
9659 ;;
9660 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
9661   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9662         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9663                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9664                       (match_operand:HI 4 "vpr_register_operand" "Up")
9665                       (mem:BLK (scratch))]
9666          VLDRWQGBWB_F))
9667    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9668         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9669          VLDRWQGBWB_F))
9670   ]
9671   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9672 {
9673    rtx ops[3];
9674    ops[0] = operands[0];
9675    ops[1] = operands[2];
9676    ops[2] = operands[3];
9677    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9678    return "";
9679 }
9680   [(set_attr "length" "8")])
9681
9682 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
9683   [(match_operand:V2DI 0 "s_register_operand")
9684    (match_operand:V2DI 1 "s_register_operand")
9685    (match_operand:SI 2 "mve_vldrd_immediate")
9686    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9687   "TARGET_HAVE_MVE"
9688 {
9689   rtx ignore_result = gen_reg_rtx (V2DImode);
9690   emit_insn (
9691   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
9692                                                  operands[1], operands[2]));
9693   DONE;
9694 })
9695
9696 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
9697   [(match_operand:V2DI 0 "s_register_operand")
9698    (match_operand:V2DI 1 "s_register_operand")
9699    (match_operand:SI 2 "mve_vldrd_immediate")
9700    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9701   "TARGET_HAVE_MVE"
9702 {
9703   rtx ignore_wb = gen_reg_rtx (V2DImode);
9704   emit_insn (
9705   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
9706                                                  operands[1], operands[2]));
9707   DONE;
9708 })
9709
9710
9711 ;;
9712 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
9713 ;;
9714 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
9715   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9716         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9717                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9718                       (mem:BLK (scratch))]
9719          VLDRDGBWBQ))
9720    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9721         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9722          VLDRDGBWBQ))
9723   ]
9724   "TARGET_HAVE_MVE"
9725 {
9726    rtx ops[3];
9727    ops[0] = operands[0];
9728    ops[1] = operands[2];
9729    ops[2] = operands[3];
9730    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
9731    return "";
9732 }
9733   [(set_attr "length" "4")])
9734
9735 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
9736   [(match_operand:V2DI 0 "s_register_operand")
9737    (match_operand:V2DI 1 "s_register_operand")
9738    (match_operand:SI 2 "mve_vldrd_immediate")
9739    (match_operand:HI 3 "vpr_register_operand")
9740    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9741   "TARGET_HAVE_MVE"
9742 {
9743   rtx ignore_result = gen_reg_rtx (V2DImode);
9744   emit_insn (
9745   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
9746                                                    operands[1], operands[2],
9747                                                    operands[3]));
9748   DONE;
9749 })
9750
9751 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
9752   [(match_operand:V2DI 0 "s_register_operand")
9753    (match_operand:V2DI 1 "s_register_operand")
9754    (match_operand:SI 2 "mve_vldrd_immediate")
9755    (match_operand:HI 3 "vpr_register_operand")
9756    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9757   "TARGET_HAVE_MVE"
9758 {
9759   rtx ignore_wb = gen_reg_rtx (V2DImode);
9760   emit_insn (
9761   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
9762                                                    operands[1], operands[2],
9763                                                    operands[3]));
9764   DONE;
9765 })
9766
9767 (define_insn "get_fpscr_nzcvqc"
9768  [(set (match_operand:SI 0 "register_operand" "=r")
9769    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
9770  "TARGET_HAVE_MVE"
9771  "vmrs\\t%0, FPSCR_nzcvqc"
9772   [(set_attr "type" "mve_move")])
9773
9774 (define_insn "set_fpscr_nzcvqc"
9775  [(set (reg:SI VFPCC_REGNUM)
9776    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
9777     VUNSPEC_SET_FPSCR_NZCVQC))]
9778  "TARGET_HAVE_MVE"
9779  "vmsr\\tFPSCR_nzcvqc, %0"
9780   [(set_attr "type" "mve_move")])
9781
9782 ;;
9783 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
9784 ;;
9785 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
9786   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9787         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9788                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9789                       (match_operand:HI 4 "vpr_register_operand" "Up")
9790                       (mem:BLK (scratch))]
9791          VLDRDGBWBQ))
9792    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9793         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9794          VLDRDGBWBQ))
9795   ]
9796   "TARGET_HAVE_MVE"
9797 {
9798    rtx ops[3];
9799    ops[0] = operands[0];
9800    ops[1] = operands[2];
9801    ops[2] = operands[3];
9802    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
9803    return "";
9804 }
9805   [(set_attr "length" "8")])
9806 ;;
9807 ;; [vadciq_m_s, vadciq_m_u])
9808 ;;
9809 (define_insn "mve_vadciq_m_<supf>v4si"
9810   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9811         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9812                       (match_operand:V4SI 2 "s_register_operand" "w")
9813                       (match_operand:V4SI 3 "s_register_operand" "w")
9814                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9815          VADCIQ_M))
9816    (set (reg:SI VFPCC_REGNUM)
9817         (unspec:SI [(const_int 0)]
9818          VADCIQ_M))
9819   ]
9820   "TARGET_HAVE_MVE"
9821   "vpst\;vadcit.i32\t%q0, %q2, %q3"
9822   [(set_attr "type" "mve_move")
9823    (set_attr "length" "8")])
9824
9825 ;;
9826 ;; [vadciq_u, vadciq_s])
9827 ;;
9828 (define_insn "mve_vadciq_<supf>v4si"
9829   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9830         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9831                       (match_operand:V4SI 2 "s_register_operand" "w")]
9832          VADCIQ))
9833    (set (reg:SI VFPCC_REGNUM)
9834         (unspec:SI [(const_int 0)]
9835          VADCIQ))
9836   ]
9837   "TARGET_HAVE_MVE"
9838   "vadci.i32\t%q0, %q1, %q2"
9839   [(set_attr "type" "mve_move")
9840    (set_attr "length" "4")])
9841
9842 ;;
9843 ;; [vadcq_m_s, vadcq_m_u])
9844 ;;
9845 (define_insn "mve_vadcq_m_<supf>v4si"
9846   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9847         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9848                       (match_operand:V4SI 2 "s_register_operand" "w")
9849                       (match_operand:V4SI 3 "s_register_operand" "w")
9850                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9851          VADCQ_M))
9852    (set (reg:SI VFPCC_REGNUM)
9853         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9854          VADCQ_M))
9855   ]
9856   "TARGET_HAVE_MVE"
9857   "vpst\;vadct.i32\t%q0, %q2, %q3"
9858   [(set_attr "type" "mve_move")
9859    (set_attr "length" "8")])
9860
9861 ;;
9862 ;; [vadcq_u, vadcq_s])
9863 ;;
9864 (define_insn "mve_vadcq_<supf>v4si"
9865   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9866         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9867                        (match_operand:V4SI 2 "s_register_operand" "w")]
9868          VADCQ))
9869    (set (reg:SI VFPCC_REGNUM)
9870         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9871          VADCQ))
9872   ]
9873   "TARGET_HAVE_MVE"
9874   "vadc.i32\t%q0, %q1, %q2"
9875   [(set_attr "type" "mve_move")
9876    (set_attr "length" "4")
9877    (set_attr "conds" "set")])
9878
9879 ;;
9880 ;; [vsbciq_m_u, vsbciq_m_s])
9881 ;;
9882 (define_insn "mve_vsbciq_m_<supf>v4si"
9883   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9884         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9885                       (match_operand:V4SI 2 "s_register_operand" "w")
9886                       (match_operand:V4SI 3 "s_register_operand" "w")
9887                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9888          VSBCIQ_M))
9889    (set (reg:SI VFPCC_REGNUM)
9890         (unspec:SI [(const_int 0)]
9891          VSBCIQ_M))
9892   ]
9893   "TARGET_HAVE_MVE"
9894   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
9895   [(set_attr "type" "mve_move")
9896    (set_attr "length" "8")])
9897
9898 ;;
9899 ;; [vsbciq_s, vsbciq_u])
9900 ;;
9901 (define_insn "mve_vsbciq_<supf>v4si"
9902   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9903         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9904                       (match_operand:V4SI 2 "s_register_operand" "w")]
9905          VSBCIQ))
9906    (set (reg:SI VFPCC_REGNUM)
9907         (unspec:SI [(const_int 0)]
9908          VSBCIQ))
9909   ]
9910   "TARGET_HAVE_MVE"
9911   "vsbci.i32\t%q0, %q1, %q2"
9912   [(set_attr "type" "mve_move")
9913    (set_attr "length" "4")])
9914
9915 ;;
9916 ;; [vsbcq_m_u, vsbcq_m_s])
9917 ;;
9918 (define_insn "mve_vsbcq_m_<supf>v4si"
9919   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9920         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9921                       (match_operand:V4SI 2 "s_register_operand" "w")
9922                       (match_operand:V4SI 3 "s_register_operand" "w")
9923                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9924          VSBCQ_M))
9925    (set (reg:SI VFPCC_REGNUM)
9926         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9927          VSBCQ_M))
9928   ]
9929   "TARGET_HAVE_MVE"
9930   "vpst\;vsbct.i32\t%q0, %q2, %q3"
9931   [(set_attr "type" "mve_move")
9932    (set_attr "length" "8")])
9933
9934 ;;
9935 ;; [vsbcq_s, vsbcq_u])
9936 ;;
9937 (define_insn "mve_vsbcq_<supf>v4si"
9938   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9939         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9940                       (match_operand:V4SI 2 "s_register_operand" "w")]
9941          VSBCQ))
9942    (set (reg:SI VFPCC_REGNUM)
9943         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9944          VSBCQ))
9945   ]
9946   "TARGET_HAVE_MVE"
9947   "vsbc.i32\t%q0, %q1, %q2"
9948   [(set_attr "type" "mve_move")
9949    (set_attr "length" "4")])
9950
9951 ;;
9952 ;; [vst2q])
9953 ;;
9954 (define_insn "mve_vst2q<mode>"
9955   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
9956         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
9957                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9958          VST2Q))
9959   ]
9960   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9961    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9962 {
9963    rtx ops[4];
9964    int regno = REGNO (operands[1]);
9965    ops[0] = gen_rtx_REG (TImode, regno);
9966    ops[1] = gen_rtx_REG (TImode, regno + 4);
9967    rtx reg  = operands[0];
9968    while (reg && !REG_P (reg))
9969     reg = XEXP (reg, 0);
9970    gcc_assert (REG_P (reg));
9971    ops[2] = reg;
9972    ops[3] = operands[0];
9973    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
9974                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
9975    return "";
9976 }
9977   [(set_attr "length" "8")])
9978
9979 ;;
9980 ;; [vld2q])
9981 ;;
9982 (define_insn "mve_vld2q<mode>"
9983   [(set (match_operand:OI 0 "s_register_operand" "=w")
9984         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
9985                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9986          VLD2Q))
9987   ]
9988   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9989    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9990 {
9991    rtx ops[4];
9992    int regno = REGNO (operands[0]);
9993    ops[0] = gen_rtx_REG (TImode, regno);
9994    ops[1] = gen_rtx_REG (TImode, regno + 4);
9995    rtx reg  = operands[1];
9996    while (reg && !REG_P (reg))
9997     reg = XEXP (reg, 0);
9998    gcc_assert (REG_P (reg));
9999    ops[2] = reg;
10000    ops[3] = operands[1];
10001    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10002                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10003    return "";
10004 }
10005   [(set_attr "length" "8")])
10006
10007 ;;
10008 ;; [vld4q])
10009 ;;
10010 (define_insn "mve_vld4q<mode>"
10011   [(set (match_operand:XI 0 "s_register_operand" "=w")
10012         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10013                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10014          VLD4Q))
10015   ]
10016   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10017    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10018 {
10019    rtx ops[6];
10020    int regno = REGNO (operands[0]);
10021    ops[0] = gen_rtx_REG (TImode, regno);
10022    ops[1] = gen_rtx_REG (TImode, regno+4);
10023    ops[2] = gen_rtx_REG (TImode, regno+8);
10024    ops[3] = gen_rtx_REG (TImode, regno + 12);
10025    rtx reg  = operands[1];
10026    while (reg && !REG_P (reg))
10027     reg = XEXP (reg, 0);
10028    gcc_assert (REG_P (reg));
10029    ops[4] = reg;
10030    ops[5] = operands[1];
10031    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10032                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10033                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10034                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10035    return "";
10036 }
10037   [(set_attr "length" "16")])
10038 ;;
10039 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10040 ;;
10041 (define_insn "mve_vec_extract<mode><V_elem_l>"
10042  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10043    (vec_select:<V_elem>
10044     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10045     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10046   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10047    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10048 {
10049   if (BYTES_BIG_ENDIAN)
10050     {
10051       int elt = INTVAL (operands[2]);
10052       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10053       operands[2] = GEN_INT (elt);
10054     }
10055   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10056 }
10057  [(set_attr "type" "mve_move")])
10058
10059 (define_insn "mve_vec_extractv2didi"
10060  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10061    (vec_select:DI
10062     (match_operand:V2DI 1 "s_register_operand" "w")
10063     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10064   "TARGET_HAVE_MVE"
10065 {
10066   int elt = INTVAL (operands[2]);
10067   if (BYTES_BIG_ENDIAN)
10068     elt = 1 - elt;
10069
10070   if (elt == 0)
10071    return "vmov\t%Q0, %R0, %e1";
10072   else
10073    return "vmov\t%Q0, %R0, %f1";
10074 }
10075  [(set_attr "type" "mve_move")])
10076
10077 (define_insn "*mve_vec_extract_sext_internal<mode>"
10078  [(set (match_operand:SI 0 "s_register_operand" "=r")
10079    (sign_extend:SI
10080     (vec_select:<V_elem>
10081      (match_operand:MVE_2 1 "s_register_operand" "w")
10082      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10083   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10084    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10085 {
10086   if (BYTES_BIG_ENDIAN)
10087     {
10088       int elt = INTVAL (operands[2]);
10089       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10090       operands[2] = GEN_INT (elt);
10091     }
10092   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10093 }
10094  [(set_attr "type" "mve_move")])
10095
10096 (define_insn "*mve_vec_extract_zext_internal<mode>"
10097  [(set (match_operand:SI 0 "s_register_operand" "=r")
10098    (zero_extend:SI
10099     (vec_select:<V_elem>
10100      (match_operand:MVE_2 1 "s_register_operand" "w")
10101      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10102   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10103    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10104 {
10105   if (BYTES_BIG_ENDIAN)
10106     {
10107       int elt = INTVAL (operands[2]);
10108       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10109       operands[2] = GEN_INT (elt);
10110     }
10111   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10112 }
10113  [(set_attr "type" "mve_move")])
10114
10115 ;;
10116 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10117 ;;
10118 (define_insn "mve_vec_set<mode>_internal"
10119  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10120        (vec_merge:VQ2
10121         (vec_duplicate:VQ2
10122           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10123         (match_operand:VQ2 3 "s_register_operand" "0")
10124         (match_operand:SI 2 "immediate_operand" "i")))]
10125   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10126    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10127 {
10128   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10129   if (BYTES_BIG_ENDIAN)
10130     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10131   operands[2] = GEN_INT (elt);
10132
10133   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10134 }
10135  [(set_attr "type" "mve_move")])
10136
10137 (define_insn "mve_vec_setv2di_internal"
10138  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10139        (vec_merge:V2DI
10140         (vec_duplicate:V2DI
10141           (match_operand:DI 1 "nonimmediate_operand" "r"))
10142         (match_operand:V2DI 3 "s_register_operand" "0")
10143         (match_operand:SI 2 "immediate_operand" "i")))]
10144  "TARGET_HAVE_MVE"
10145 {
10146   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10147   if (BYTES_BIG_ENDIAN)
10148     elt = 1 - elt;
10149
10150   if (elt == 0)
10151    return "vmov\t%e0, %Q1, %R1";
10152   else
10153    return "vmov\t%f0, %J1, %K1";
10154 }
10155  [(set_attr "type" "mve_move")])
10156
10157 ;;
10158 ;; [uqrshll_di]
10159 ;;
10160 (define_insn "mve_uqrshll_sat<supf>_di"
10161   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10162         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10163                     (match_operand:SI 2 "register_operand" "r")]
10164          UQRSHLLQ))]
10165   "TARGET_HAVE_MVE"
10166   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10167   [(set_attr "predicable" "yes")])
10168
10169 ;;
10170 ;; [sqrshrl_di]
10171 ;;
10172 (define_insn "mve_sqrshrl_sat<supf>_di"
10173   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10174         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10175                     (match_operand:SI 2 "register_operand" "r")]
10176          SQRSHRLQ))]
10177   "TARGET_HAVE_MVE"
10178   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10179   [(set_attr "predicable" "yes")])
10180
10181 ;;
10182 ;; [uqrshl_si]
10183 ;;
10184 (define_insn "mve_uqrshl_si"
10185   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10186         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10187                     (match_operand:SI 2 "register_operand" "r")]
10188          UQRSHL))]
10189   "TARGET_HAVE_MVE"
10190   "uqrshl%?\\t%1, %2"
10191   [(set_attr "predicable" "yes")])
10192
10193 ;;
10194 ;; [sqrshr_si]
10195 ;;
10196 (define_insn "mve_sqrshr_si"
10197   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10198         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10199                     (match_operand:SI 2 "register_operand" "r")]
10200          SQRSHR))]
10201   "TARGET_HAVE_MVE"
10202   "sqrshr%?\\t%1, %2"
10203   [(set_attr "predicable" "yes")])
10204
10205 ;;
10206 ;; [uqshll_di]
10207 ;;
10208 (define_insn "mve_uqshll_di"
10209   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10210         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10211                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10212   "TARGET_HAVE_MVE"
10213   "uqshll%?\\t%Q1, %R1, %2"
10214   [(set_attr "predicable" "yes")])
10215
10216 ;;
10217 ;; [urshrl_di]
10218 ;;
10219 (define_insn "mve_urshrl_di"
10220   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10221         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10222                     (match_operand:SI 2 "immediate_operand" "Pg")]
10223          URSHRL))]
10224   "TARGET_HAVE_MVE"
10225   "urshrl%?\\t%Q1, %R1, %2"
10226   [(set_attr "predicable" "yes")])
10227
10228 ;;
10229 ;; [uqshl_si]
10230 ;;
10231 (define_insn "mve_uqshl_si"
10232   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10233         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10234                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10235   "TARGET_HAVE_MVE"
10236   "uqshl%?\\t%1, %2"
10237   [(set_attr "predicable" "yes")])
10238
10239 ;;
10240 ;; [urshr_si]
10241 ;;
10242 (define_insn "mve_urshr_si"
10243   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10244         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10245                     (match_operand:SI 2 "immediate_operand" "Pg")]
10246          URSHR))]
10247   "TARGET_HAVE_MVE"
10248   "urshr%?\\t%1, %2"
10249   [(set_attr "predicable" "yes")])
10250
10251 ;;
10252 ;; [sqshl_si]
10253 ;;
10254 (define_insn "mve_sqshl_si"
10255   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10256         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10257                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10258   "TARGET_HAVE_MVE"
10259   "sqshl%?\\t%1, %2"
10260   [(set_attr "predicable" "yes")])
10261
10262 ;;
10263 ;; [srshr_si]
10264 ;;
10265 (define_insn "mve_srshr_si"
10266   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10267         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10268                     (match_operand:SI 2 "immediate_operand" "Pg")]
10269          SRSHR))]
10270   "TARGET_HAVE_MVE"
10271   "srshr%?\\t%1, %2"
10272   [(set_attr "predicable" "yes")])
10273
10274 ;;
10275 ;; [srshrl_di]
10276 ;;
10277 (define_insn "mve_srshrl_di"
10278   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10279         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10280                     (match_operand:SI 2 "immediate_operand" "Pg")]
10281          SRSHRL))]
10282   "TARGET_HAVE_MVE"
10283   "srshrl%?\\t%Q1, %R1, %2"
10284   [(set_attr "predicable" "yes")])
10285
10286 ;;
10287 ;; [sqshll_di]
10288 ;;
10289 (define_insn "mve_sqshll_di"
10290   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10291         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10292                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10293   "TARGET_HAVE_MVE"
10294   "sqshll%?\\t%Q1, %R1, %2"
10295   [(set_attr "predicable" "yes")])
10296
10297 ;;
10298 ;; [vshlcq_m_u vshlcq_m_s]
10299 ;;
10300 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10301  [(match_operand:MVE_2 0 "s_register_operand")
10302   (match_operand:MVE_2 1 "s_register_operand")
10303   (match_operand:SI 2 "s_register_operand")
10304   (match_operand:SI 3 "mve_imm_32")
10305   (match_operand:HI 4 "vpr_register_operand")
10306   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10307  "TARGET_HAVE_MVE"
10308 {
10309   rtx ignore_wb = gen_reg_rtx (SImode);
10310   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10311                                             operands[2], operands[3],
10312                                             operands[4]));
10313   DONE;
10314 })
10315
10316 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10317  [(match_operand:SI 0 "s_register_operand")
10318   (match_operand:MVE_2 1 "s_register_operand")
10319   (match_operand:SI 2 "s_register_operand")
10320   (match_operand:SI 3 "mve_imm_32")
10321   (match_operand:HI 4 "vpr_register_operand")
10322   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10323  "TARGET_HAVE_MVE"
10324 {
10325   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10326   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10327                                             operands[1], operands[2],
10328                                             operands[3], operands[4]));
10329   DONE;
10330 })
10331
10332 (define_insn "mve_vshlcq_m_<supf><mode>"
10333  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10334        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10335                       (match_operand:SI 3 "s_register_operand" "1")
10336                       (match_operand:SI 4 "mve_imm_32" "Rf")
10337                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10338         VSHLCQ_M))
10339   (set (match_operand:SI  1 "s_register_operand" "=r")
10340        (unspec:SI [(match_dup 2)
10341                    (match_dup 3)
10342                    (match_dup 4)
10343                    (match_dup 5)]
10344         VSHLCQ_M))
10345  ]
10346  "TARGET_HAVE_MVE"
10347  "vpst\;vshlct\t%q0, %1, %4"
10348  [(set_attr "type" "mve_move")
10349   (set_attr "length" "8")])
10350
10351 ;; CDE instructions on MVE registers.
10352
10353 (define_insn "arm_vcx1qv16qi"
10354   [(set (match_operand:V16QI 0 "register_operand" "=t")
10355         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10356                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10357          UNSPEC_VCDE))]
10358   "TARGET_CDE && TARGET_HAVE_MVE"
10359   "vcx1\\tp%c1, %q0, #%c2"
10360   [(set_attr "type" "coproc")]
10361 )
10362
10363 (define_insn "arm_vcx1qav16qi"
10364   [(set (match_operand:V16QI 0 "register_operand" "=t")
10365         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10366                             (match_operand:V16QI 2 "register_operand" "0")
10367                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10368          UNSPEC_VCDEA))]
10369   "TARGET_CDE && TARGET_HAVE_MVE"
10370   "vcx1a\\tp%c1, %q0, #%c3"
10371   [(set_attr "type" "coproc")]
10372 )
10373
10374 (define_insn "arm_vcx2qv16qi"
10375   [(set (match_operand:V16QI 0 "register_operand" "=t")
10376         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10377                           (match_operand:V16QI 2 "register_operand" "t")
10378                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10379          UNSPEC_VCDE))]
10380   "TARGET_CDE && TARGET_HAVE_MVE"
10381   "vcx2\\tp%c1, %q0, %q2, #%c3"
10382   [(set_attr "type" "coproc")]
10383 )
10384
10385 (define_insn "arm_vcx2qav16qi"
10386   [(set (match_operand:V16QI 0 "register_operand" "=t")
10387         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10388                           (match_operand:V16QI 2 "register_operand" "0")
10389                           (match_operand:V16QI 3 "register_operand" "t")
10390                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10391          UNSPEC_VCDEA))]
10392   "TARGET_CDE && TARGET_HAVE_MVE"
10393   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10394   [(set_attr "type" "coproc")]
10395 )
10396
10397 (define_insn "arm_vcx3qv16qi"
10398   [(set (match_operand:V16QI 0 "register_operand" "=t")
10399         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10400                           (match_operand:V16QI 2 "register_operand" "t")
10401                           (match_operand:V16QI 3 "register_operand" "t")
10402                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10403          UNSPEC_VCDE))]
10404   "TARGET_CDE && TARGET_HAVE_MVE"
10405   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10406   [(set_attr "type" "coproc")]
10407 )
10408
10409 (define_insn "arm_vcx3qav16qi"
10410   [(set (match_operand:V16QI 0 "register_operand" "=t")
10411         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10412                           (match_operand:V16QI 2 "register_operand" "0")
10413                           (match_operand:V16QI 3 "register_operand" "t")
10414                           (match_operand:V16QI 4 "register_operand" "t")
10415                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10416          UNSPEC_VCDEA))]
10417   "TARGET_CDE && TARGET_HAVE_MVE"
10418   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10419   [(set_attr "type" "coproc")]
10420 )
10421
10422 (define_insn "arm_vcx1q<a>_p_v16qi"
10423   [(set (match_operand:V16QI 0 "register_operand" "=t")
10424         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10425                            (match_operand:V16QI 2 "register_operand" "0")
10426                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10427                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10428          CDE_VCX))]
10429   "TARGET_CDE && TARGET_HAVE_MVE"
10430   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10431   [(set_attr "type" "coproc")
10432    (set_attr "length" "8")]
10433 )
10434
10435 (define_insn "arm_vcx2q<a>_p_v16qi"
10436   [(set (match_operand:V16QI 0 "register_operand" "=t")
10437         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10438                           (match_operand:V16QI 2 "register_operand" "0")
10439                           (match_operand:V16QI 3 "register_operand" "t")
10440                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10441                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10442          CDE_VCX))]
10443   "TARGET_CDE && TARGET_HAVE_MVE"
10444   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10445   [(set_attr "type" "coproc")
10446    (set_attr "length" "8")]
10447 )
10448
10449 (define_insn "arm_vcx3q<a>_p_v16qi"
10450   [(set (match_operand:V16QI 0 "register_operand" "=t")
10451         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10452                           (match_operand:V16QI 2 "register_operand" "0")
10453                           (match_operand:V16QI 3 "register_operand" "t")
10454                           (match_operand:V16QI 4 "register_operand" "t")
10455                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10456                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10457          CDE_VCX))]
10458   "TARGET_CDE && TARGET_HAVE_MVE"
10459   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10460   [(set_attr "type" "coproc")
10461    (set_attr "length" "8")]
10462 )
10463
10464 (define_insn "*movmisalign<mode>_mve_store"
10465   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Ux")
10466         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10467          UNSPEC_MISALIGNED_ACCESS))]
10468   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10469     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10470    && !BYTES_BIG_ENDIAN && unaligned_access"
10471   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10472   [(set_attr "type" "mve_store")]
10473 )
10474
10475
10476 (define_insn "*movmisalign<mode>_mve_load"
10477   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10478         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Ux")]
10479          UNSPEC_MISALIGNED_ACCESS))]
10480   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10481     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10482    && !BYTES_BIG_ENDIAN && unaligned_access"
10483   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10484   [(set_attr "type" "mve_load")]
10485 )