gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2022 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,UxUi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode || <MODE>mode == TImode)
  45         {
  46           if (which_alternative == 7)
  47             output_asm_insn ("vstrw.32\t%q1, %E0", operands);
  48           else
  49             output_asm_insn ("vldrw.u32\t%q0, %E1",operands);
  50         }
  51       else
  52         {
  53           if (which_alternative == 7)
  54             output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0", operands);
  55           else
  56             output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1", operands);
  57         }
  58       return "";
  59     }
  60   switch (which_alternative)
  61     {
  62     case 0:
  63       return "vmov\t%q0, %q1";
  64     case 1:
  65       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  66     case 2:
  67       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  68     case 5:
  69       return output_move_quad (operands);
  70     case 8:
  71         return output_move_neon (operands);
  72     default:
  73       gcc_unreachable ();
  74       return "";
  75     }
  76 }
  77   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
  78    (set_attr "length" "4,8,8,4,8,8,4,4,4")
  79    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
  80    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
  81
  82 (define_insn "*mve_vdup<mode>"
  83   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  84         (vec_duplicate:MVE_vecs
  85           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  86   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  87   "vdup.<V_sz_elem>\t%q0, %1"
  88   [(set_attr "length" "4")
  89    (set_attr "type" "mve_move")])
  90
  91 ;;
  92 ;; [vst4q])
  93 ;;
  94 (define_insn "mve_vst4q<mode>"
  95   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
  96         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
  97                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
  98          VST4Q))
  99   ]
 100   "TARGET_HAVE_MVE"
 101 {
 102    rtx ops[6];
 103    int regno = REGNO (operands[1]);
 104    ops[0] = gen_rtx_REG (TImode, regno);
 105    ops[1] = gen_rtx_REG (TImode, regno+4);
 106    ops[2] = gen_rtx_REG (TImode, regno+8);
 107    ops[3] = gen_rtx_REG (TImode, regno+12);
 108    rtx reg  = operands[0];
 109    while (reg && !REG_P (reg))
 110     reg = XEXP (reg, 0);
 111    gcc_assert (REG_P (reg));
 112    ops[4] = reg;
 113    ops[5] = operands[0];
 114    /* Here in first three instructions data is stored to ops[4]'s location but
 115       in the fourth instruction data is stored to operands[0], this is to
 116       support the writeback.  */
 117    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 118                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 119                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 120                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 121    return "";
 122 }
 123   [(set_attr "length" "16")])
 124
 125 ;;
 126 ;; [vrndq_m_f])
 127 ;;
 128 (define_insn "mve_vrndq_m_f<mode>"
 129   [
 130    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 131         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 132                        (match_operand:MVE_0 2 "s_register_operand" "w")
 133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 134          VRNDQ_M_F))
 135   ]
 136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 137   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 138   [(set_attr "type" "mve_move")
 139    (set_attr "length""8")])
 140
 141 ;;
 142 ;; [vrndxq_f])
 143 ;;
 144 (define_insn "mve_vrndxq_f<mode>"
 145   [
 146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 148          VRNDXQ_F))
 149   ]
 150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 151   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 152   [(set_attr "type" "mve_move")
 153 ])
 154
 155 ;;
 156 ;; [vrndq_f])
 157 ;;
 158 (define_insn "mve_vrndq_f<mode>"
 159   [
 160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 162          VRNDQ_F))
 163   ]
 164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 165   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 166   [(set_attr "type" "mve_move")
 167 ])
 168
 169 ;;
 170 ;; [vrndpq_f])
 171 ;;
 172 (define_insn "mve_vrndpq_f<mode>"
 173   [
 174    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 175         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 176          VRNDPQ_F))
 177   ]
 178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 179   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 180   [(set_attr "type" "mve_move")
 181 ])
 182
 183 ;;
 184 ;; [vrndnq_f])
 185 ;;
 186 (define_insn "mve_vrndnq_f<mode>"
 187   [
 188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 190          VRNDNQ_F))
 191   ]
 192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 193   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 194   [(set_attr "type" "mve_move")
 195 ])
 196
 197 ;;
 198 ;; [vrndmq_f])
 199 ;;
 200 (define_insn "mve_vrndmq_f<mode>"
 201   [
 202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 204          VRNDMQ_F))
 205   ]
 206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 207   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 208   [(set_attr "type" "mve_move")
 209 ])
 210
 211 ;;
 212 ;; [vrndaq_f])
 213 ;;
 214 (define_insn "mve_vrndaq_f<mode>"
 215   [
 216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 217         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 218          VRNDAQ_F))
 219   ]
 220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 221   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 222   [(set_attr "type" "mve_move")
 223 ])
 224
 225 ;;
 226 ;; [vrev64q_f])
 227 ;;
 228 (define_insn "mve_vrev64q_f<mode>"
 229   [
 230    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 232          VREV64Q_F))
 233   ]
 234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 235   "vrev64.%#<V_sz_elem> %q0, %q1"
 236   [(set_attr "type" "mve_move")
 237 ])
 238
 239 ;;
 240 ;; [vnegq_f])
 241 ;;
 242 (define_insn "mve_vnegq_f<mode>"
 243   [
 244    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 245         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 246   ]
 247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 248   "vneg.f%#<V_sz_elem>  %q0, %q1"
 249   [(set_attr "type" "mve_move")
 250 ])
 251
 252 ;;
 253 ;; [vdupq_n_f])
 254 ;;
 255 (define_insn "mve_vdupq_n_f<mode>"
 256   [
 257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 258         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 259          VDUPQ_N_F))
 260   ]
 261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 262   "vdup.%#<V_sz_elem>   %q0, %1"
 263   [(set_attr "type" "mve_move")
 264 ])
 265
 266 ;;
 267 ;; [vabsq_f])
 268 ;;
 269 (define_insn "mve_vabsq_f<mode>"
 270   [
 271    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 272         (abs:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 273   ]
 274   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 275   "vabs.f%#<V_sz_elem>  %q0, %q1"
 276   [(set_attr "type" "mve_move")
 277 ])
 278
 279 ;;
 280 ;; [vrev32q_f])
 281 ;;
 282 (define_insn "mve_vrev32q_fv8hf"
 283   [
 284    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 285         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 286          VREV32Q_F))
 287   ]
 288   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 289   "vrev32.16 %q0, %q1"
 290   [(set_attr "type" "mve_move")
 291 ])
 292 ;;
 293 ;; [vcvttq_f32_f16])
 294 ;;
 295 (define_insn "mve_vcvttq_f32_f16v4sf"
 296   [
 297    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 298         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 299          VCVTTQ_F32_F16))
 300   ]
 301   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 302   "vcvtt.f32.f16 %q0, %q1"
 303   [(set_attr "type" "mve_move")
 304 ])
 305
 306 ;;
 307 ;; [vcvtbq_f32_f16])
 308 ;;
 309 (define_insn "mve_vcvtbq_f32_f16v4sf"
 310   [
 311    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 312         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 313          VCVTBQ_F32_F16))
 314   ]
 315   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 316   "vcvtb.f32.f16 %q0, %q1"
 317   [(set_attr "type" "mve_move")
 318 ])
 319
 320 ;;
 321 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 322 ;;
 323 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 324   [
 325    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 326         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 327          VCVTQ_TO_F))
 328   ]
 329   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 330   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 331   [(set_attr "type" "mve_move")
 332 ])
 333
 334 ;;
 335 ;; [vrev64q_u, vrev64q_s])
 336 ;;
 337 (define_insn "mve_vrev64q_<supf><mode>"
 338   [
 339    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 340         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 341          VREV64Q))
 342   ]
 343   "TARGET_HAVE_MVE"
 344   "vrev64.%#<V_sz_elem> %q0, %q1"
 345   [(set_attr "type" "mve_move")
 346 ])
 347
 348 ;;
 349 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 350 ;;
 351 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 352   [
 353    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 354         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 355          VCVTQ_FROM_F))
 356   ]
 357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 358   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 359   [(set_attr "type" "mve_move")
 360 ])
 361 ;; [vqnegq_s])
 362 ;;
 363 (define_insn "mve_vqnegq_s<mode>"
 364   [
 365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 367          VQNEGQ_S))
 368   ]
 369   "TARGET_HAVE_MVE"
 370   "vqneg.s%#<V_sz_elem> %q0, %q1"
 371   [(set_attr "type" "mve_move")
 372 ])
 373
 374 ;;
 375 ;; [vqabsq_s])
 376 ;;
 377 (define_insn "mve_vqabsq_s<mode>"
 378   [
 379    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 380         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 381          VQABSQ_S))
 382   ]
 383   "TARGET_HAVE_MVE"
 384   "vqabs.s%#<V_sz_elem> %q0, %q1"
 385   [(set_attr "type" "mve_move")
 386 ])
 387
 388 ;;
 389 ;; [vnegq_s])
 390 ;;
 391 (define_insn "mve_vnegq_s<mode>"
 392   [
 393    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 394         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 395   ]
 396   "TARGET_HAVE_MVE"
 397   "vneg.s%#<V_sz_elem>  %q0, %q1"
 398   [(set_attr "type" "mve_move")
 399 ])
 400
 401 ;;
 402 ;; [vmvnq_u, vmvnq_s])
 403 ;;
 404 (define_insn "mve_vmvnq_u<mode>"
 405   [
 406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 407         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 408   ]
 409   "TARGET_HAVE_MVE"
 410   "vmvn\t%q0, %q1"
 411   [(set_attr "type" "mve_move")
 412 ])
 413 (define_expand "mve_vmvnq_s<mode>"
 414   [
 415    (set (match_operand:MVE_2 0 "s_register_operand")
 416         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 417   ]
 418   "TARGET_HAVE_MVE"
 419 )
 420
 421 ;;
 422 ;; [vdupq_n_u, vdupq_n_s])
 423 ;;
 424 (define_insn "mve_vdupq_n_<supf><mode>"
 425   [
 426    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 427         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 428          VDUPQ_N))
 429   ]
 430   "TARGET_HAVE_MVE"
 431   "vdup.%#<V_sz_elem>   %q0, %1"
 432   [(set_attr "type" "mve_move")
 433 ])
 434
 435 ;;
 436 ;; [vclzq_u, vclzq_s])
 437 ;;
 438 (define_insn "@mve_vclzq_s<mode>"
 439   [
 440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 441         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 442   ]
 443   "TARGET_HAVE_MVE"
 444   "vclz.i%#<V_sz_elem>  %q0, %q1"
 445   [(set_attr "type" "mve_move")
 446 ])
 447 (define_expand "mve_vclzq_u<mode>"
 448   [
 449    (set (match_operand:MVE_2 0 "s_register_operand")
 450         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 451   ]
 452   "TARGET_HAVE_MVE"
 453 )
 454
 455 ;;
 456 ;; [vclsq_s])
 457 ;;
 458 (define_insn "mve_vclsq_s<mode>"
 459   [
 460    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 461         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 462          VCLSQ_S))
 463   ]
 464   "TARGET_HAVE_MVE"
 465   "vcls.s%#<V_sz_elem>  %q0, %q1"
 466   [(set_attr "type" "mve_move")
 467 ])
 468
 469 ;;
 470 ;; [vaddvq_s, vaddvq_u])
 471 ;;
 472 (define_insn "@mve_vaddvq_<supf><mode>"
 473   [
 474    (set (match_operand:SI 0 "s_register_operand" "=Te")
 475         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 476          VADDVQ))
 477   ]
 478   "TARGET_HAVE_MVE"
 479   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 480   [(set_attr "type" "mve_move")
 481 ])
 482
 483 ;;
 484 ;; [vabsq_s])
 485 ;;
 486 (define_insn "mve_vabsq_s<mode>"
 487   [
 488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 489         (abs:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 490   ]
 491   "TARGET_HAVE_MVE"
 492   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 493   [(set_attr "type" "mve_move")
 494 ])
 495
 496 ;;
 497 ;; [vrev32q_u, vrev32q_s])
 498 ;;
 499 (define_insn "mve_vrev32q_<supf><mode>"
 500   [
 501    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 502         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 503          VREV32Q))
 504   ]
 505   "TARGET_HAVE_MVE"
 506   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 507   [(set_attr "type" "mve_move")
 508 ])
 509
 510 ;;
 511 ;; [vmovltq_u, vmovltq_s])
 512 ;;
 513 (define_insn "mve_vmovltq_<supf><mode>"
 514   [
 515    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 516         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 517          VMOVLTQ))
 518   ]
 519   "TARGET_HAVE_MVE"
 520   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 521   [(set_attr "type" "mve_move")
 522 ])
 523
 524 ;;
 525 ;; [vmovlbq_s, vmovlbq_u])
 526 ;;
 527 (define_insn "mve_vmovlbq_<supf><mode>"
 528   [
 529    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 530         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 531          VMOVLBQ))
 532   ]
 533   "TARGET_HAVE_MVE"
 534   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 535   [(set_attr "type" "mve_move")
 536 ])
 537
 538 (define_insn "mve_vec_unpack<US>_lo_<mode>"
 539   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
 540         (SE:<V_unpack> (vec_select:<V_HALF>
 541                           (match_operand:MVE_3 1 "register_operand" "w")
 542                           (match_operand:MVE_3 2 "vect_par_constant_low" ""))))]
 543   "TARGET_HAVE_MVE"
 544   "vmovlb.<US>%#<V_sz_elem> %q0, %q1"
 545   [(set_attr "type" "mve_move")]
 546 )
 547
 548 (define_insn "mve_vec_unpack<US>_hi_<mode>"
 549   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
 550         (SE:<V_unpack> (vec_select:<V_HALF>
 551                           (match_operand:MVE_3 1 "register_operand" "w")
 552                           (match_operand:MVE_3 2 "vect_par_constant_high" ""))))]
 553   "TARGET_HAVE_MVE"
 554   "vmovlt.<US>%#<V_sz_elem> %q0, %q1"
 555   [(set_attr "type" "mve_move")]
 556 )
 557
 558 ;;
 559 ;; [vcvtpq_s, vcvtpq_u])
 560 ;;
 561 (define_insn "mve_vcvtpq_<supf><mode>"
 562   [
 563    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 564         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 565          VCVTPQ))
 566   ]
 567   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 568   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 569   [(set_attr "type" "mve_move")
 570 ])
 571
 572 ;;
 573 ;; [vcvtnq_s, vcvtnq_u])
 574 ;;
 575 (define_insn "mve_vcvtnq_<supf><mode>"
 576   [
 577    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 578         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 579          VCVTNQ))
 580   ]
 581   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 582   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 583   [(set_attr "type" "mve_move")
 584 ])
 585
 586 ;;
 587 ;; [vcvtmq_s, vcvtmq_u])
 588 ;;
 589 (define_insn "mve_vcvtmq_<supf><mode>"
 590   [
 591    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 592         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 593          VCVTMQ))
 594   ]
 595   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 596   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 597   [(set_attr "type" "mve_move")
 598 ])
 599
 600 ;;
 601 ;; [vcvtaq_u, vcvtaq_s])
 602 ;;
 603 (define_insn "mve_vcvtaq_<supf><mode>"
 604   [
 605    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 606         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 607          VCVTAQ))
 608   ]
 609   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 610   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 611   [(set_attr "type" "mve_move")
 612 ])
 613
 614 ;;
 615 ;; [vmvnq_n_u, vmvnq_n_s])
 616 ;;
 617 (define_insn "mve_vmvnq_n_<supf><mode>"
 618   [
 619    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 620         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 621          VMVNQ_N))
 622   ]
 623   "TARGET_HAVE_MVE"
 624   "vmvn.i%#<V_sz_elem>  %q0, %1"
 625   [(set_attr "type" "mve_move")
 626 ])
 627
 628 ;;
 629 ;; [vrev16q_u, vrev16q_s])
 630 ;;
 631 (define_insn "mve_vrev16q_<supf>v16qi"
 632   [
 633    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 634         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 635          VREV16Q))
 636   ]
 637   "TARGET_HAVE_MVE"
 638   "vrev16.8 %q0, %q1"
 639   [(set_attr "type" "mve_move")
 640 ])
 641
 642 ;;
 643 ;; [vaddlvq_s vaddlvq_u])
 644 ;;
 645 (define_insn "mve_vaddlvq_<supf>v4si"
 646   [
 647    (set (match_operand:DI 0 "s_register_operand" "=r")
 648         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 649          VADDLVQ))
 650   ]
 651   "TARGET_HAVE_MVE"
 652   "vaddlv.<supf>32 %Q0, %R0, %q1"
 653   [(set_attr "type" "mve_move")
 654 ])
 655
 656 ;;
 657 ;; [vctp8q vctp16q vctp32q vctp64q])
 658 ;;
 659 (define_insn "mve_vctp<mode1>qhi"
 660   [
 661    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 662         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 663         VCTPQ))
 664   ]
 665   "TARGET_HAVE_MVE"
 666   "vctp.<mode1> %1"
 667   [(set_attr "type" "mve_move")
 668 ])
 669
 670 ;;
 671 ;; [vpnot])
 672 ;;
 673 (define_insn "mve_vpnothi"
 674   [
 675    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 676         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 677          VPNOT))
 678   ]
 679   "TARGET_HAVE_MVE"
 680   "vpnot"
 681   [(set_attr "type" "mve_move")
 682 ])
 683
 684 ;;
 685 ;; [vsubq_n_f])
 686 ;;
 687 (define_insn "mve_vsubq_n_f<mode>"
 688   [
 689    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 690         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 691                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 692          VSUBQ_N_F))
 693   ]
 694   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 695   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 696   [(set_attr "type" "mve_move")
 697 ])
 698
 699 ;;
 700 ;; [vbrsrq_n_f])
 701 ;;
 702 (define_insn "mve_vbrsrq_n_f<mode>"
 703   [
 704    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 705         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 706                        (match_operand:SI 2 "s_register_operand" "r")]
 707          VBRSRQ_N_F))
 708   ]
 709   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 710   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 711   [(set_attr "type" "mve_move")
 712 ])
 713
 714 ;;
 715 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 716 ;;
 717 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 718   [
 719    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 720         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 721                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 722          VCVTQ_N_TO_F))
 723   ]
 724   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 725   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 726   [(set_attr "type" "mve_move")
 727 ])
 728
 729 ;; [vcreateq_f])
 730 ;;
 731 (define_insn "mve_vcreateq_f<mode>"
 732   [
 733    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 734         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 735                        (match_operand:DI 2 "s_register_operand" "r")]
 736          VCREATEQ_F))
 737   ]
 738   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 739   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 740   [(set_attr "type" "mve_move")
 741    (set_attr "length""8")])
 742
 743 ;;
 744 ;; [vcreateq_u, vcreateq_s])
 745 ;;
 746 (define_insn "mve_vcreateq_<supf><mode>"
 747   [
 748    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 749         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 750                        (match_operand:DI 2 "s_register_operand" "r")]
 751          VCREATEQ))
 752   ]
 753   "TARGET_HAVE_MVE"
 754   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 755   [(set_attr "type" "mve_move")
 756    (set_attr "length""8")])
 757
 758 ;;
 759 ;; [vshrq_n_s, vshrq_n_u])
 760 ;;
 761 ;; Version that takes an immediate as operand 2.
 762 (define_insn "mve_vshrq_n_<supf><mode>"
 763   [
 764    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 765         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 766                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 767          VSHRQ_N))
 768   ]
 769   "TARGET_HAVE_MVE"
 770   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 771   [(set_attr "type" "mve_move")
 772 ])
 773
 774 ;; Versions that take constant vectors as operand 2 (with all elements
 775 ;; equal).
 776 (define_insn "mve_vshrq_n_s<mode>_imm"
 777   [
 778    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 779         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 780                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 781   ]
 782   "TARGET_HAVE_MVE"
 783   {
 784     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 785                                         <MODE>mode,
 786                                         VALID_NEON_QREG_MODE (<MODE>mode),
 787                                         true);
 788   }
 789   [(set_attr "type" "mve_move")
 790 ])
 791 (define_insn "mve_vshrq_n_u<mode>_imm"
 792   [
 793    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 794         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 795                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 796   ]
 797   "TARGET_HAVE_MVE"
 798   {
 799     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 800                                         <MODE>mode,
 801                                         VALID_NEON_QREG_MODE (<MODE>mode),
 802                                         true);
 803   }
 804   [(set_attr "type" "mve_move")
 805 ])
 806
 807 ;;
 808 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 809 ;;
 810 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 811   [
 812    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 813         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 814                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 815          VCVTQ_N_FROM_F))
 816   ]
 817   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 818   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 819   [(set_attr "type" "mve_move")
 820 ])
 821
 822 ;;
 823 ;; [vaddlvq_p_s])
 824 ;;
 825 (define_insn "mve_vaddlvq_p_<supf>v4si"
 826   [
 827    (set (match_operand:DI 0 "s_register_operand" "=r")
 828         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 829                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 830          VADDLVQ_P))
 831   ]
 832   "TARGET_HAVE_MVE"
 833   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 834   [(set_attr "type" "mve_move")
 835    (set_attr "length""8")])
 836
 837 ;;
 838 ;; [vcmpneq_, vcmpcsq_, vcmpeqq_, vcmpgeq_, vcmpgtq_, vcmphiq_, vcmpleq_, vcmpltq_])
 839 ;;
 840 (define_insn "@mve_vcmp<mve_cmp_op>q_<mode>"
 841   [
 842    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 843         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 844                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 845   ]
 846   "TARGET_HAVE_MVE"
 847   "vcmp.<mve_cmp_type>%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
 848   [(set_attr "type" "mve_move")
 849 ])
 850
 851 ;;
 852 ;; [vcmpcsq_n_, vcmpeqq_n_, vcmpgeq_n_, vcmpgtq_n_, vcmphiq_n_, vcmpleq_n_, vcmpltq_n_, vcmpneq_n_])
 853 ;;
 854 (define_insn "mve_vcmp<mve_cmp_op>q_n_<mode>"
 855   [
 856    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 857         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 858                     (match_operand:<V_elem> 2 "s_register_operand" "r")))
 859   ]
 860   "TARGET_HAVE_MVE"
 861   "vcmp.<mve_cmp_type>%#<V_sz_elem>     <mve_cmp_op>, %q1, %2"
 862   [(set_attr "type" "mve_move")
 863 ])
 864
 865 ;;
 866 ;; [vshlq_s, vshlq_u])
 867 ;; See vec-common.md
 868
 869 ;;
 870 ;; [vabdq_s, vabdq_u])
 871 ;;
 872 (define_insn "mve_vabdq_<supf><mode>"
 873   [
 874    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 875         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 876                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 877          VABDQ))
 878   ]
 879   "TARGET_HAVE_MVE"
 880   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 881   [(set_attr "type" "mve_move")
 882 ])
 883
 884 ;;
 885 ;; [vaddq_n_s, vaddq_n_u])
 886 ;;
 887 (define_insn "mve_vaddq_n_<supf><mode>"
 888   [
 889    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 890         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 891                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 892          VADDQ_N))
 893   ]
 894   "TARGET_HAVE_MVE"
 895   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 896   [(set_attr "type" "mve_move")
 897 ])
 898
 899 ;;
 900 ;; [vaddvaq_s, vaddvaq_u])
 901 ;;
 902 (define_insn "mve_vaddvaq_<supf><mode>"
 903   [
 904    (set (match_operand:SI 0 "s_register_operand" "=Te")
 905         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 906                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 907          VADDVAQ))
 908   ]
 909   "TARGET_HAVE_MVE"
 910   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 911   [(set_attr "type" "mve_move")
 912 ])
 913
 914 ;;
 915 ;; [vaddvq_p_u, vaddvq_p_s])
 916 ;;
 917 (define_insn "mve_vaddvq_p_<supf><mode>"
 918   [
 919    (set (match_operand:SI 0 "s_register_operand" "=Te")
 920         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 921                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 922          VADDVQ_P))
 923   ]
 924   "TARGET_HAVE_MVE"
 925   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 926   [(set_attr "type" "mve_move")
 927    (set_attr "length""8")])
 928
 929 ;;
 930 ;; [vandq_u, vandq_s])
 931 ;;
 932 ;; signed and unsigned versions are the same: define the unsigned
 933 ;; insn, and use an expander for the signed one as we still reference
 934 ;; both names from arm_mve.h.
 935 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 936 (define_insn "mve_vandq_u<mode>"
 937   [
 938    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 939         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 940                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 941   ]
 942   "TARGET_HAVE_MVE"
 943   "@
 944    vand\t%q0, %q1, %q2
 945    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 946   [(set_attr "type" "mve_move")
 947 ])
 948 (define_expand "mve_vandq_s<mode>"
 949   [
 950    (set (match_operand:MVE_2 0 "s_register_operand")
 951         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 952                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 953   ]
 954   "TARGET_HAVE_MVE"
 955 )
 956
 957 ;;
 958 ;; [vbicq_s, vbicq_u])
 959 ;;
 960 (define_insn "mve_vbicq_u<mode>"
 961   [
 962    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 963         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 964                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 965   ]
 966   "TARGET_HAVE_MVE"
 967   "vbic\t%q0, %q1, %q2"
 968   [(set_attr "type" "mve_move")
 969 ])
 970
 971 (define_expand "mve_vbicq_s<mode>"
 972   [
 973    (set (match_operand:MVE_2 0 "s_register_operand")
 974         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 975                    (match_operand:MVE_2 1 "s_register_operand")))
 976   ]
 977   "TARGET_HAVE_MVE"
 978 )
 979
 980 ;;
 981 ;; [vbrsrq_n_u, vbrsrq_n_s])
 982 ;;
 983 (define_insn "mve_vbrsrq_n_<supf><mode>"
 984   [
 985    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 986         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 987                        (match_operand:SI 2 "s_register_operand" "r")]
 988          VBRSRQ_N))
 989   ]
 990   "TARGET_HAVE_MVE"
 991   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 992   [(set_attr "type" "mve_move")
 993 ])
 994
 995 ;;
 996 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 997 ;;
 998 (define_insn "mve_vcaddq<mve_rot><mode>"
 999   [
1000    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1001         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1002                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1003          VCADD))
1004   ]
1005   "TARGET_HAVE_MVE"
1006   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1007   [(set_attr "type" "mve_move")
1008 ])
1009
1010 ;; Auto vectorizer pattern for int vcadd
1011 (define_expand "cadd<rot><mode>3"
1012   [(set (match_operand:MVE_2 0 "register_operand")
1013         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
1014                        (match_operand:MVE_2 2 "register_operand")]
1015           VCADD))]
1016   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
1017 )
1018
1019 ;;
1020 ;; [veorq_u, veorq_s])
1021 ;;
1022 (define_insn "mve_veorq_u<mode>"
1023   [
1024    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1025         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1026                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1027   ]
1028   "TARGET_HAVE_MVE"
1029   "veor\t%q0, %q1, %q2"
1030   [(set_attr "type" "mve_move")
1031 ])
1032 (define_expand "mve_veorq_s<mode>"
1033   [
1034    (set (match_operand:MVE_2 0 "s_register_operand")
1035         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1036                    (match_operand:MVE_2 2 "s_register_operand")))
1037   ]
1038   "TARGET_HAVE_MVE"
1039 )
1040
1041 ;;
1042 ;; [vhaddq_n_u, vhaddq_n_s])
1043 ;;
1044 (define_insn "mve_vhaddq_n_<supf><mode>"
1045   [
1046    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1047         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1048                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1049          VHADDQ_N))
1050   ]
1051   "TARGET_HAVE_MVE"
1052   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1053   [(set_attr "type" "mve_move")
1054 ])
1055
1056 ;;
1057 ;; [vhaddq_s, vhaddq_u])
1058 ;;
1059 (define_insn "@mve_vhaddq_<supf><mode>"
1060   [
1061    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1062         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1063                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1064          VHADDQ))
1065   ]
1066   "TARGET_HAVE_MVE"
1067   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1068   [(set_attr "type" "mve_move")
1069 ])
1070
1071 ;;
1072 ;; [vhcaddq_rot270_s])
1073 ;;
1074 (define_insn "mve_vhcaddq_rot270_s<mode>"
1075   [
1076    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1077         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1078                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1079          VHCADDQ_ROT270_S))
1080   ]
1081   "TARGET_HAVE_MVE"
1082   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1083   [(set_attr "type" "mve_move")
1084 ])
1085
1086 ;;
1087 ;; [vhcaddq_rot90_s])
1088 ;;
1089 (define_insn "mve_vhcaddq_rot90_s<mode>"
1090   [
1091    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1092         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1093                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1094          VHCADDQ_ROT90_S))
1095   ]
1096   "TARGET_HAVE_MVE"
1097   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1098   [(set_attr "type" "mve_move")
1099 ])
1100
1101 ;;
1102 ;; [vhsubq_n_u, vhsubq_n_s])
1103 ;;
1104 (define_insn "mve_vhsubq_n_<supf><mode>"
1105   [
1106    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1107         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1108                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1109          VHSUBQ_N))
1110   ]
1111   "TARGET_HAVE_MVE"
1112   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1113   [(set_attr "type" "mve_move")
1114 ])
1115
1116 ;;
1117 ;; [vhsubq_s, vhsubq_u])
1118 ;;
1119 (define_insn "mve_vhsubq_<supf><mode>"
1120   [
1121    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1122         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1123                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1124          VHSUBQ))
1125   ]
1126   "TARGET_HAVE_MVE"
1127   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1128   [(set_attr "type" "mve_move")
1129 ])
1130
1131 ;;
1132 ;; [vmaxaq_s])
1133 ;;
1134 (define_insn "mve_vmaxaq_s<mode>"
1135   [
1136    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1137         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1138                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1139          VMAXAQ_S))
1140   ]
1141   "TARGET_HAVE_MVE"
1142   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1143   [(set_attr "type" "mve_move")
1144 ])
1145
1146 ;;
1147 ;; [vmaxavq_s])
1148 ;;
1149 (define_insn "mve_vmaxavq_s<mode>"
1150   [
1151    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1152         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1153                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1154          VMAXAVQ_S))
1155   ]
1156   "TARGET_HAVE_MVE"
1157   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1158   [(set_attr "type" "mve_move")
1159 ])
1160
1161 ;;
1162 ;; [vmaxq_u, vmaxq_s])
1163 ;;
1164 (define_insn "mve_vmaxq_s<mode>"
1165   [
1166    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1167         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1168                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1169   ]
1170   "TARGET_HAVE_MVE"
1171   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1172   [(set_attr "type" "mve_move")
1173 ])
1174
1175 (define_insn "mve_vmaxq_u<mode>"
1176   [
1177    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1178         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1179                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1180   ]
1181   "TARGET_HAVE_MVE"
1182   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1183   [(set_attr "type" "mve_move")
1184 ])
1185
1186 ;;
1187 ;; [vmaxvq_u, vmaxvq_s])
1188 ;;
1189 (define_insn "mve_vmaxvq_<supf><mode>"
1190   [
1191    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1192         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1193                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1194          VMAXVQ))
1195   ]
1196   "TARGET_HAVE_MVE"
1197   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1198   [(set_attr "type" "mve_move")
1199 ])
1200
1201 ;;
1202 ;; [vminaq_s])
1203 ;;
1204 (define_insn "mve_vminaq_s<mode>"
1205   [
1206    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1207         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1208                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1209          VMINAQ_S))
1210   ]
1211   "TARGET_HAVE_MVE"
1212   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1213   [(set_attr "type" "mve_move")
1214 ])
1215
1216 ;;
1217 ;; [vminavq_s])
1218 ;;
1219 (define_insn "mve_vminavq_s<mode>"
1220   [
1221    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1222         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1223                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1224          VMINAVQ_S))
1225   ]
1226   "TARGET_HAVE_MVE"
1227   "vminav.s%#<V_sz_elem>\t%0, %q2"
1228   [(set_attr "type" "mve_move")
1229 ])
1230
1231 ;;
1232 ;; [vminq_s, vminq_u])
1233 ;;
1234 (define_insn "mve_vminq_s<mode>"
1235   [
1236    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1237         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1238                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1239   ]
1240   "TARGET_HAVE_MVE"
1241   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1242   [(set_attr "type" "mve_move")
1243 ])
1244
1245 (define_insn "mve_vminq_u<mode>"
1246   [
1247    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1248         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1249                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1250   ]
1251   "TARGET_HAVE_MVE"
1252   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1253   [(set_attr "type" "mve_move")
1254 ])
1255
1256 ;;
1257 ;; [vminvq_u, vminvq_s])
1258 ;;
1259 (define_insn "mve_vminvq_<supf><mode>"
1260   [
1261    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1262         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1263                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1264          VMINVQ))
1265   ]
1266   "TARGET_HAVE_MVE"
1267   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1268   [(set_attr "type" "mve_move")
1269 ])
1270
1271 ;;
1272 ;; [vmladavq_u, vmladavq_s])
1273 ;;
1274 (define_insn "mve_vmladavq_<supf><mode>"
1275   [
1276    (set (match_operand:SI 0 "s_register_operand" "=Te")
1277         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1278                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1279          VMLADAVQ))
1280   ]
1281   "TARGET_HAVE_MVE"
1282   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1283   [(set_attr "type" "mve_move")
1284 ])
1285
1286 ;;
1287 ;; [vmladavxq_s])
1288 ;;
1289 (define_insn "mve_vmladavxq_s<mode>"
1290   [
1291    (set (match_operand:SI 0 "s_register_operand" "=Te")
1292         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1293                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1294          VMLADAVXQ_S))
1295   ]
1296   "TARGET_HAVE_MVE"
1297   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1298   [(set_attr "type" "mve_move")
1299 ])
1300
1301 ;;
1302 ;; [vmlsdavq_s])
1303 ;;
1304 (define_insn "mve_vmlsdavq_s<mode>"
1305   [
1306    (set (match_operand:SI 0 "s_register_operand" "=Te")
1307         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1308                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1309          VMLSDAVQ_S))
1310   ]
1311   "TARGET_HAVE_MVE"
1312   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1313   [(set_attr "type" "mve_move")
1314 ])
1315
1316 ;;
1317 ;; [vmlsdavxq_s])
1318 ;;
1319 (define_insn "mve_vmlsdavxq_s<mode>"
1320   [
1321    (set (match_operand:SI 0 "s_register_operand" "=Te")
1322         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1323                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1324          VMLSDAVXQ_S))
1325   ]
1326   "TARGET_HAVE_MVE"
1327   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1328   [(set_attr "type" "mve_move")
1329 ])
1330
1331 ;;
1332 ;; [vmulhq_s, vmulhq_u])
1333 ;;
1334 (define_insn "mve_vmulhq_<supf><mode>"
1335   [
1336    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1337         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1338                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1339          VMULHQ))
1340   ]
1341   "TARGET_HAVE_MVE"
1342   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1343   [(set_attr "type" "mve_move")
1344 ])
1345
1346 ;;
1347 ;; [vmullbq_int_u, vmullbq_int_s])
1348 ;;
1349 (define_insn "mve_vmullbq_int_<supf><mode>"
1350   [
1351    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1352         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1353                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1354          VMULLBQ_INT))
1355   ]
1356   "TARGET_HAVE_MVE"
1357   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1358   [(set_attr "type" "mve_move")
1359 ])
1360
1361 ;;
1362 ;; [vmulltq_int_u, vmulltq_int_s])
1363 ;;
1364 (define_insn "mve_vmulltq_int_<supf><mode>"
1365   [
1366    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1367         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1368                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1369          VMULLTQ_INT))
1370   ]
1371   "TARGET_HAVE_MVE"
1372   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1373   [(set_attr "type" "mve_move")
1374 ])
1375
1376 ;;
1377 ;; [vmulq_n_u, vmulq_n_s])
1378 ;;
1379 (define_insn "mve_vmulq_n_<supf><mode>"
1380   [
1381    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1382         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1383                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1384          VMULQ_N))
1385   ]
1386   "TARGET_HAVE_MVE"
1387   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1388   [(set_attr "type" "mve_move")
1389 ])
1390
1391 ;;
1392 ;; [vmulq_u, vmulq_s])
1393 ;;
1394 (define_insn "mve_vmulq_<supf><mode>"
1395   [
1396    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1397         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1398                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1399          VMULQ))
1400   ]
1401   "TARGET_HAVE_MVE"
1402   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1403   [(set_attr "type" "mve_move")
1404 ])
1405
1406 (define_insn "mve_vmulq<mode>"
1407   [
1408    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1409         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1410                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1411   ]
1412   "TARGET_HAVE_MVE"
1413   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1414   [(set_attr "type" "mve_move")
1415 ])
1416
1417 ;;
1418 ;; [vornq_u, vornq_s])
1419 ;;
1420 (define_insn "mve_vornq_s<mode>"
1421   [
1422    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1423         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1424                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1425   ]
1426   "TARGET_HAVE_MVE"
1427    "vorn\t%q0, %q1, %q2"
1428   [(set_attr "type" "mve_move")
1429 ])
1430
1431 (define_expand "mve_vornq_u<mode>"
1432   [
1433    (set (match_operand:MVE_2 0 "s_register_operand")
1434         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1435                    (match_operand:MVE_2 1 "s_register_operand")))
1436   ]
1437   "TARGET_HAVE_MVE"
1438 )
1439
1440 ;;
1441 ;; [vorrq_s, vorrq_u])
1442 ;;
1443 ;; signed and unsigned versions are the same: define the unsigned
1444 ;; insn, and use an expander for the signed one as we still reference
1445 ;; both names from arm_mve.h.
1446 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1447 (define_insn "mve_vorrq_s<mode>"
1448   [
1449    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1450         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1451                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1452   ]
1453   "TARGET_HAVE_MVE"
1454   "@
1455    vorr\t%q0, %q1, %q2
1456    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1457   [(set_attr "type" "mve_move")
1458 ])
1459 (define_expand "mve_vorrq_u<mode>"
1460   [
1461    (set (match_operand:MVE_2 0 "s_register_operand")
1462         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1463                    (match_operand:MVE_2 2 "neon_logic_op2")))
1464   ]
1465   "TARGET_HAVE_MVE"
1466 )
1467
1468 ;;
1469 ;; [vqaddq_n_s, vqaddq_n_u])
1470 ;;
1471 (define_insn "mve_vqaddq_n_<supf><mode>"
1472   [
1473    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1474         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1475                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1476          VQADDQ_N))
1477   ]
1478   "TARGET_HAVE_MVE"
1479   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1480   [(set_attr "type" "mve_move")
1481 ])
1482
1483 ;;
1484 ;; [vqaddq_u, vqaddq_s])
1485 ;;
1486 (define_insn "mve_vqaddq_<supf><mode>"
1487   [
1488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1489         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1490                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1491          VQADDQ))
1492   ]
1493   "TARGET_HAVE_MVE"
1494   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1495   [(set_attr "type" "mve_move")
1496 ])
1497
1498 ;;
1499 ;; [vqdmulhq_n_s])
1500 ;;
1501 (define_insn "mve_vqdmulhq_n_s<mode>"
1502   [
1503    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1504         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1505                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1506          VQDMULHQ_N_S))
1507   ]
1508   "TARGET_HAVE_MVE"
1509   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1510   [(set_attr "type" "mve_move")
1511 ])
1512
1513 ;;
1514 ;; [vqdmulhq_s])
1515 ;;
1516 (define_insn "mve_vqdmulhq_s<mode>"
1517   [
1518    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1519         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1520                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1521          VQDMULHQ_S))
1522   ]
1523   "TARGET_HAVE_MVE"
1524   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1525   [(set_attr "type" "mve_move")
1526 ])
1527
1528 ;;
1529 ;; [vqrdmulhq_n_s])
1530 ;;
1531 (define_insn "mve_vqrdmulhq_n_s<mode>"
1532   [
1533    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1534         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1535                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1536          VQRDMULHQ_N_S))
1537   ]
1538   "TARGET_HAVE_MVE"
1539   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1540   [(set_attr "type" "mve_move")
1541 ])
1542
1543 ;;
1544 ;; [vqrdmulhq_s])
1545 ;;
1546 (define_insn "mve_vqrdmulhq_s<mode>"
1547   [
1548    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1549         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1550                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1551          VQRDMULHQ_S))
1552   ]
1553   "TARGET_HAVE_MVE"
1554   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1555   [(set_attr "type" "mve_move")
1556 ])
1557
1558 ;;
1559 ;; [vqrshlq_n_s, vqrshlq_n_u])
1560 ;;
1561 (define_insn "mve_vqrshlq_n_<supf><mode>"
1562   [
1563    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1564         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1565                        (match_operand:SI 2 "s_register_operand" "r")]
1566          VQRSHLQ_N))
1567   ]
1568   "TARGET_HAVE_MVE"
1569   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1570   [(set_attr "type" "mve_move")
1571 ])
1572
1573 ;;
1574 ;; [vqrshlq_s, vqrshlq_u])
1575 ;;
1576 (define_insn "mve_vqrshlq_<supf><mode>"
1577   [
1578    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1579         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1580                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1581          VQRSHLQ))
1582   ]
1583   "TARGET_HAVE_MVE"
1584   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1585   [(set_attr "type" "mve_move")
1586 ])
1587
1588 ;;
1589 ;; [vqshlq_n_s, vqshlq_n_u])
1590 ;;
1591 (define_insn "mve_vqshlq_n_<supf><mode>"
1592   [
1593    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1594         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1595                        (match_operand:SI 2 "immediate_operand" "i")]
1596          VQSHLQ_N))
1597   ]
1598   "TARGET_HAVE_MVE"
1599   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1600   [(set_attr "type" "mve_move")
1601 ])
1602
1603 ;;
1604 ;; [vqshlq_r_u, vqshlq_r_s])
1605 ;;
1606 (define_insn "mve_vqshlq_r_<supf><mode>"
1607   [
1608    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1609         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1610                        (match_operand:SI 2 "s_register_operand" "r")]
1611          VQSHLQ_R))
1612   ]
1613   "TARGET_HAVE_MVE"
1614   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1615   [(set_attr "type" "mve_move")
1616 ])
1617
1618 ;;
1619 ;; [vqshlq_s, vqshlq_u])
1620 ;;
1621 (define_insn "mve_vqshlq_<supf><mode>"
1622   [
1623    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1624         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1625                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1626          VQSHLQ))
1627   ]
1628   "TARGET_HAVE_MVE"
1629   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1630   [(set_attr "type" "mve_move")
1631 ])
1632
1633 ;;
1634 ;; [vqshluq_n_s])
1635 ;;
1636 (define_insn "mve_vqshluq_n_s<mode>"
1637   [
1638    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1639         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1640                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1641          VQSHLUQ_N_S))
1642   ]
1643   "TARGET_HAVE_MVE"
1644   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1645   [(set_attr "type" "mve_move")
1646 ])
1647
1648 ;;
1649 ;; [vqsubq_n_s, vqsubq_n_u])
1650 ;;
1651 (define_insn "mve_vqsubq_n_<supf><mode>"
1652   [
1653    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1654         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1655                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1656          VQSUBQ_N))
1657   ]
1658   "TARGET_HAVE_MVE"
1659   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1660   [(set_attr "type" "mve_move")
1661 ])
1662
1663 ;;
1664 ;; [vqsubq_u, vqsubq_s])
1665 ;;
1666 (define_insn "mve_vqsubq_<supf><mode>"
1667   [
1668    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1669         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1670                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1671          VQSUBQ))
1672   ]
1673   "TARGET_HAVE_MVE"
1674   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1675   [(set_attr "type" "mve_move")
1676 ])
1677
1678 ;;
1679 ;; [vrhaddq_s, vrhaddq_u])
1680 ;;
1681 (define_insn "@mve_vrhaddq_<supf><mode>"
1682   [
1683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1685                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1686          VRHADDQ))
1687   ]
1688   "TARGET_HAVE_MVE"
1689   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1690   [(set_attr "type" "mve_move")
1691 ])
1692
1693 ;;
1694 ;; [vrmulhq_s, vrmulhq_u])
1695 ;;
1696 (define_insn "mve_vrmulhq_<supf><mode>"
1697   [
1698    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1699         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1700                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1701          VRMULHQ))
1702   ]
1703   "TARGET_HAVE_MVE"
1704   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1705   [(set_attr "type" "mve_move")
1706 ])
1707
1708 ;;
1709 ;; [vrshlq_n_u, vrshlq_n_s])
1710 ;;
1711 (define_insn "mve_vrshlq_n_<supf><mode>"
1712   [
1713    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1714         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1715                        (match_operand:SI 2 "s_register_operand" "r")]
1716          VRSHLQ_N))
1717   ]
1718   "TARGET_HAVE_MVE"
1719   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1720   [(set_attr "type" "mve_move")
1721 ])
1722
1723 ;;
1724 ;; [vrshlq_s, vrshlq_u])
1725 ;;
1726 (define_insn "mve_vrshlq_<supf><mode>"
1727   [
1728    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1729         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1730                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1731          VRSHLQ))
1732   ]
1733   "TARGET_HAVE_MVE"
1734   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1735   [(set_attr "type" "mve_move")
1736 ])
1737
1738 ;;
1739 ;; [vrshrq_n_s, vrshrq_n_u])
1740 ;;
1741 (define_insn "mve_vrshrq_n_<supf><mode>"
1742   [
1743    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1744         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1745                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1746          VRSHRQ_N))
1747   ]
1748   "TARGET_HAVE_MVE"
1749   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1750   [(set_attr "type" "mve_move")
1751 ])
1752
1753 ;;
1754 ;; [vshlq_n_u, vshlq_n_s])
1755 ;;
1756 (define_insn "mve_vshlq_n_<supf><mode>"
1757   [
1758    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1759         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1760                        (match_operand:SI 2 "immediate_operand" "i")]
1761          VSHLQ_N))
1762   ]
1763   "TARGET_HAVE_MVE"
1764   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1765   [(set_attr "type" "mve_move")
1766 ])
1767
1768 ;;
1769 ;; [vshlq_r_s, vshlq_r_u])
1770 ;;
1771 (define_insn "mve_vshlq_r_<supf><mode>"
1772   [
1773    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1774         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1775                        (match_operand:SI 2 "s_register_operand" "r")]
1776          VSHLQ_R))
1777   ]
1778   "TARGET_HAVE_MVE"
1779   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1780   [(set_attr "type" "mve_move")
1781 ])
1782
1783 ;;
1784 ;; [vsubq_n_s, vsubq_n_u])
1785 ;;
1786 (define_insn "mve_vsubq_n_<supf><mode>"
1787   [
1788    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1789         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1790                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1791          VSUBQ_N))
1792   ]
1793   "TARGET_HAVE_MVE"
1794   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1795   [(set_attr "type" "mve_move")
1796 ])
1797
1798 ;;
1799 ;; [vsubq_s, vsubq_u])
1800 ;;
1801 (define_insn "mve_vsubq_<supf><mode>"
1802   [
1803    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1804         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1805                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1806          VSUBQ))
1807   ]
1808   "TARGET_HAVE_MVE"
1809   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1810   [(set_attr "type" "mve_move")
1811 ])
1812
1813 (define_insn "mve_vsubq<mode>"
1814   [
1815    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1816         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1817                      (match_operand:MVE_2 2 "s_register_operand" "w")))
1818   ]
1819   "TARGET_HAVE_MVE"
1820   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1821   [(set_attr "type" "mve_move")
1822 ])
1823
1824 ;;
1825 ;; [vabdq_f])
1826 ;;
1827 (define_insn "mve_vabdq_f<mode>"
1828   [
1829    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1830         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1831                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1832          VABDQ_F))
1833   ]
1834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1835   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
1836   [(set_attr "type" "mve_move")
1837 ])
1838
1839 ;;
1840 ;; [vaddlvaq_s vaddlvaq_u])
1841 ;;
1842 (define_insn "mve_vaddlvaq_<supf>v4si"
1843   [
1844    (set (match_operand:DI 0 "s_register_operand" "=r")
1845         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1846                     (match_operand:V4SI 2 "s_register_operand" "w")]
1847          VADDLVAQ))
1848   ]
1849   "TARGET_HAVE_MVE"
1850   "vaddlva.<supf>32 %Q0, %R0, %q2"
1851   [(set_attr "type" "mve_move")
1852 ])
1853
1854 ;;
1855 ;; [vaddq_n_f])
1856 ;;
1857 (define_insn "mve_vaddq_n_f<mode>"
1858   [
1859    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1860         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1861                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1862          VADDQ_N_F))
1863   ]
1864   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1865   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
1866   [(set_attr "type" "mve_move")
1867 ])
1868
1869 ;;
1870 ;; [vandq_f])
1871 ;;
1872 (define_insn "mve_vandq_f<mode>"
1873   [
1874    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1875         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1876                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1877   ]
1878   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1879   "vand %q0, %q1, %q2"
1880   [(set_attr "type" "mve_move")
1881 ])
1882
1883 ;;
1884 ;; [vbicq_f])
1885 ;;
1886 (define_insn "mve_vbicq_f<mode>"
1887   [
1888    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1889         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
1890                               (match_operand:MVE_0 2 "s_register_operand" "w")))
1891   ]
1892   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1893   "vbic %q0, %q1, %q2"
1894   [(set_attr "type" "mve_move")
1895 ])
1896
1897 ;;
1898 ;; [vbicq_n_s, vbicq_n_u])
1899 ;;
1900 (define_insn "mve_vbicq_n_<supf><mode>"
1901   [
1902    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1903         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1904                        (match_operand:SI 2 "immediate_operand" "i")]
1905          VBICQ_N))
1906   ]
1907   "TARGET_HAVE_MVE"
1908   "vbic.i%#<V_sz_elem>  %q0, %2"
1909   [(set_attr "type" "mve_move")
1910 ])
1911
1912 ;;
1913 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
1914 ;;
1915 (define_insn "mve_vcaddq<mve_rot><mode>"
1916   [
1917    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1918         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1919                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1920          VCADD))
1921   ]
1922   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1923   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1924   [(set_attr "type" "mve_move")
1925 ])
1926
1927 ;;
1928 ;; [vcmpeqq_f, vcmpgeq_f, vcmpgtq_f, vcmpleq_f, vcmpltq_f, vcmpneq_f])
1929 ;;
1930 (define_insn "@mve_vcmp<mve_cmp_op>q_f<mode>"
1931   [
1932    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1933         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1934                                (match_operand:MVE_0 2 "s_register_operand" "w")))
1935   ]
1936   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1937   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
1938   [(set_attr "type" "mve_move")
1939 ])
1940
1941 ;;
1942 ;; [vcmpeqq_n_f, vcmpgeq_n_f, vcmpgtq_n_f, vcmpleq_n_f, vcmpltq_n_f, vcmpneq_n_f])
1943 ;;
1944 (define_insn "@mve_vcmp<mve_cmp_op>q_n_f<mode>"
1945   [
1946    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1947         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1948                                (match_operand:<V_elem> 2 "s_register_operand" "r")))
1949   ]
1950   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1951   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %2"
1952   [(set_attr "type" "mve_move")
1953 ])
1954
1955 ;;
1956 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
1957 ;;
1958 (define_insn "mve_vcmulq<mve_rot><mode>"
1959   [
1960    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1961         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1962                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1963          VCMUL))
1964   ]
1965   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1966   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1967   [(set_attr "type" "mve_move")
1968 ])
1969
1970 ;;
1971 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
1972 ;;
1973 (define_insn "mve_vctp<mode1>q_mhi"
1974   [
1975    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1976         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
1977                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1978          VCTPQ_M))
1979   ]
1980   "TARGET_HAVE_MVE"
1981   "vpst\;vctpt.<mode1> %1"
1982   [(set_attr "type" "mve_move")
1983    (set_attr "length""8")])
1984
1985 ;;
1986 ;; [vcvtbq_f16_f32])
1987 ;;
1988 (define_insn "mve_vcvtbq_f16_f32v8hf"
1989   [
1990    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1991         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1992                       (match_operand:V4SF 2 "s_register_operand" "w")]
1993          VCVTBQ_F16_F32))
1994   ]
1995   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1996   "vcvtb.f16.f32 %q0, %q2"
1997   [(set_attr "type" "mve_move")
1998 ])
1999
2000 ;;
2001 ;; [vcvttq_f16_f32])
2002 ;;
2003 (define_insn "mve_vcvttq_f16_f32v8hf"
2004   [
2005    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2006         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2007                       (match_operand:V4SF 2 "s_register_operand" "w")]
2008          VCVTTQ_F16_F32))
2009   ]
2010   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2011   "vcvtt.f16.f32 %q0, %q2"
2012   [(set_attr "type" "mve_move")
2013 ])
2014
2015 ;;
2016 ;; [veorq_f])
2017 ;;
2018 (define_insn "mve_veorq_f<mode>"
2019   [
2020    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2021         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2022                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2023   ]
2024   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2025   "veor %q0, %q1, %q2"
2026   [(set_attr "type" "mve_move")
2027 ])
2028
2029 ;;
2030 ;; [vmaxnmaq_f])
2031 ;;
2032 (define_insn "mve_vmaxnmaq_f<mode>"
2033   [
2034    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2035         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2036                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2037          VMAXNMAQ_F))
2038   ]
2039   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2040   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2041   [(set_attr "type" "mve_move")
2042 ])
2043
2044 ;;
2045 ;; [vmaxnmavq_f])
2046 ;;
2047 (define_insn "mve_vmaxnmavq_f<mode>"
2048   [
2049    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2050         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2051                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2052          VMAXNMAVQ_F))
2053   ]
2054   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2055   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2056   [(set_attr "type" "mve_move")
2057 ])
2058
2059 ;;
2060 ;; [vmaxnmq_f])
2061 ;;
2062 (define_insn "mve_vmaxnmq_f<mode>"
2063   [
2064    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2065         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2066                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2067   ]
2068   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2069   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2070   [(set_attr "type" "mve_move")
2071 ])
2072
2073 ;;
2074 ;; [vmaxnmvq_f])
2075 ;;
2076 (define_insn "mve_vmaxnmvq_f<mode>"
2077   [
2078    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2079         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2080                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2081          VMAXNMVQ_F))
2082   ]
2083   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2084   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2085   [(set_attr "type" "mve_move")
2086 ])
2087
2088 ;;
2089 ;; [vminnmaq_f])
2090 ;;
2091 (define_insn "mve_vminnmaq_f<mode>"
2092   [
2093    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2094         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2095                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2096          VMINNMAQ_F))
2097   ]
2098   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2099   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2100   [(set_attr "type" "mve_move")
2101 ])
2102
2103 ;;
2104 ;; [vminnmavq_f])
2105 ;;
2106 (define_insn "mve_vminnmavq_f<mode>"
2107   [
2108    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2109         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2110                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2111          VMINNMAVQ_F))
2112   ]
2113   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2114   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2115   [(set_attr "type" "mve_move")
2116 ])
2117
2118 ;;
2119 ;; [vminnmq_f])
2120 ;;
2121 (define_insn "mve_vminnmq_f<mode>"
2122   [
2123    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2124         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2125                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2126   ]
2127   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2128   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2129   [(set_attr "type" "mve_move")
2130 ])
2131
2132 ;;
2133 ;; [vminnmvq_f])
2134 ;;
2135 (define_insn "mve_vminnmvq_f<mode>"
2136   [
2137    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2138         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2139                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2140          VMINNMVQ_F))
2141   ]
2142   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2143   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2144   [(set_attr "type" "mve_move")
2145 ])
2146
2147 ;;
2148 ;; [vmlaldavq_u, vmlaldavq_s])
2149 ;;
2150 (define_insn "mve_vmlaldavq_<supf><mode>"
2151   [
2152    (set (match_operand:DI 0 "s_register_operand" "=r")
2153         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2154                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2155          VMLALDAVQ))
2156   ]
2157   "TARGET_HAVE_MVE"
2158   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2159   [(set_attr "type" "mve_move")
2160 ])
2161
2162 ;;
2163 ;; [vmlaldavxq_s])
2164 ;;
2165 (define_insn "mve_vmlaldavxq_s<mode>"
2166   [
2167    (set (match_operand:DI 0 "s_register_operand" "=r")
2168         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2169                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2170          VMLALDAVXQ_S))
2171   ]
2172   "TARGET_HAVE_MVE"
2173   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2174   [(set_attr "type" "mve_move")
2175 ])
2176
2177 ;;
2178 ;; [vmlsldavq_s])
2179 ;;
2180 (define_insn "mve_vmlsldavq_s<mode>"
2181   [
2182    (set (match_operand:DI 0 "s_register_operand" "=r")
2183         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2184                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2185          VMLSLDAVQ_S))
2186   ]
2187   "TARGET_HAVE_MVE"
2188   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2189   [(set_attr "type" "mve_move")
2190 ])
2191
2192 ;;
2193 ;; [vmlsldavxq_s])
2194 ;;
2195 (define_insn "mve_vmlsldavxq_s<mode>"
2196   [
2197    (set (match_operand:DI 0 "s_register_operand" "=r")
2198         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2199                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2200          VMLSLDAVXQ_S))
2201   ]
2202   "TARGET_HAVE_MVE"
2203   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2204   [(set_attr "type" "mve_move")
2205 ])
2206
2207 ;;
2208 ;; [vmovnbq_u, vmovnbq_s])
2209 ;;
2210 (define_insn "mve_vmovnbq_<supf><mode>"
2211   [
2212    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2213         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2214                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2215          VMOVNBQ))
2216   ]
2217   "TARGET_HAVE_MVE"
2218   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2219   [(set_attr "type" "mve_move")
2220 ])
2221
2222 ;; vmovnb pattern used by the vec_pack_trunc expander to avoid the
2223 ;; need for an uninitialized input operand.
2224 (define_insn "@mve_vec_pack_trunc_lo_<mode>"
2225   [
2226    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2227         (unspec:<V_narrow_pack> [(match_operand:MVE_5 1 "s_register_operand" "w")]
2228          VMOVNBQ_S))
2229   ]
2230   "TARGET_HAVE_MVE"
2231   "vmovnb.i%#<V_sz_elem>        %q0, %q1"
2232   [(set_attr "type" "mve_move")
2233 ])
2234
2235 ;;
2236 ;; [vmovntq_s, vmovntq_u])
2237 ;;
2238 (define_insn "@mve_vmovntq_<supf><mode>"
2239   [
2240    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2241         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2242                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2243          VMOVNTQ))
2244   ]
2245   "TARGET_HAVE_MVE"
2246   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2247   [(set_attr "type" "mve_move")
2248 ])
2249
2250 ;;
2251 ;; [vmulq_f])
2252 ;;
2253 (define_insn "mve_vmulq_f<mode>"
2254   [
2255    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2256         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2257                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2258   ]
2259   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2260   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2261   [(set_attr "type" "mve_move")
2262 ])
2263
2264 ;;
2265 ;; [vmulq_n_f])
2266 ;;
2267 (define_insn "mve_vmulq_n_f<mode>"
2268   [
2269    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2270         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2271                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2272          VMULQ_N_F))
2273   ]
2274   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2275   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2276   [(set_attr "type" "mve_move")
2277 ])
2278
2279 ;;
2280 ;; [vornq_f])
2281 ;;
2282 (define_insn "mve_vornq_f<mode>"
2283   [
2284    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2285         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
2286                    (match_operand:MVE_0 1 "s_register_operand" "w")))
2287   ]
2288   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2289   "vorn %q0, %q1, %q2"
2290   [(set_attr "type" "mve_move")
2291 ])
2292
2293 ;;
2294 ;; [vorrq_f])
2295 ;;
2296 (define_insn "mve_vorrq_f<mode>"
2297   [
2298    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2299         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2300                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2301   ]
2302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2303   "vorr %q0, %q1, %q2"
2304   [(set_attr "type" "mve_move")
2305 ])
2306
2307 ;;
2308 ;; [vorrq_n_u, vorrq_n_s])
2309 ;;
2310 (define_insn "mve_vorrq_n_<supf><mode>"
2311   [
2312    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2313         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2314                        (match_operand:SI 2 "immediate_operand" "i")]
2315          VORRQ_N))
2316   ]
2317   "TARGET_HAVE_MVE"
2318   "vorr.i%#<V_sz_elem>  %q0, %2"
2319   [(set_attr "type" "mve_move")
2320 ])
2321
2322 ;;
2323 ;; [vqdmullbq_n_s])
2324 ;;
2325 (define_insn "mve_vqdmullbq_n_s<mode>"
2326   [
2327    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2328         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2329                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2330          VQDMULLBQ_N_S))
2331   ]
2332   "TARGET_HAVE_MVE"
2333   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2334   [(set_attr "type" "mve_move")
2335 ])
2336
2337 ;;
2338 ;; [vqdmullbq_s])
2339 ;;
2340 (define_insn "mve_vqdmullbq_s<mode>"
2341   [
2342    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2343         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2344                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2345          VQDMULLBQ_S))
2346   ]
2347   "TARGET_HAVE_MVE"
2348   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2349   [(set_attr "type" "mve_move")
2350 ])
2351
2352 ;;
2353 ;; [vqdmulltq_n_s])
2354 ;;
2355 (define_insn "mve_vqdmulltq_n_s<mode>"
2356   [
2357    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2358         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2359                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2360          VQDMULLTQ_N_S))
2361   ]
2362   "TARGET_HAVE_MVE"
2363   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2364   [(set_attr "type" "mve_move")
2365 ])
2366
2367 ;;
2368 ;; [vqdmulltq_s])
2369 ;;
2370 (define_insn "mve_vqdmulltq_s<mode>"
2371   [
2372    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2373         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2374                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2375          VQDMULLTQ_S))
2376   ]
2377   "TARGET_HAVE_MVE"
2378   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2379   [(set_attr "type" "mve_move")
2380 ])
2381
2382 ;;
2383 ;; [vqmovnbq_u, vqmovnbq_s])
2384 ;;
2385 (define_insn "mve_vqmovnbq_<supf><mode>"
2386   [
2387    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2388         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2389                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2390          VQMOVNBQ))
2391   ]
2392   "TARGET_HAVE_MVE"
2393   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2394   [(set_attr "type" "mve_move")
2395 ])
2396
2397 ;;
2398 ;; [vqmovntq_u, vqmovntq_s])
2399 ;;
2400 (define_insn "mve_vqmovntq_<supf><mode>"
2401   [
2402    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2403         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2404                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2405          VQMOVNTQ))
2406   ]
2407   "TARGET_HAVE_MVE"
2408   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2409   [(set_attr "type" "mve_move")
2410 ])
2411
2412 ;;
2413 ;; [vqmovunbq_s])
2414 ;;
2415 (define_insn "mve_vqmovunbq_s<mode>"
2416   [
2417    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2418         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2419                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2420          VQMOVUNBQ_S))
2421   ]
2422   "TARGET_HAVE_MVE"
2423   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2424   [(set_attr "type" "mve_move")
2425 ])
2426
2427 ;;
2428 ;; [vqmovuntq_s])
2429 ;;
2430 (define_insn "mve_vqmovuntq_s<mode>"
2431   [
2432    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2433         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2434                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2435          VQMOVUNTQ_S))
2436   ]
2437   "TARGET_HAVE_MVE"
2438   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2439   [(set_attr "type" "mve_move")
2440 ])
2441
2442 ;;
2443 ;; [vrmlaldavhxq_s])
2444 ;;
2445 (define_insn "mve_vrmlaldavhxq_sv4si"
2446   [
2447    (set (match_operand:DI 0 "s_register_operand" "=r")
2448         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2449                     (match_operand:V4SI 2 "s_register_operand" "w")]
2450          VRMLALDAVHXQ_S))
2451   ]
2452   "TARGET_HAVE_MVE"
2453   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2454   [(set_attr "type" "mve_move")
2455 ])
2456
2457 ;;
2458 ;; [vrmlsldavhq_s])
2459 ;;
2460 (define_insn "mve_vrmlsldavhq_sv4si"
2461   [
2462    (set (match_operand:DI 0 "s_register_operand" "=r")
2463         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2464                     (match_operand:V4SI 2 "s_register_operand" "w")]
2465          VRMLSLDAVHQ_S))
2466   ]
2467   "TARGET_HAVE_MVE"
2468   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2469   [(set_attr "type" "mve_move")
2470 ])
2471
2472 ;;
2473 ;; [vrmlsldavhxq_s])
2474 ;;
2475 (define_insn "mve_vrmlsldavhxq_sv4si"
2476   [
2477    (set (match_operand:DI 0 "s_register_operand" "=r")
2478         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2479                     (match_operand:V4SI 2 "s_register_operand" "w")]
2480          VRMLSLDAVHXQ_S))
2481   ]
2482   "TARGET_HAVE_MVE"
2483   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2484   [(set_attr "type" "mve_move")
2485 ])
2486
2487 ;;
2488 ;; [vshllbq_n_s, vshllbq_n_u])
2489 ;;
2490 (define_insn "mve_vshllbq_n_<supf><mode>"
2491   [
2492    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2493         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2494                                   (match_operand:SI 2 "immediate_operand" "i")]
2495          VSHLLBQ_N))
2496   ]
2497   "TARGET_HAVE_MVE"
2498   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2499   [(set_attr "type" "mve_move")
2500 ])
2501
2502 ;;
2503 ;; [vshlltq_n_u, vshlltq_n_s])
2504 ;;
2505 (define_insn "mve_vshlltq_n_<supf><mode>"
2506   [
2507    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2508         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2509                                   (match_operand:SI 2 "immediate_operand" "i")]
2510          VSHLLTQ_N))
2511   ]
2512   "TARGET_HAVE_MVE"
2513   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2514   [(set_attr "type" "mve_move")
2515 ])
2516
2517 ;;
2518 ;; [vsubq_f])
2519 ;;
2520 (define_insn "mve_vsubq_f<mode>"
2521   [
2522    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2523         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2524                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2525   ]
2526   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2527   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2528   [(set_attr "type" "mve_move")
2529 ])
2530
2531 ;;
2532 ;; [vmulltq_poly_p])
2533 ;;
2534 (define_insn "mve_vmulltq_poly_p<mode>"
2535   [
2536    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2537         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2538                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2539          VMULLTQ_POLY_P))
2540   ]
2541   "TARGET_HAVE_MVE"
2542   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2543   [(set_attr "type" "mve_move")
2544 ])
2545
2546 ;;
2547 ;; [vmullbq_poly_p])
2548 ;;
2549 (define_insn "mve_vmullbq_poly_p<mode>"
2550   [
2551    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2552         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2553                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2554          VMULLBQ_POLY_P))
2555   ]
2556   "TARGET_HAVE_MVE"
2557   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2558   [(set_attr "type" "mve_move")
2559 ])
2560
2561 ;;
2562 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2563 ;;
2564 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2565   [
2566    (set (match_operand:DI 0 "s_register_operand" "=r")
2567         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2568                     (match_operand:V4SI 2 "s_register_operand" "w")]
2569          VRMLALDAVHQ))
2570   ]
2571   "TARGET_HAVE_MVE"
2572   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2573   [(set_attr "type" "mve_move")
2574 ])
2575
2576 ;;
2577 ;; [vbicq_m_n_s, vbicq_m_n_u])
2578 ;;
2579 (define_insn "mve_vbicq_m_n_<supf><mode>"
2580   [
2581    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2582         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2583                        (match_operand:SI 2 "immediate_operand" "i")
2584                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2585          VBICQ_M_N))
2586   ]
2587   "TARGET_HAVE_MVE"
2588   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2589   [(set_attr "type" "mve_move")
2590    (set_attr "length""8")])
2591 ;;
2592 ;; [vcmpeqq_m_f])
2593 ;;
2594 (define_insn "mve_vcmpeqq_m_f<mode>"
2595   [
2596    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2597         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2598                     (match_operand:MVE_0 2 "s_register_operand" "w")
2599                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2600          VCMPEQQ_M_F))
2601   ]
2602   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2603   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2604   [(set_attr "type" "mve_move")
2605    (set_attr "length""8")])
2606 ;;
2607 ;; [vcvtaq_m_u, vcvtaq_m_s])
2608 ;;
2609 (define_insn "mve_vcvtaq_m_<supf><mode>"
2610   [
2611    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2612         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2613                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2614                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2615          VCVTAQ_M))
2616   ]
2617   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2618   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2619   [(set_attr "type" "mve_move")
2620    (set_attr "length""8")])
2621 ;;
2622 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2623 ;;
2624 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2625   [
2626    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2627         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2628                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2629                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2630          VCVTQ_M_TO_F))
2631   ]
2632   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2633   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2634   [(set_attr "type" "mve_move")
2635    (set_attr "length""8")])
2636 ;;
2637 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2638 ;;
2639 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2640   [
2641    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2642         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2643                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2644                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2645          VQRSHRNBQ_N))
2646   ]
2647   "TARGET_HAVE_MVE"
2648   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2649   [(set_attr "type" "mve_move")
2650 ])
2651 ;;
2652 ;; [vqrshrunbq_n_s])
2653 ;;
2654 (define_insn "mve_vqrshrunbq_n_s<mode>"
2655   [
2656    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2657         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2658                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2659                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2660          VQRSHRUNBQ_N_S))
2661   ]
2662   "TARGET_HAVE_MVE"
2663   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
2664   [(set_attr "type" "mve_move")
2665 ])
2666 ;;
2667 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
2668 ;;
2669 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
2670   [
2671    (set (match_operand:DI 0 "s_register_operand" "=r")
2672         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2673                     (match_operand:V4SI 2 "s_register_operand" "w")
2674                     (match_operand:V4SI 3 "s_register_operand" "w")]
2675          VRMLALDAVHAQ))
2676   ]
2677   "TARGET_HAVE_MVE"
2678   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
2679   [(set_attr "type" "mve_move")
2680 ])
2681
2682 ;;
2683 ;; [vabavq_s, vabavq_u])
2684 ;;
2685 (define_insn "mve_vabavq_<supf><mode>"
2686   [
2687    (set (match_operand:SI 0 "s_register_operand" "=r")
2688         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2689                     (match_operand:MVE_2 2 "s_register_operand" "w")
2690                     (match_operand:MVE_2 3 "s_register_operand" "w")]
2691          VABAVQ))
2692   ]
2693   "TARGET_HAVE_MVE"
2694   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2695   [(set_attr "type" "mve_move")
2696 ])
2697
2698 ;;
2699 ;; [vshlcq_u vshlcq_s]
2700 ;;
2701 (define_expand "mve_vshlcq_vec_<supf><mode>"
2702  [(match_operand:MVE_2 0 "s_register_operand")
2703   (match_operand:MVE_2 1 "s_register_operand")
2704   (match_operand:SI 2 "s_register_operand")
2705   (match_operand:SI 3 "mve_imm_32")
2706   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2707  "TARGET_HAVE_MVE"
2708 {
2709   rtx ignore_wb = gen_reg_rtx (SImode);
2710   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
2711                                       operands[2], operands[3]));
2712   DONE;
2713 })
2714
2715 (define_expand "mve_vshlcq_carry_<supf><mode>"
2716  [(match_operand:SI 0 "s_register_operand")
2717   (match_operand:MVE_2 1 "s_register_operand")
2718   (match_operand:SI 2 "s_register_operand")
2719   (match_operand:SI 3 "mve_imm_32")
2720   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2721  "TARGET_HAVE_MVE"
2722 {
2723   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
2724   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
2725                                       operands[2], operands[3]));
2726   DONE;
2727 })
2728
2729 (define_insn "mve_vshlcq_<supf><mode>"
2730  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
2731        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
2732                       (match_operand:SI 3 "s_register_operand" "1")
2733                       (match_operand:SI 4 "mve_imm_32" "Rf")]
2734         VSHLCQ))
2735   (set (match_operand:SI  1 "s_register_operand" "=r")
2736        (unspec:SI [(match_dup 2)
2737                    (match_dup 3)
2738                    (match_dup 4)]
2739         VSHLCQ))]
2740  "TARGET_HAVE_MVE"
2741  "vshlc %q0, %1, %4")
2742
2743 ;;
2744 ;; [vabsq_m_s])
2745 ;;
2746 (define_insn "mve_vabsq_m_s<mode>"
2747   [
2748    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2749         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2750                        (match_operand:MVE_2 2 "s_register_operand" "w")
2751                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2752          VABSQ_M_S))
2753   ]
2754   "TARGET_HAVE_MVE"
2755   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
2756   [(set_attr "type" "mve_move")
2757    (set_attr "length""8")])
2758
2759 ;;
2760 ;; [vaddvaq_p_u, vaddvaq_p_s])
2761 ;;
2762 (define_insn "mve_vaddvaq_p_<supf><mode>"
2763   [
2764    (set (match_operand:SI 0 "s_register_operand" "=Te")
2765         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2766                        (match_operand:MVE_2 2 "s_register_operand" "w")
2767                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2768          VADDVAQ_P))
2769   ]
2770   "TARGET_HAVE_MVE"
2771   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
2772   [(set_attr "type" "mve_move")
2773    (set_attr "length""8")])
2774
2775 ;;
2776 ;; [vclsq_m_s])
2777 ;;
2778 (define_insn "mve_vclsq_m_s<mode>"
2779   [
2780    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2781         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2782                        (match_operand:MVE_2 2 "s_register_operand" "w")
2783                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2784          VCLSQ_M_S))
2785   ]
2786   "TARGET_HAVE_MVE"
2787   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
2788   [(set_attr "type" "mve_move")
2789    (set_attr "length""8")])
2790
2791 ;;
2792 ;; [vclzq_m_s, vclzq_m_u])
2793 ;;
2794 (define_insn "mve_vclzq_m_<supf><mode>"
2795   [
2796    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2797         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2798                        (match_operand:MVE_2 2 "s_register_operand" "w")
2799                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2800          VCLZQ_M))
2801   ]
2802   "TARGET_HAVE_MVE"
2803   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
2804   [(set_attr "type" "mve_move")
2805    (set_attr "length""8")])
2806
2807 ;;
2808 ;; [vcmpcsq_m_n_u])
2809 ;;
2810 (define_insn "mve_vcmpcsq_m_n_u<mode>"
2811   [
2812    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2813         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2814                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2815                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2816          VCMPCSQ_M_N_U))
2817   ]
2818   "TARGET_HAVE_MVE"
2819   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
2820   [(set_attr "type" "mve_move")
2821    (set_attr "length""8")])
2822
2823 ;;
2824 ;; [vcmpcsq_m_u])
2825 ;;
2826 (define_insn "mve_vcmpcsq_m_u<mode>"
2827   [
2828    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2829         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2830                        (match_operand:MVE_2 2 "s_register_operand" "w")
2831                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2832          VCMPCSQ_M_U))
2833   ]
2834   "TARGET_HAVE_MVE"
2835   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
2836   [(set_attr "type" "mve_move")
2837    (set_attr "length""8")])
2838
2839 ;;
2840 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
2841 ;;
2842 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
2843   [
2844    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2845         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2846                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2847                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2848          VCMPEQQ_M_N))
2849   ]
2850   "TARGET_HAVE_MVE"
2851   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
2852   [(set_attr "type" "mve_move")
2853    (set_attr "length""8")])
2854
2855 ;;
2856 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
2857 ;;
2858 (define_insn "mve_vcmpeqq_m_<supf><mode>"
2859   [
2860    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2861         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2862                        (match_operand:MVE_2 2 "s_register_operand" "w")
2863                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2864          VCMPEQQ_M))
2865   ]
2866   "TARGET_HAVE_MVE"
2867   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
2868   [(set_attr "type" "mve_move")
2869    (set_attr "length""8")])
2870
2871 ;;
2872 ;; [vcmpgeq_m_n_s])
2873 ;;
2874 (define_insn "mve_vcmpgeq_m_n_s<mode>"
2875   [
2876    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2877         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2878                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2879                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2880          VCMPGEQ_M_N_S))
2881   ]
2882   "TARGET_HAVE_MVE"
2883   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
2884   [(set_attr "type" "mve_move")
2885    (set_attr "length""8")])
2886
2887 ;;
2888 ;; [vcmpgeq_m_s])
2889 ;;
2890 (define_insn "mve_vcmpgeq_m_s<mode>"
2891   [
2892    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2893         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2894                        (match_operand:MVE_2 2 "s_register_operand" "w")
2895                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2896          VCMPGEQ_M_S))
2897   ]
2898   "TARGET_HAVE_MVE"
2899   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
2900   [(set_attr "type" "mve_move")
2901    (set_attr "length""8")])
2902
2903 ;;
2904 ;; [vcmpgtq_m_n_s])
2905 ;;
2906 (define_insn "mve_vcmpgtq_m_n_s<mode>"
2907   [
2908    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2909         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2910                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2911                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2912          VCMPGTQ_M_N_S))
2913   ]
2914   "TARGET_HAVE_MVE"
2915   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
2916   [(set_attr "type" "mve_move")
2917    (set_attr "length""8")])
2918
2919 ;;
2920 ;; [vcmpgtq_m_s])
2921 ;;
2922 (define_insn "mve_vcmpgtq_m_s<mode>"
2923   [
2924    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2925         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2926                        (match_operand:MVE_2 2 "s_register_operand" "w")
2927                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2928          VCMPGTQ_M_S))
2929   ]
2930   "TARGET_HAVE_MVE"
2931   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
2932   [(set_attr "type" "mve_move")
2933    (set_attr "length""8")])
2934
2935 ;;
2936 ;; [vcmphiq_m_n_u])
2937 ;;
2938 (define_insn "mve_vcmphiq_m_n_u<mode>"
2939   [
2940    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2941         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2942                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2943                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2944          VCMPHIQ_M_N_U))
2945   ]
2946   "TARGET_HAVE_MVE"
2947   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
2948   [(set_attr "type" "mve_move")
2949    (set_attr "length""8")])
2950
2951 ;;
2952 ;; [vcmphiq_m_u])
2953 ;;
2954 (define_insn "mve_vcmphiq_m_u<mode>"
2955   [
2956    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2957         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2958                        (match_operand:MVE_2 2 "s_register_operand" "w")
2959                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2960          VCMPHIQ_M_U))
2961   ]
2962   "TARGET_HAVE_MVE"
2963   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
2964   [(set_attr "type" "mve_move")
2965    (set_attr "length""8")])
2966
2967 ;;
2968 ;; [vcmpleq_m_n_s])
2969 ;;
2970 (define_insn "mve_vcmpleq_m_n_s<mode>"
2971   [
2972    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2973         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2974                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2975                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2976          VCMPLEQ_M_N_S))
2977   ]
2978   "TARGET_HAVE_MVE"
2979   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
2980   [(set_attr "type" "mve_move")
2981    (set_attr "length""8")])
2982
2983 ;;
2984 ;; [vcmpleq_m_s])
2985 ;;
2986 (define_insn "mve_vcmpleq_m_s<mode>"
2987   [
2988    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2989         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2990                        (match_operand:MVE_2 2 "s_register_operand" "w")
2991                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2992          VCMPLEQ_M_S))
2993   ]
2994   "TARGET_HAVE_MVE"
2995   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
2996   [(set_attr "type" "mve_move")
2997    (set_attr "length""8")])
2998
2999 ;;
3000 ;; [vcmpltq_m_n_s])
3001 ;;
3002 (define_insn "mve_vcmpltq_m_n_s<mode>"
3003   [
3004    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3005         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3006                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3007                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3008          VCMPLTQ_M_N_S))
3009   ]
3010   "TARGET_HAVE_MVE"
3011   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3012   [(set_attr "type" "mve_move")
3013    (set_attr "length""8")])
3014
3015 ;;
3016 ;; [vcmpltq_m_s])
3017 ;;
3018 (define_insn "mve_vcmpltq_m_s<mode>"
3019   [
3020    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3021         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3022                        (match_operand:MVE_2 2 "s_register_operand" "w")
3023                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3024          VCMPLTQ_M_S))
3025   ]
3026   "TARGET_HAVE_MVE"
3027   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3028   [(set_attr "type" "mve_move")
3029    (set_attr "length""8")])
3030
3031 ;;
3032 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3033 ;;
3034 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3035   [
3036    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3037         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3038                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3039                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3040          VCMPNEQ_M_N))
3041   ]
3042   "TARGET_HAVE_MVE"
3043   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3044   [(set_attr "type" "mve_move")
3045    (set_attr "length""8")])
3046
3047 ;;
3048 ;; [vcmpneq_m_s, vcmpneq_m_u])
3049 ;;
3050 (define_insn "mve_vcmpneq_m_<supf><mode>"
3051   [
3052    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3053         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3054                        (match_operand:MVE_2 2 "s_register_operand" "w")
3055                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3056          VCMPNEQ_M))
3057   ]
3058   "TARGET_HAVE_MVE"
3059   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3060   [(set_attr "type" "mve_move")
3061    (set_attr "length""8")])
3062
3063 ;;
3064 ;; [vdupq_m_n_s, vdupq_m_n_u])
3065 ;;
3066 (define_insn "mve_vdupq_m_n_<supf><mode>"
3067   [
3068    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3069         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3070                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3071                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3072          VDUPQ_M_N))
3073   ]
3074   "TARGET_HAVE_MVE"
3075   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3076   [(set_attr "type" "mve_move")
3077    (set_attr "length""8")])
3078
3079 ;;
3080 ;; [vmaxaq_m_s])
3081 ;;
3082 (define_insn "mve_vmaxaq_m_s<mode>"
3083   [
3084    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3085         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3086                        (match_operand:MVE_2 2 "s_register_operand" "w")
3087                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3088          VMAXAQ_M_S))
3089   ]
3090   "TARGET_HAVE_MVE"
3091   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3092   [(set_attr "type" "mve_move")
3093    (set_attr "length""8")])
3094
3095 ;;
3096 ;; [vmaxavq_p_s])
3097 ;;
3098 (define_insn "mve_vmaxavq_p_s<mode>"
3099   [
3100    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3101         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3102                        (match_operand:MVE_2 2 "s_register_operand" "w")
3103                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3104          VMAXAVQ_P_S))
3105   ]
3106   "TARGET_HAVE_MVE"
3107   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3108   [(set_attr "type" "mve_move")
3109    (set_attr "length""8")])
3110
3111 ;;
3112 ;; [vmaxvq_p_u, vmaxvq_p_s])
3113 ;;
3114 (define_insn "mve_vmaxvq_p_<supf><mode>"
3115   [
3116    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3117         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3118                        (match_operand:MVE_2 2 "s_register_operand" "w")
3119                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3120          VMAXVQ_P))
3121   ]
3122   "TARGET_HAVE_MVE"
3123   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3124   [(set_attr "type" "mve_move")
3125    (set_attr "length""8")])
3126
3127 ;;
3128 ;; [vminaq_m_s])
3129 ;;
3130 (define_insn "mve_vminaq_m_s<mode>"
3131   [
3132    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3133         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3134                        (match_operand:MVE_2 2 "s_register_operand" "w")
3135                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3136          VMINAQ_M_S))
3137   ]
3138   "TARGET_HAVE_MVE"
3139   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3140   [(set_attr "type" "mve_move")
3141    (set_attr "length""8")])
3142
3143 ;;
3144 ;; [vminavq_p_s])
3145 ;;
3146 (define_insn "mve_vminavq_p_s<mode>"
3147   [
3148    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3149         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3150                        (match_operand:MVE_2 2 "s_register_operand" "w")
3151                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3152          VMINAVQ_P_S))
3153   ]
3154   "TARGET_HAVE_MVE"
3155   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3156   [(set_attr "type" "mve_move")
3157    (set_attr "length""8")])
3158
3159 ;;
3160 ;; [vminvq_p_s, vminvq_p_u])
3161 ;;
3162 (define_insn "mve_vminvq_p_<supf><mode>"
3163   [
3164    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3165         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3166                        (match_operand:MVE_2 2 "s_register_operand" "w")
3167                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3168          VMINVQ_P))
3169   ]
3170   "TARGET_HAVE_MVE"
3171   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3172   [(set_attr "type" "mve_move")
3173    (set_attr "length""8")])
3174
3175 ;;
3176 ;; [vmladavaq_u, vmladavaq_s])
3177 ;;
3178 (define_insn "mve_vmladavaq_<supf><mode>"
3179   [
3180    (set (match_operand:SI 0 "s_register_operand" "=Te")
3181         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3182                        (match_operand:MVE_2 2 "s_register_operand" "w")
3183                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3184          VMLADAVAQ))
3185   ]
3186   "TARGET_HAVE_MVE"
3187   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3188   [(set_attr "type" "mve_move")
3189 ])
3190
3191 ;;
3192 ;; [vmladavq_p_u, vmladavq_p_s])
3193 ;;
3194 (define_insn "mve_vmladavq_p_<supf><mode>"
3195   [
3196    (set (match_operand:SI 0 "s_register_operand" "=Te")
3197         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3198                        (match_operand:MVE_2 2 "s_register_operand" "w")
3199                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3200          VMLADAVQ_P))
3201   ]
3202   "TARGET_HAVE_MVE"
3203   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3204   [(set_attr "type" "mve_move")
3205    (set_attr "length""8")])
3206
3207 ;;
3208 ;; [vmladavxq_p_s])
3209 ;;
3210 (define_insn "mve_vmladavxq_p_s<mode>"
3211   [
3212    (set (match_operand:SI 0 "s_register_operand" "=Te")
3213         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3214                        (match_operand:MVE_2 2 "s_register_operand" "w")
3215                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3216          VMLADAVXQ_P_S))
3217   ]
3218   "TARGET_HAVE_MVE"
3219   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3220   [(set_attr "type" "mve_move")
3221    (set_attr "length""8")])
3222
3223 ;;
3224 ;; [vmlaq_n_u, vmlaq_n_s])
3225 ;;
3226 (define_insn "mve_vmlaq_n_<supf><mode>"
3227   [
3228    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3229         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3230                        (match_operand:MVE_2 2 "s_register_operand" "w")
3231                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3232          VMLAQ_N))
3233   ]
3234   "TARGET_HAVE_MVE"
3235   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3236   [(set_attr "type" "mve_move")
3237 ])
3238
3239 ;;
3240 ;; [vmlasq_n_u, vmlasq_n_s])
3241 ;;
3242 (define_insn "mve_vmlasq_n_<supf><mode>"
3243   [
3244    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3245         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3246                        (match_operand:MVE_2 2 "s_register_operand" "w")
3247                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3248          VMLASQ_N))
3249   ]
3250   "TARGET_HAVE_MVE"
3251   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3252   [(set_attr "type" "mve_move")
3253 ])
3254
3255 ;;
3256 ;; [vmlsdavq_p_s])
3257 ;;
3258 (define_insn "mve_vmlsdavq_p_s<mode>"
3259   [
3260    (set (match_operand:SI 0 "s_register_operand" "=Te")
3261         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3262                        (match_operand:MVE_2 2 "s_register_operand" "w")
3263                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3264          VMLSDAVQ_P_S))
3265   ]
3266   "TARGET_HAVE_MVE"
3267   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3268   [(set_attr "type" "mve_move")
3269    (set_attr "length""8")])
3270
3271 ;;
3272 ;; [vmlsdavxq_p_s])
3273 ;;
3274 (define_insn "mve_vmlsdavxq_p_s<mode>"
3275   [
3276    (set (match_operand:SI 0 "s_register_operand" "=Te")
3277         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3278                        (match_operand:MVE_2 2 "s_register_operand" "w")
3279                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3280          VMLSDAVXQ_P_S))
3281   ]
3282   "TARGET_HAVE_MVE"
3283   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3284   [(set_attr "type" "mve_move")
3285    (set_attr "length""8")])
3286
3287 ;;
3288 ;; [vmvnq_m_s, vmvnq_m_u])
3289 ;;
3290 (define_insn "mve_vmvnq_m_<supf><mode>"
3291   [
3292    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3293         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3294                        (match_operand:MVE_2 2 "s_register_operand" "w")
3295                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3296          VMVNQ_M))
3297   ]
3298   "TARGET_HAVE_MVE"
3299   "vpst\;vmvnt %q0, %q2"
3300   [(set_attr "type" "mve_move")
3301    (set_attr "length""8")])
3302
3303 ;;
3304 ;; [vnegq_m_s])
3305 ;;
3306 (define_insn "mve_vnegq_m_s<mode>"
3307   [
3308    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3309         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3310                        (match_operand:MVE_2 2 "s_register_operand" "w")
3311                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3312          VNEGQ_M_S))
3313   ]
3314   "TARGET_HAVE_MVE"
3315   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3316   [(set_attr "type" "mve_move")
3317    (set_attr "length""8")])
3318
3319 ;;
3320 ;; [vpselq_u, vpselq_s])
3321 ;;
3322 (define_insn "@mve_vpselq_<supf><mode>"
3323   [
3324    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3325         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3326                        (match_operand:MVE_1 2 "s_register_operand" "w")
3327                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3328          VPSELQ))
3329   ]
3330   "TARGET_HAVE_MVE"
3331   "vpsel %q0, %q1, %q2"
3332   [(set_attr "type" "mve_move")
3333 ])
3334
3335 ;;
3336 ;; [vqabsq_m_s])
3337 ;;
3338 (define_insn "mve_vqabsq_m_s<mode>"
3339   [
3340    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3341         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3342                        (match_operand:MVE_2 2 "s_register_operand" "w")
3343                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3344          VQABSQ_M_S))
3345   ]
3346   "TARGET_HAVE_MVE"
3347   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3348   [(set_attr "type" "mve_move")
3349    (set_attr "length""8")])
3350
3351 ;;
3352 ;; [vqdmlahq_n_s])
3353 ;;
3354 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3355   [
3356    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3357         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3358                        (match_operand:MVE_2 2 "s_register_operand" "w")
3359                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3360          VQDMLAHQ_N))
3361   ]
3362   "TARGET_HAVE_MVE"
3363   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3364   [(set_attr "type" "mve_move")
3365 ])
3366
3367 ;;
3368 ;; [vqdmlashq_n_s])
3369 ;;
3370 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3371   [
3372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3374                        (match_operand:MVE_2 2 "s_register_operand" "w")
3375                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3376          VQDMLASHQ_N))
3377   ]
3378   "TARGET_HAVE_MVE"
3379   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3380   [(set_attr "type" "mve_move")
3381 ])
3382
3383 ;;
3384 ;; [vqnegq_m_s])
3385 ;;
3386 (define_insn "mve_vqnegq_m_s<mode>"
3387   [
3388    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3389         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3390                        (match_operand:MVE_2 2 "s_register_operand" "w")
3391                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3392          VQNEGQ_M_S))
3393   ]
3394   "TARGET_HAVE_MVE"
3395   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3396   [(set_attr "type" "mve_move")
3397    (set_attr "length""8")])
3398
3399 ;;
3400 ;; [vqrdmladhq_s])
3401 ;;
3402 (define_insn "mve_vqrdmladhq_s<mode>"
3403   [
3404    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3405         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3406                        (match_operand:MVE_2 2 "s_register_operand" "w")
3407                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3408          VQRDMLADHQ_S))
3409   ]
3410   "TARGET_HAVE_MVE"
3411   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3412   [(set_attr "type" "mve_move")
3413 ])
3414
3415 ;;
3416 ;; [vqrdmladhxq_s])
3417 ;;
3418 (define_insn "mve_vqrdmladhxq_s<mode>"
3419   [
3420    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3421         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3422                        (match_operand:MVE_2 2 "s_register_operand" "w")
3423                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3424          VQRDMLADHXQ_S))
3425   ]
3426   "TARGET_HAVE_MVE"
3427   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3428   [(set_attr "type" "mve_move")
3429 ])
3430
3431 ;;
3432 ;; [vqrdmlahq_n_s])
3433 ;;
3434 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3435   [
3436    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3437         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3438                        (match_operand:MVE_2 2 "s_register_operand" "w")
3439                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3440          VQRDMLAHQ_N))
3441   ]
3442   "TARGET_HAVE_MVE"
3443   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3444   [(set_attr "type" "mve_move")
3445 ])
3446
3447 ;;
3448 ;; [vqrdmlashq_n_s])
3449 ;;
3450 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3451   [
3452    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3453         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3454                        (match_operand:MVE_2 2 "s_register_operand" "w")
3455                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3456          VQRDMLASHQ_N))
3457   ]
3458   "TARGET_HAVE_MVE"
3459   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3460   [(set_attr "type" "mve_move")
3461 ])
3462
3463 ;;
3464 ;; [vqrdmlsdhq_s])
3465 ;;
3466 (define_insn "mve_vqrdmlsdhq_s<mode>"
3467   [
3468    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3469         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3470                        (match_operand:MVE_2 2 "s_register_operand" "w")
3471                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3472          VQRDMLSDHQ_S))
3473   ]
3474   "TARGET_HAVE_MVE"
3475   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3476   [(set_attr "type" "mve_move")
3477 ])
3478
3479 ;;
3480 ;; [vqrdmlsdhxq_s])
3481 ;;
3482 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3483   [
3484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3486                        (match_operand:MVE_2 2 "s_register_operand" "w")
3487                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3488          VQRDMLSDHXQ_S))
3489   ]
3490   "TARGET_HAVE_MVE"
3491   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3492   [(set_attr "type" "mve_move")
3493 ])
3494
3495 ;;
3496 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3497 ;;
3498 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3499   [
3500    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3501         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3502                        (match_operand:SI 2 "s_register_operand" "r")
3503                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3504          VQRSHLQ_M_N))
3505   ]
3506   "TARGET_HAVE_MVE"
3507   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3508   [(set_attr "type" "mve_move")
3509    (set_attr "length""8")])
3510
3511 ;;
3512 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3513 ;;
3514 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3515   [
3516    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3517         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3518                        (match_operand:SI 2 "s_register_operand" "r")
3519                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3520          VQSHLQ_M_R))
3521   ]
3522   "TARGET_HAVE_MVE"
3523   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3524   [(set_attr "type" "mve_move")
3525    (set_attr "length""8")])
3526
3527 ;;
3528 ;; [vrev64q_m_u, vrev64q_m_s])
3529 ;;
3530 (define_insn "mve_vrev64q_m_<supf><mode>"
3531   [
3532    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3533         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3534                        (match_operand:MVE_2 2 "s_register_operand" "w")
3535                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3536          VREV64Q_M))
3537   ]
3538   "TARGET_HAVE_MVE"
3539   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3540   [(set_attr "type" "mve_move")
3541    (set_attr "length""8")])
3542
3543 ;;
3544 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3545 ;;
3546 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3547   [
3548    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3549         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3550                        (match_operand:SI 2 "s_register_operand" "r")
3551                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3552          VRSHLQ_M_N))
3553   ]
3554   "TARGET_HAVE_MVE"
3555   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3556   [(set_attr "type" "mve_move")
3557    (set_attr "length""8")])
3558
3559 ;;
3560 ;; [vshlq_m_r_u, vshlq_m_r_s])
3561 ;;
3562 (define_insn "mve_vshlq_m_r_<supf><mode>"
3563   [
3564    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3565         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3566                        (match_operand:SI 2 "s_register_operand" "r")
3567                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3568          VSHLQ_M_R))
3569   ]
3570   "TARGET_HAVE_MVE"
3571   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3572   [(set_attr "type" "mve_move")
3573    (set_attr "length""8")])
3574
3575 ;;
3576 ;; [vsliq_n_u, vsliq_n_s])
3577 ;;
3578 (define_insn "mve_vsliq_n_<supf><mode>"
3579   [
3580    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3581         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3582                        (match_operand:MVE_2 2 "s_register_operand" "w")
3583                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3584          VSLIQ_N))
3585   ]
3586   "TARGET_HAVE_MVE"
3587   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3588   [(set_attr "type" "mve_move")
3589 ])
3590
3591 ;;
3592 ;; [vsriq_n_u, vsriq_n_s])
3593 ;;
3594 (define_insn "mve_vsriq_n_<supf><mode>"
3595   [
3596    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3597         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3598                        (match_operand:MVE_2 2 "s_register_operand" "w")
3599                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3600          VSRIQ_N))
3601   ]
3602   "TARGET_HAVE_MVE"
3603   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3604   [(set_attr "type" "mve_move")
3605 ])
3606
3607 ;;
3608 ;; [vqdmlsdhxq_s])
3609 ;;
3610 (define_insn "mve_vqdmlsdhxq_s<mode>"
3611   [
3612    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3613         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3614                        (match_operand:MVE_2 2 "s_register_operand" "w")
3615                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3616          VQDMLSDHXQ_S))
3617   ]
3618   "TARGET_HAVE_MVE"
3619   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3620   [(set_attr "type" "mve_move")
3621 ])
3622
3623 ;;
3624 ;; [vqdmlsdhq_s])
3625 ;;
3626 (define_insn "mve_vqdmlsdhq_s<mode>"
3627   [
3628    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3629         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3630                        (match_operand:MVE_2 2 "s_register_operand" "w")
3631                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3632          VQDMLSDHQ_S))
3633   ]
3634   "TARGET_HAVE_MVE"
3635   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3636   [(set_attr "type" "mve_move")
3637 ])
3638
3639 ;;
3640 ;; [vqdmladhxq_s])
3641 ;;
3642 (define_insn "mve_vqdmladhxq_s<mode>"
3643   [
3644    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3645         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3646                        (match_operand:MVE_2 2 "s_register_operand" "w")
3647                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3648          VQDMLADHXQ_S))
3649   ]
3650   "TARGET_HAVE_MVE"
3651   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3652   [(set_attr "type" "mve_move")
3653 ])
3654
3655 ;;
3656 ;; [vqdmladhq_s])
3657 ;;
3658 (define_insn "mve_vqdmladhq_s<mode>"
3659   [
3660    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3661         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3662                        (match_operand:MVE_2 2 "s_register_operand" "w")
3663                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3664          VQDMLADHQ_S))
3665   ]
3666   "TARGET_HAVE_MVE"
3667   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3668   [(set_attr "type" "mve_move")
3669 ])
3670
3671 ;;
3672 ;; [vmlsdavaxq_s])
3673 ;;
3674 (define_insn "mve_vmlsdavaxq_s<mode>"
3675   [
3676    (set (match_operand:SI 0 "s_register_operand" "=Te")
3677         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3678                     (match_operand:MVE_2 2 "s_register_operand" "w")
3679                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3680          VMLSDAVAXQ_S))
3681   ]
3682   "TARGET_HAVE_MVE"
3683   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3684   [(set_attr "type" "mve_move")
3685 ])
3686
3687 ;;
3688 ;; [vmlsdavaq_s])
3689 ;;
3690 (define_insn "mve_vmlsdavaq_s<mode>"
3691   [
3692    (set (match_operand:SI 0 "s_register_operand" "=Te")
3693         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3694                     (match_operand:MVE_2 2 "s_register_operand" "w")
3695                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3696          VMLSDAVAQ_S))
3697   ]
3698   "TARGET_HAVE_MVE"
3699   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
3700   [(set_attr "type" "mve_move")
3701 ])
3702
3703 ;;
3704 ;; [vmladavaxq_s])
3705 ;;
3706 (define_insn "mve_vmladavaxq_s<mode>"
3707   [
3708    (set (match_operand:SI 0 "s_register_operand" "=Te")
3709         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3710                     (match_operand:MVE_2 2 "s_register_operand" "w")
3711                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3712          VMLADAVAXQ_S))
3713   ]
3714   "TARGET_HAVE_MVE"
3715   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3716   [(set_attr "type" "mve_move")
3717 ])
3718 ;;
3719 ;; [vabsq_m_f])
3720 ;;
3721 (define_insn "mve_vabsq_m_f<mode>"
3722   [
3723    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3724         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3725                        (match_operand:MVE_0 2 "s_register_operand" "w")
3726                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3727          VABSQ_M_F))
3728   ]
3729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3730   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
3731   [(set_attr "type" "mve_move")
3732    (set_attr "length""8")])
3733
3734 ;;
3735 ;; [vaddlvaq_p_s vaddlvaq_p_u])
3736 ;;
3737 (define_insn "mve_vaddlvaq_p_<supf>v4si"
3738   [
3739    (set (match_operand:DI 0 "s_register_operand" "=r")
3740         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3741                        (match_operand:V4SI 2 "s_register_operand" "w")
3742                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3743          VADDLVAQ_P))
3744   ]
3745   "TARGET_HAVE_MVE"
3746   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
3747   [(set_attr "type" "mve_move")
3748    (set_attr "length""8")])
3749 ;;
3750 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
3751 ;;
3752 (define_insn "mve_vcmlaq<mve_rot><mode>"
3753   [
3754    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
3755         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
3756                     (unspec:MVE_0
3757                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
3758                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
3759                      VCMLA)))
3760   ]
3761   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3762   "@
3763    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
3764    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
3765   [(set_attr "type" "mve_move")
3766 ])
3767
3768 ;;
3769 ;; [vcmpeqq_m_n_f])
3770 ;;
3771 (define_insn "mve_vcmpeqq_m_n_f<mode>"
3772   [
3773    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3774         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3775                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3776                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3777          VCMPEQQ_M_N_F))
3778   ]
3779   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3780   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
3781   [(set_attr "type" "mve_move")
3782    (set_attr "length""8")])
3783
3784 ;;
3785 ;; [vcmpgeq_m_f])
3786 ;;
3787 (define_insn "mve_vcmpgeq_m_f<mode>"
3788   [
3789    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3790         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3791                        (match_operand:MVE_0 2 "s_register_operand" "w")
3792                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3793          VCMPGEQ_M_F))
3794   ]
3795   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3796   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
3797   [(set_attr "type" "mve_move")
3798    (set_attr "length""8")])
3799
3800 ;;
3801 ;; [vcmpgeq_m_n_f])
3802 ;;
3803 (define_insn "mve_vcmpgeq_m_n_f<mode>"
3804   [
3805    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3806         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3807                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3808                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3809          VCMPGEQ_M_N_F))
3810   ]
3811   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3812   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
3813   [(set_attr "type" "mve_move")
3814    (set_attr "length""8")])
3815
3816 ;;
3817 ;; [vcmpgtq_m_f])
3818 ;;
3819 (define_insn "mve_vcmpgtq_m_f<mode>"
3820   [
3821    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3822         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3823                        (match_operand:MVE_0 2 "s_register_operand" "w")
3824                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3825          VCMPGTQ_M_F))
3826   ]
3827   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3828   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
3829   [(set_attr "type" "mve_move")
3830    (set_attr "length""8")])
3831
3832 ;;
3833 ;; [vcmpgtq_m_n_f])
3834 ;;
3835 (define_insn "mve_vcmpgtq_m_n_f<mode>"
3836   [
3837    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3838         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3839                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3840                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3841          VCMPGTQ_M_N_F))
3842   ]
3843   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3844   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
3845   [(set_attr "type" "mve_move")
3846    (set_attr "length""8")])
3847
3848 ;;
3849 ;; [vcmpleq_m_f])
3850 ;;
3851 (define_insn "mve_vcmpleq_m_f<mode>"
3852   [
3853    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3854         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3855                        (match_operand:MVE_0 2 "s_register_operand" "w")
3856                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3857          VCMPLEQ_M_F))
3858   ]
3859   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3860   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
3861   [(set_attr "type" "mve_move")
3862    (set_attr "length""8")])
3863
3864 ;;
3865 ;; [vcmpleq_m_n_f])
3866 ;;
3867 (define_insn "mve_vcmpleq_m_n_f<mode>"
3868   [
3869    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3870         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3871                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3872                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3873          VCMPLEQ_M_N_F))
3874   ]
3875   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3876   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
3877   [(set_attr "type" "mve_move")
3878    (set_attr "length""8")])
3879
3880 ;;
3881 ;; [vcmpltq_m_f])
3882 ;;
3883 (define_insn "mve_vcmpltq_m_f<mode>"
3884   [
3885    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3886         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3887                        (match_operand:MVE_0 2 "s_register_operand" "w")
3888                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3889          VCMPLTQ_M_F))
3890   ]
3891   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3892   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
3893   [(set_attr "type" "mve_move")
3894    (set_attr "length""8")])
3895
3896 ;;
3897 ;; [vcmpltq_m_n_f])
3898 ;;
3899 (define_insn "mve_vcmpltq_m_n_f<mode>"
3900   [
3901    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3902         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3903                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3904                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3905          VCMPLTQ_M_N_F))
3906   ]
3907   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3908   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
3909   [(set_attr "type" "mve_move")
3910    (set_attr "length""8")])
3911
3912 ;;
3913 ;; [vcmpneq_m_f])
3914 ;;
3915 (define_insn "mve_vcmpneq_m_f<mode>"
3916   [
3917    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3918         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3919                        (match_operand:MVE_0 2 "s_register_operand" "w")
3920                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3921          VCMPNEQ_M_F))
3922   ]
3923   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3924   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
3925   [(set_attr "type" "mve_move")
3926    (set_attr "length""8")])
3927
3928 ;;
3929 ;; [vcmpneq_m_n_f])
3930 ;;
3931 (define_insn "mve_vcmpneq_m_n_f<mode>"
3932   [
3933    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3934         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3935                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3936                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3937          VCMPNEQ_M_N_F))
3938   ]
3939   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3940   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
3941   [(set_attr "type" "mve_move")
3942    (set_attr "length""8")])
3943
3944 ;;
3945 ;; [vcvtbq_m_f16_f32])
3946 ;;
3947 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
3948   [
3949    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3950         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3951                        (match_operand:V4SF 2 "s_register_operand" "w")
3952                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3953          VCVTBQ_M_F16_F32))
3954   ]
3955   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3956   "vpst\;vcvtbt.f16.f32 %q0, %q2"
3957   [(set_attr "type" "mve_move")
3958    (set_attr "length""8")])
3959
3960 ;;
3961 ;; [vcvtbq_m_f32_f16])
3962 ;;
3963 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
3964   [
3965    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3966         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3967                        (match_operand:V8HF 2 "s_register_operand" "w")
3968                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3969          VCVTBQ_M_F32_F16))
3970   ]
3971   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3972   "vpst\;vcvtbt.f32.f16 %q0, %q2"
3973   [(set_attr "type" "mve_move")
3974    (set_attr "length""8")])
3975
3976 ;;
3977 ;; [vcvttq_m_f16_f32])
3978 ;;
3979 (define_insn "mve_vcvttq_m_f16_f32v8hf"
3980   [
3981    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3982         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3983                        (match_operand:V4SF 2 "s_register_operand" "w")
3984                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3985          VCVTTQ_M_F16_F32))
3986   ]
3987   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3988   "vpst\;vcvttt.f16.f32 %q0, %q2"
3989   [(set_attr "type" "mve_move")
3990    (set_attr "length""8")])
3991
3992 ;;
3993 ;; [vcvttq_m_f32_f16])
3994 ;;
3995 (define_insn "mve_vcvttq_m_f32_f16v4sf"
3996   [
3997    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3998         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3999                        (match_operand:V8HF 2 "s_register_operand" "w")
4000                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4001          VCVTTQ_M_F32_F16))
4002   ]
4003   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4004   "vpst\;vcvttt.f32.f16 %q0, %q2"
4005   [(set_attr "type" "mve_move")
4006    (set_attr "length""8")])
4007
4008 ;;
4009 ;; [vdupq_m_n_f])
4010 ;;
4011 (define_insn "mve_vdupq_m_n_f<mode>"
4012   [
4013    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4014         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4015                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4016                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4017          VDUPQ_M_N_F))
4018   ]
4019   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4020   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4021   [(set_attr "type" "mve_move")
4022    (set_attr "length""8")])
4023
4024 ;;
4025 ;; [vfmaq_f])
4026 ;;
4027 (define_insn "mve_vfmaq_f<mode>"
4028   [
4029    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4030         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4031                        (match_operand:MVE_0 2 "s_register_operand" "w")
4032                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4033          VFMAQ_F))
4034   ]
4035   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4036   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4037   [(set_attr "type" "mve_move")
4038 ])
4039
4040 ;;
4041 ;; [vfmaq_n_f])
4042 ;;
4043 (define_insn "mve_vfmaq_n_f<mode>"
4044   [
4045    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4046         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4047                        (match_operand:MVE_0 2 "s_register_operand" "w")
4048                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4049          VFMAQ_N_F))
4050   ]
4051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4052   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4053   [(set_attr "type" "mve_move")
4054 ])
4055
4056 ;;
4057 ;; [vfmasq_n_f])
4058 ;;
4059 (define_insn "mve_vfmasq_n_f<mode>"
4060   [
4061    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4062         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4063                        (match_operand:MVE_0 2 "s_register_operand" "w")
4064                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4065          VFMASQ_N_F))
4066   ]
4067   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4068   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4069   [(set_attr "type" "mve_move")
4070 ])
4071 ;;
4072 ;; [vfmsq_f])
4073 ;;
4074 (define_insn "mve_vfmsq_f<mode>"
4075   [
4076    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4077         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4078                        (match_operand:MVE_0 2 "s_register_operand" "w")
4079                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4080          VFMSQ_F))
4081   ]
4082   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4083   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4084   [(set_attr "type" "mve_move")
4085 ])
4086
4087 ;;
4088 ;; [vmaxnmaq_m_f])
4089 ;;
4090 (define_insn "mve_vmaxnmaq_m_f<mode>"
4091   [
4092    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4093         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4094                        (match_operand:MVE_0 2 "s_register_operand" "w")
4095                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4096          VMAXNMAQ_M_F))
4097   ]
4098   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4099   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4100   [(set_attr "type" "mve_move")
4101    (set_attr "length""8")])
4102 ;;
4103 ;; [vmaxnmavq_p_f])
4104 ;;
4105 (define_insn "mve_vmaxnmavq_p_f<mode>"
4106   [
4107    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4108         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4109                        (match_operand:MVE_0 2 "s_register_operand" "w")
4110                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4111          VMAXNMAVQ_P_F))
4112   ]
4113   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4114   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4115   [(set_attr "type" "mve_move")
4116    (set_attr "length""8")])
4117
4118 ;;
4119 ;; [vmaxnmvq_p_f])
4120 ;;
4121 (define_insn "mve_vmaxnmvq_p_f<mode>"
4122   [
4123    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4124         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4125                        (match_operand:MVE_0 2 "s_register_operand" "w")
4126                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4127          VMAXNMVQ_P_F))
4128   ]
4129   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4130   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4131   [(set_attr "type" "mve_move")
4132    (set_attr "length""8")])
4133 ;;
4134 ;; [vminnmaq_m_f])
4135 ;;
4136 (define_insn "mve_vminnmaq_m_f<mode>"
4137   [
4138    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4139         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4140                        (match_operand:MVE_0 2 "s_register_operand" "w")
4141                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4142          VMINNMAQ_M_F))
4143   ]
4144   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4145   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4146   [(set_attr "type" "mve_move")
4147    (set_attr "length""8")])
4148
4149 ;;
4150 ;; [vminnmavq_p_f])
4151 ;;
4152 (define_insn "mve_vminnmavq_p_f<mode>"
4153   [
4154    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4155         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4156                        (match_operand:MVE_0 2 "s_register_operand" "w")
4157                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4158          VMINNMAVQ_P_F))
4159   ]
4160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4161   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4162   [(set_attr "type" "mve_move")
4163    (set_attr "length""8")])
4164 ;;
4165 ;; [vminnmvq_p_f])
4166 ;;
4167 (define_insn "mve_vminnmvq_p_f<mode>"
4168   [
4169    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4170         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4171                        (match_operand:MVE_0 2 "s_register_operand" "w")
4172                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4173          VMINNMVQ_P_F))
4174   ]
4175   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4176   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4177   [(set_attr "type" "mve_move")
4178    (set_attr "length""8")])
4179
4180 ;;
4181 ;; [vmlaldavaq_s, vmlaldavaq_u])
4182 ;;
4183 (define_insn "mve_vmlaldavaq_<supf><mode>"
4184   [
4185    (set (match_operand:DI 0 "s_register_operand" "=r")
4186         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4187                        (match_operand:MVE_5 2 "s_register_operand" "w")
4188                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4189          VMLALDAVAQ))
4190   ]
4191   "TARGET_HAVE_MVE"
4192   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4193   [(set_attr "type" "mve_move")
4194 ])
4195
4196 ;;
4197 ;; [vmlaldavaxq_s])
4198 ;;
4199 (define_insn "mve_vmlaldavaxq_s<mode>"
4200   [
4201    (set (match_operand:DI 0 "s_register_operand" "=r")
4202         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4203                        (match_operand:MVE_5 2 "s_register_operand" "w")
4204                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4205          VMLALDAVAXQ_S))
4206   ]
4207   "TARGET_HAVE_MVE"
4208   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4209   [(set_attr "type" "mve_move")
4210 ])
4211
4212 ;;
4213 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4214 ;;
4215 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4216   [
4217    (set (match_operand:DI 0 "s_register_operand" "=r")
4218         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4219                        (match_operand:MVE_5 2 "s_register_operand" "w")
4220                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4221          VMLALDAVQ_P))
4222   ]
4223   "TARGET_HAVE_MVE"
4224   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4225   [(set_attr "type" "mve_move")
4226    (set_attr "length""8")])
4227
4228 ;;
4229 ;; [vmlaldavxq_p_s])
4230 ;;
4231 (define_insn "mve_vmlaldavxq_p_s<mode>"
4232   [
4233    (set (match_operand:DI 0 "s_register_operand" "=r")
4234         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4235                        (match_operand:MVE_5 2 "s_register_operand" "w")
4236                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4237          VMLALDAVXQ_P_S))
4238   ]
4239   "TARGET_HAVE_MVE"
4240   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4241   [(set_attr "type" "mve_move")
4242    (set_attr "length""8")])
4243 ;;
4244 ;; [vmlsldavaq_s])
4245 ;;
4246 (define_insn "mve_vmlsldavaq_s<mode>"
4247   [
4248    (set (match_operand:DI 0 "s_register_operand" "=r")
4249         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4250                        (match_operand:MVE_5 2 "s_register_operand" "w")
4251                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4252          VMLSLDAVAQ_S))
4253   ]
4254   "TARGET_HAVE_MVE"
4255   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4256   [(set_attr "type" "mve_move")
4257 ])
4258
4259 ;;
4260 ;; [vmlsldavaxq_s])
4261 ;;
4262 (define_insn "mve_vmlsldavaxq_s<mode>"
4263   [
4264    (set (match_operand:DI 0 "s_register_operand" "=r")
4265         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4266                        (match_operand:MVE_5 2 "s_register_operand" "w")
4267                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4268          VMLSLDAVAXQ_S))
4269   ]
4270   "TARGET_HAVE_MVE"
4271   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4272   [(set_attr "type" "mve_move")
4273 ])
4274
4275 ;;
4276 ;; [vmlsldavq_p_s])
4277 ;;
4278 (define_insn "mve_vmlsldavq_p_s<mode>"
4279   [
4280    (set (match_operand:DI 0 "s_register_operand" "=r")
4281         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4282                        (match_operand:MVE_5 2 "s_register_operand" "w")
4283                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4284          VMLSLDAVQ_P_S))
4285   ]
4286   "TARGET_HAVE_MVE"
4287   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4288   [(set_attr "type" "mve_move")
4289    (set_attr "length""8")])
4290
4291 ;;
4292 ;; [vmlsldavxq_p_s])
4293 ;;
4294 (define_insn "mve_vmlsldavxq_p_s<mode>"
4295   [
4296    (set (match_operand:DI 0 "s_register_operand" "=r")
4297         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4298                        (match_operand:MVE_5 2 "s_register_operand" "w")
4299                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4300          VMLSLDAVXQ_P_S))
4301   ]
4302   "TARGET_HAVE_MVE"
4303   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4304   [(set_attr "type" "mve_move")
4305    (set_attr "length""8")])
4306 ;;
4307 ;; [vmovlbq_m_u, vmovlbq_m_s])
4308 ;;
4309 (define_insn "mve_vmovlbq_m_<supf><mode>"
4310   [
4311    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4312         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4313                        (match_operand:MVE_3 2 "s_register_operand" "w")
4314                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4315          VMOVLBQ_M))
4316   ]
4317   "TARGET_HAVE_MVE"
4318   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4319   [(set_attr "type" "mve_move")
4320    (set_attr "length""8")])
4321 ;;
4322 ;; [vmovltq_m_u, vmovltq_m_s])
4323 ;;
4324 (define_insn "mve_vmovltq_m_<supf><mode>"
4325   [
4326    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4327         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4328                        (match_operand:MVE_3 2 "s_register_operand" "w")
4329                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4330          VMOVLTQ_M))
4331   ]
4332   "TARGET_HAVE_MVE"
4333   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4334   [(set_attr "type" "mve_move")
4335    (set_attr "length""8")])
4336 ;;
4337 ;; [vmovnbq_m_u, vmovnbq_m_s])
4338 ;;
4339 (define_insn "mve_vmovnbq_m_<supf><mode>"
4340   [
4341    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4342         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4343                        (match_operand:MVE_5 2 "s_register_operand" "w")
4344                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4345          VMOVNBQ_M))
4346   ]
4347   "TARGET_HAVE_MVE"
4348   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4349   [(set_attr "type" "mve_move")
4350    (set_attr "length""8")])
4351
4352 ;;
4353 ;; [vmovntq_m_u, vmovntq_m_s])
4354 ;;
4355 (define_insn "mve_vmovntq_m_<supf><mode>"
4356   [
4357    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4358         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4359                        (match_operand:MVE_5 2 "s_register_operand" "w")
4360                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4361          VMOVNTQ_M))
4362   ]
4363   "TARGET_HAVE_MVE"
4364   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4365   [(set_attr "type" "mve_move")
4366    (set_attr "length""8")])
4367
4368 ;;
4369 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4370 ;;
4371 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4372   [
4373    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4374         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4375                        (match_operand:SI 2 "immediate_operand" "i")
4376                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4377          VMVNQ_M_N))
4378   ]
4379   "TARGET_HAVE_MVE"
4380   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4381   [(set_attr "type" "mve_move")
4382    (set_attr "length""8")])
4383 ;;
4384 ;; [vnegq_m_f])
4385 ;;
4386 (define_insn "mve_vnegq_m_f<mode>"
4387   [
4388    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4389         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4390                        (match_operand:MVE_0 2 "s_register_operand" "w")
4391                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4392          VNEGQ_M_F))
4393   ]
4394   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4395   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4396   [(set_attr "type" "mve_move")
4397    (set_attr "length""8")])
4398
4399 ;;
4400 ;; [vorrq_m_n_s, vorrq_m_n_u])
4401 ;;
4402 (define_insn "mve_vorrq_m_n_<supf><mode>"
4403   [
4404    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4405         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4406                        (match_operand:SI 2 "immediate_operand" "i")
4407                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4408          VORRQ_M_N))
4409   ]
4410   "TARGET_HAVE_MVE"
4411   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4412   [(set_attr "type" "mve_move")
4413    (set_attr "length""8")])
4414 ;;
4415 ;; [vpselq_f])
4416 ;;
4417 (define_insn "@mve_vpselq_f<mode>"
4418   [
4419    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4420         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4421                        (match_operand:MVE_0 2 "s_register_operand" "w")
4422                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4423          VPSELQ_F))
4424   ]
4425   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4426   "vpsel %q0, %q1, %q2"
4427   [(set_attr "type" "mve_move")
4428 ])
4429
4430 ;;
4431 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4432 ;;
4433 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4434   [
4435    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4436         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4437                        (match_operand:MVE_5 2 "s_register_operand" "w")
4438                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4439          VQMOVNBQ_M))
4440   ]
4441   "TARGET_HAVE_MVE"
4442   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4443   [(set_attr "type" "mve_move")
4444    (set_attr "length""8")])
4445
4446 ;;
4447 ;; [vqmovntq_m_u, vqmovntq_m_s])
4448 ;;
4449 (define_insn "mve_vqmovntq_m_<supf><mode>"
4450   [
4451    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4452         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4453                        (match_operand:MVE_5 2 "s_register_operand" "w")
4454                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4455          VQMOVNTQ_M))
4456   ]
4457   "TARGET_HAVE_MVE"
4458   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4459   [(set_attr "type" "mve_move")
4460    (set_attr "length""8")])
4461
4462 ;;
4463 ;; [vqmovunbq_m_s])
4464 ;;
4465 (define_insn "mve_vqmovunbq_m_s<mode>"
4466   [
4467    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4468         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4469                        (match_operand:MVE_5 2 "s_register_operand" "w")
4470                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4471          VQMOVUNBQ_M_S))
4472   ]
4473   "TARGET_HAVE_MVE"
4474   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4475   [(set_attr "type" "mve_move")
4476    (set_attr "length""8")])
4477
4478 ;;
4479 ;; [vqmovuntq_m_s])
4480 ;;
4481 (define_insn "mve_vqmovuntq_m_s<mode>"
4482   [
4483    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4484         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4485                        (match_operand:MVE_5 2 "s_register_operand" "w")
4486                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4487          VQMOVUNTQ_M_S))
4488   ]
4489   "TARGET_HAVE_MVE"
4490   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4491   [(set_attr "type" "mve_move")
4492    (set_attr "length""8")])
4493
4494 ;;
4495 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4496 ;;
4497 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4498   [
4499    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4500         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4501                        (match_operand:MVE_5 2 "s_register_operand" "w")
4502                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4503          VQRSHRNTQ_N))
4504   ]
4505   "TARGET_HAVE_MVE"
4506   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4507   [(set_attr "type" "mve_move")
4508 ])
4509
4510 ;;
4511 ;; [vqrshruntq_n_s])
4512 ;;
4513 (define_insn "mve_vqrshruntq_n_s<mode>"
4514   [
4515    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4516         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4517                        (match_operand:MVE_5 2 "s_register_operand" "w")
4518                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4519          VQRSHRUNTQ_N_S))
4520   ]
4521   "TARGET_HAVE_MVE"
4522   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4523   [(set_attr "type" "mve_move")
4524 ])
4525
4526 ;;
4527 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4528 ;;
4529 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4530   [
4531    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4532         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4533                        (match_operand:MVE_5 2 "s_register_operand" "w")
4534                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4535          VQSHRNBQ_N))
4536   ]
4537   "TARGET_HAVE_MVE"
4538   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4539   [(set_attr "type" "mve_move")
4540 ])
4541
4542 ;;
4543 ;; [vqshrntq_n_u, vqshrntq_n_s])
4544 ;;
4545 (define_insn "mve_vqshrntq_n_<supf><mode>"
4546   [
4547    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4548         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4549                        (match_operand:MVE_5 2 "s_register_operand" "w")
4550                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4551          VQSHRNTQ_N))
4552   ]
4553   "TARGET_HAVE_MVE"
4554   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4555   [(set_attr "type" "mve_move")
4556 ])
4557
4558 ;;
4559 ;; [vqshrunbq_n_s])
4560 ;;
4561 (define_insn "mve_vqshrunbq_n_s<mode>"
4562   [
4563    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4564         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4565                        (match_operand:MVE_5 2 "s_register_operand" "w")
4566                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4567          VQSHRUNBQ_N_S))
4568   ]
4569   "TARGET_HAVE_MVE"
4570   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4571   [(set_attr "type" "mve_move")
4572 ])
4573
4574 ;;
4575 ;; [vqshruntq_n_s])
4576 ;;
4577 (define_insn "mve_vqshruntq_n_s<mode>"
4578   [
4579    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4580         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4581                        (match_operand:MVE_5 2 "s_register_operand" "w")
4582                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4583          VQSHRUNTQ_N_S))
4584   ]
4585   "TARGET_HAVE_MVE"
4586   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4587   [(set_attr "type" "mve_move")
4588 ])
4589
4590 ;;
4591 ;; [vrev32q_m_f])
4592 ;;
4593 (define_insn "mve_vrev32q_m_fv8hf"
4594   [
4595    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4596         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4597                        (match_operand:V8HF 2 "s_register_operand" "w")
4598                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4599          VREV32Q_M_F))
4600   ]
4601   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4602   "vpst\;vrev32t.16 %q0, %q2"
4603   [(set_attr "type" "mve_move")
4604    (set_attr "length""8")])
4605
4606 ;;
4607 ;; [vrev32q_m_s, vrev32q_m_u])
4608 ;;
4609 (define_insn "mve_vrev32q_m_<supf><mode>"
4610   [
4611    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4612         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4613                        (match_operand:MVE_3 2 "s_register_operand" "w")
4614                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4615          VREV32Q_M))
4616   ]
4617   "TARGET_HAVE_MVE"
4618   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4619   [(set_attr "type" "mve_move")
4620    (set_attr "length""8")])
4621
4622 ;;
4623 ;; [vrev64q_m_f])
4624 ;;
4625 (define_insn "mve_vrev64q_m_f<mode>"
4626   [
4627    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4628         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4629                        (match_operand:MVE_0 2 "s_register_operand" "w")
4630                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4631          VREV64Q_M_F))
4632   ]
4633   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4634   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4635   [(set_attr "type" "mve_move")
4636    (set_attr "length""8")])
4637
4638 ;;
4639 ;; [vrmlaldavhaxq_s])
4640 ;;
4641 (define_insn "mve_vrmlaldavhaxq_sv4si"
4642   [
4643    (set (match_operand:DI 0 "s_register_operand" "=r")
4644         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4645                        (match_operand:V4SI 2 "s_register_operand" "w")
4646                        (match_operand:V4SI 3 "s_register_operand" "w")]
4647          VRMLALDAVHAXQ_S))
4648   ]
4649   "TARGET_HAVE_MVE"
4650   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
4651   [(set_attr "type" "mve_move")
4652 ])
4653
4654 ;;
4655 ;; [vrmlaldavhxq_p_s])
4656 ;;
4657 (define_insn "mve_vrmlaldavhxq_p_sv4si"
4658   [
4659    (set (match_operand:DI 0 "s_register_operand" "=r")
4660         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4661                        (match_operand:V4SI 2 "s_register_operand" "w")
4662                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4663          VRMLALDAVHXQ_P_S))
4664   ]
4665   "TARGET_HAVE_MVE"
4666   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
4667   [(set_attr "type" "mve_move")
4668    (set_attr "length""8")])
4669
4670 ;;
4671 ;; [vrmlsldavhaxq_s])
4672 ;;
4673 (define_insn "mve_vrmlsldavhaxq_sv4si"
4674   [
4675    (set (match_operand:DI 0 "s_register_operand" "=r")
4676         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4677                        (match_operand:V4SI 2 "s_register_operand" "w")
4678                        (match_operand:V4SI 3 "s_register_operand" "w")]
4679          VRMLSLDAVHAXQ_S))
4680   ]
4681   "TARGET_HAVE_MVE"
4682   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
4683   [(set_attr "type" "mve_move")
4684 ])
4685
4686 ;;
4687 ;; [vrmlsldavhq_p_s])
4688 ;;
4689 (define_insn "mve_vrmlsldavhq_p_sv4si"
4690   [
4691    (set (match_operand:DI 0 "s_register_operand" "=r")
4692         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4693                        (match_operand:V4SI 2 "s_register_operand" "w")
4694                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4695          VRMLSLDAVHQ_P_S))
4696   ]
4697   "TARGET_HAVE_MVE"
4698   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
4699   [(set_attr "type" "mve_move")
4700    (set_attr "length""8")])
4701
4702 ;;
4703 ;; [vrmlsldavhxq_p_s])
4704 ;;
4705 (define_insn "mve_vrmlsldavhxq_p_sv4si"
4706   [
4707    (set (match_operand:DI 0 "s_register_operand" "=r")
4708         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4709                        (match_operand:V4SI 2 "s_register_operand" "w")
4710                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4711          VRMLSLDAVHXQ_P_S))
4712   ]
4713   "TARGET_HAVE_MVE"
4714   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
4715   [(set_attr "type" "mve_move")
4716    (set_attr "length""8")])
4717
4718 ;;
4719 ;; [vrndaq_m_f])
4720 ;;
4721 (define_insn "mve_vrndaq_m_f<mode>"
4722   [
4723    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4724         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4725                        (match_operand:MVE_0 2 "s_register_operand" "w")
4726                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4727          VRNDAQ_M_F))
4728   ]
4729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4730   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
4731   [(set_attr "type" "mve_move")
4732    (set_attr "length""8")])
4733
4734 ;;
4735 ;; [vrndmq_m_f])
4736 ;;
4737 (define_insn "mve_vrndmq_m_f<mode>"
4738   [
4739    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4740         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4741                        (match_operand:MVE_0 2 "s_register_operand" "w")
4742                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4743          VRNDMQ_M_F))
4744   ]
4745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4746   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
4747   [(set_attr "type" "mve_move")
4748    (set_attr "length""8")])
4749
4750 ;;
4751 ;; [vrndnq_m_f])
4752 ;;
4753 (define_insn "mve_vrndnq_m_f<mode>"
4754   [
4755    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4756         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4757                        (match_operand:MVE_0 2 "s_register_operand" "w")
4758                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4759          VRNDNQ_M_F))
4760   ]
4761   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4762   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
4763   [(set_attr "type" "mve_move")
4764    (set_attr "length""8")])
4765
4766 ;;
4767 ;; [vrndpq_m_f])
4768 ;;
4769 (define_insn "mve_vrndpq_m_f<mode>"
4770   [
4771    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4772         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4773                        (match_operand:MVE_0 2 "s_register_operand" "w")
4774                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4775          VRNDPQ_M_F))
4776   ]
4777   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4778   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
4779   [(set_attr "type" "mve_move")
4780    (set_attr "length""8")])
4781
4782 ;;
4783 ;; [vrndxq_m_f])
4784 ;;
4785 (define_insn "mve_vrndxq_m_f<mode>"
4786   [
4787    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4788         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4789                        (match_operand:MVE_0 2 "s_register_operand" "w")
4790                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4791          VRNDXQ_M_F))
4792   ]
4793   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4794   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
4795   [(set_attr "type" "mve_move")
4796    (set_attr "length""8")])
4797
4798 ;;
4799 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
4800 ;;
4801 (define_insn "mve_vrshrnbq_n_<supf><mode>"
4802   [
4803    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4804         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4805                        (match_operand:MVE_5 2 "s_register_operand" "w")
4806                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4807          VRSHRNBQ_N))
4808   ]
4809   "TARGET_HAVE_MVE"
4810   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
4811   [(set_attr "type" "mve_move")
4812 ])
4813
4814 ;;
4815 ;; [vrshrntq_n_u, vrshrntq_n_s])
4816 ;;
4817 (define_insn "mve_vrshrntq_n_<supf><mode>"
4818   [
4819    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4820         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4821                        (match_operand:MVE_5 2 "s_register_operand" "w")
4822                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4823          VRSHRNTQ_N))
4824   ]
4825   "TARGET_HAVE_MVE"
4826   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
4827   [(set_attr "type" "mve_move")
4828 ])
4829
4830 ;;
4831 ;; [vshrnbq_n_u, vshrnbq_n_s])
4832 ;;
4833 (define_insn "mve_vshrnbq_n_<supf><mode>"
4834   [
4835    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4836         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4837                        (match_operand:MVE_5 2 "s_register_operand" "w")
4838                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4839          VSHRNBQ_N))
4840   ]
4841   "TARGET_HAVE_MVE"
4842   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
4843   [(set_attr "type" "mve_move")
4844 ])
4845
4846 ;;
4847 ;; [vshrntq_n_s, vshrntq_n_u])
4848 ;;
4849 (define_insn "mve_vshrntq_n_<supf><mode>"
4850   [
4851    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4852         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4853                                  (match_operand:MVE_5 2 "s_register_operand" "w")
4854                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4855          VSHRNTQ_N))
4856   ]
4857   "TARGET_HAVE_MVE"
4858   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
4859   [(set_attr "type" "mve_move")
4860 ])
4861
4862 ;;
4863 ;; [vcvtmq_m_s, vcvtmq_m_u])
4864 ;;
4865 (define_insn "mve_vcvtmq_m_<supf><mode>"
4866   [
4867    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4868         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4869                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4870                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4871          VCVTMQ_M))
4872   ]
4873   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4874   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4875   [(set_attr "type" "mve_move")
4876    (set_attr "length""8")])
4877
4878 ;;
4879 ;; [vcvtpq_m_u, vcvtpq_m_s])
4880 ;;
4881 (define_insn "mve_vcvtpq_m_<supf><mode>"
4882   [
4883    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4884         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4885                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4886                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4887          VCVTPQ_M))
4888   ]
4889   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4890   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4891   [(set_attr "type" "mve_move")
4892    (set_attr "length""8")])
4893
4894 ;;
4895 ;; [vcvtnq_m_s, vcvtnq_m_u])
4896 ;;
4897 (define_insn "mve_vcvtnq_m_<supf><mode>"
4898   [
4899    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4900         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4901                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4902                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4903          VCVTNQ_M))
4904   ]
4905   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4906   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4907   [(set_attr "type" "mve_move")
4908    (set_attr "length""8")])
4909
4910 ;;
4911 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
4912 ;;
4913 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
4914   [
4915    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4916         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4917                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4918                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
4919                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4920          VCVTQ_M_N_FROM_F))
4921   ]
4922   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4923   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
4924   [(set_attr "type" "mve_move")
4925    (set_attr "length""8")])
4926
4927 ;;
4928 ;; [vrev16q_m_u, vrev16q_m_s])
4929 ;;
4930 (define_insn "mve_vrev16q_m_<supf>v16qi"
4931   [
4932    (set (match_operand:V16QI 0 "s_register_operand" "=w")
4933         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
4934                        (match_operand:V16QI 2 "s_register_operand" "w")
4935                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4936          VREV16Q_M))
4937   ]
4938   "TARGET_HAVE_MVE"
4939   "vpst\;vrev16t.8 %q0, %q2"
4940   [(set_attr "type" "mve_move")
4941    (set_attr "length""8")])
4942
4943 ;;
4944 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
4945 ;;
4946 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
4947   [
4948    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4949         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4950                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4951                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4952          VCVTQ_M_FROM_F))
4953   ]
4954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4955   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4956   [(set_attr "type" "mve_move")
4957    (set_attr "length""8")])
4958
4959 ;;
4960 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
4961 ;;
4962 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
4963   [
4964    (set (match_operand:DI 0 "s_register_operand" "=r")
4965         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4966                     (match_operand:V4SI 2 "s_register_operand" "w")
4967                     (match_operand:HI 3 "vpr_register_operand" "Up")]
4968          VRMLALDAVHQ_P))
4969   ]
4970   "TARGET_HAVE_MVE"
4971   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
4972   [(set_attr "type" "mve_move")
4973    (set_attr "length""8")])
4974
4975 ;;
4976 ;; [vrmlsldavhaq_s])
4977 ;;
4978 (define_insn "mve_vrmlsldavhaq_sv4si"
4979   [
4980    (set (match_operand:DI 0 "s_register_operand" "=r")
4981         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4982                     (match_operand:V4SI 2 "s_register_operand" "w")
4983                     (match_operand:V4SI 3 "s_register_operand" "w")]
4984          VRMLSLDAVHAQ_S))
4985   ]
4986   "TARGET_HAVE_MVE"
4987   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
4988   [(set_attr "type" "mve_move")
4989 ])
4990
4991 ;;
4992 ;; [vabavq_p_s, vabavq_p_u])
4993 ;;
4994 (define_insn "mve_vabavq_p_<supf><mode>"
4995   [
4996    (set (match_operand:SI 0 "s_register_operand" "=r")
4997         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4998                     (match_operand:MVE_2 2 "s_register_operand" "w")
4999                     (match_operand:MVE_2 3 "s_register_operand" "w")
5000                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5001          VABAVQ_P))
5002   ]
5003   "TARGET_HAVE_MVE"
5004   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5005   [(set_attr "type" "mve_move")
5006 ])
5007
5008 ;;
5009 ;; [vqshluq_m_n_s])
5010 ;;
5011 (define_insn "mve_vqshluq_m_n_s<mode>"
5012   [
5013    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5014         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5015                        (match_operand:MVE_2 2 "s_register_operand" "w")
5016                        (match_operand:SI 3 "mve_imm_7" "Ra")
5017                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5018          VQSHLUQ_M_N_S))
5019   ]
5020   "TARGET_HAVE_MVE"
5021   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5022   [(set_attr "type" "mve_move")])
5023
5024 ;;
5025 ;; [vshlq_m_s, vshlq_m_u])
5026 ;;
5027 (define_insn "mve_vshlq_m_<supf><mode>"
5028   [
5029    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5030         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5031                        (match_operand:MVE_2 2 "s_register_operand" "w")
5032                        (match_operand:MVE_2 3 "s_register_operand" "w")
5033                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5034          VSHLQ_M))
5035   ]
5036   "TARGET_HAVE_MVE"
5037   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5038   [(set_attr "type" "mve_move")])
5039
5040 ;;
5041 ;; [vsriq_m_n_s, vsriq_m_n_u])
5042 ;;
5043 (define_insn "mve_vsriq_m_n_<supf><mode>"
5044   [
5045    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5046         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5047                        (match_operand:MVE_2 2 "s_register_operand" "w")
5048                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5049                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5050          VSRIQ_M_N))
5051   ]
5052   "TARGET_HAVE_MVE"
5053   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5054   [(set_attr "type" "mve_move")])
5055
5056 ;;
5057 ;; [vsubq_m_u, vsubq_m_s])
5058 ;;
5059 (define_insn "mve_vsubq_m_<supf><mode>"
5060   [
5061    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5062         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5063                        (match_operand:MVE_2 2 "s_register_operand" "w")
5064                        (match_operand:MVE_2 3 "s_register_operand" "w")
5065                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5066          VSUBQ_M))
5067   ]
5068   "TARGET_HAVE_MVE"
5069   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5070   [(set_attr "type" "mve_move")])
5071
5072 ;;
5073 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5074 ;;
5075 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5076   [
5077    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5078         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5079                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5080                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5081                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5082          VCVTQ_M_N_TO_F))
5083   ]
5084   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5085   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5086   [(set_attr "type" "mve_move")
5087    (set_attr "length""8")])
5088 ;;
5089 ;; [vabdq_m_s, vabdq_m_u])
5090 ;;
5091 (define_insn "mve_vabdq_m_<supf><mode>"
5092   [
5093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5095                        (match_operand:MVE_2 2 "s_register_operand" "w")
5096                        (match_operand:MVE_2 3 "s_register_operand" "w")
5097                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5098          VABDQ_M))
5099   ]
5100   "TARGET_HAVE_MVE"
5101   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5102   [(set_attr "type" "mve_move")
5103    (set_attr "length""8")])
5104
5105 ;;
5106 ;; [vaddq_m_n_s, vaddq_m_n_u])
5107 ;;
5108 (define_insn "mve_vaddq_m_n_<supf><mode>"
5109   [
5110    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5111         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5112                        (match_operand:MVE_2 2 "s_register_operand" "w")
5113                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5114                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5115          VADDQ_M_N))
5116   ]
5117   "TARGET_HAVE_MVE"
5118   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5119   [(set_attr "type" "mve_move")
5120    (set_attr "length""8")])
5121
5122 ;;
5123 ;; [vaddq_m_u, vaddq_m_s])
5124 ;;
5125 (define_insn "mve_vaddq_m_<supf><mode>"
5126   [
5127    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5128         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5129                        (match_operand:MVE_2 2 "s_register_operand" "w")
5130                        (match_operand:MVE_2 3 "s_register_operand" "w")
5131                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5132          VADDQ_M))
5133   ]
5134   "TARGET_HAVE_MVE"
5135   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5136   [(set_attr "type" "mve_move")
5137    (set_attr "length""8")])
5138
5139 ;;
5140 ;; [vandq_m_u, vandq_m_s])
5141 ;;
5142 (define_insn "mve_vandq_m_<supf><mode>"
5143   [
5144    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5145         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5146                        (match_operand:MVE_2 2 "s_register_operand" "w")
5147                        (match_operand:MVE_2 3 "s_register_operand" "w")
5148                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5149          VANDQ_M))
5150   ]
5151   "TARGET_HAVE_MVE"
5152   "vpst\;vandt %q0, %q2, %q3"
5153   [(set_attr "type" "mve_move")
5154    (set_attr "length""8")])
5155
5156 ;;
5157 ;; [vbicq_m_u, vbicq_m_s])
5158 ;;
5159 (define_insn "mve_vbicq_m_<supf><mode>"
5160   [
5161    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5162         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5163                        (match_operand:MVE_2 2 "s_register_operand" "w")
5164                        (match_operand:MVE_2 3 "s_register_operand" "w")
5165                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5166          VBICQ_M))
5167   ]
5168   "TARGET_HAVE_MVE"
5169   "vpst\;vbict %q0, %q2, %q3"
5170   [(set_attr "type" "mve_move")
5171    (set_attr "length""8")])
5172
5173 ;;
5174 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5175 ;;
5176 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5177   [
5178    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5179         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5180                        (match_operand:MVE_2 2 "s_register_operand" "w")
5181                        (match_operand:SI 3 "s_register_operand" "r")
5182                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5183          VBRSRQ_M_N))
5184   ]
5185   "TARGET_HAVE_MVE"
5186   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5187   [(set_attr "type" "mve_move")
5188    (set_attr "length""8")])
5189
5190 ;;
5191 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5192 ;;
5193 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5194   [
5195    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5196         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5197                        (match_operand:MVE_2 2 "s_register_operand" "w")
5198                        (match_operand:MVE_2 3 "s_register_operand" "w")
5199                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5200          VCADDQ_ROT270_M))
5201   ]
5202   "TARGET_HAVE_MVE"
5203   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5204   [(set_attr "type" "mve_move")
5205    (set_attr "length""8")])
5206
5207 ;;
5208 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5209 ;;
5210 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5211   [
5212    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5213         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5214                        (match_operand:MVE_2 2 "s_register_operand" "w")
5215                        (match_operand:MVE_2 3 "s_register_operand" "w")
5216                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5217          VCADDQ_ROT90_M))
5218   ]
5219   "TARGET_HAVE_MVE"
5220   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5221   [(set_attr "type" "mve_move")
5222    (set_attr "length""8")])
5223
5224 ;;
5225 ;; [veorq_m_s, veorq_m_u])
5226 ;;
5227 (define_insn "mve_veorq_m_<supf><mode>"
5228   [
5229    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5230         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5231                        (match_operand:MVE_2 2 "s_register_operand" "w")
5232                        (match_operand:MVE_2 3 "s_register_operand" "w")
5233                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5234          VEORQ_M))
5235   ]
5236   "TARGET_HAVE_MVE"
5237   "vpst\;veort %q0, %q2, %q3"
5238   [(set_attr "type" "mve_move")
5239    (set_attr "length""8")])
5240
5241 ;;
5242 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5243 ;;
5244 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5245   [
5246    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5247         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5248                        (match_operand:MVE_2 2 "s_register_operand" "w")
5249                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5250                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5251          VHADDQ_M_N))
5252   ]
5253   "TARGET_HAVE_MVE"
5254   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5255   [(set_attr "type" "mve_move")
5256    (set_attr "length""8")])
5257
5258 ;;
5259 ;; [vhaddq_m_s, vhaddq_m_u])
5260 ;;
5261 (define_insn "mve_vhaddq_m_<supf><mode>"
5262   [
5263    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5264         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5265                        (match_operand:MVE_2 2 "s_register_operand" "w")
5266                        (match_operand:MVE_2 3 "s_register_operand" "w")
5267                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5268          VHADDQ_M))
5269   ]
5270   "TARGET_HAVE_MVE"
5271   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5272   [(set_attr "type" "mve_move")
5273    (set_attr "length""8")])
5274
5275 ;;
5276 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5277 ;;
5278 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5279   [
5280    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5281         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5282                        (match_operand:MVE_2 2 "s_register_operand" "w")
5283                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5284                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5285          VHSUBQ_M_N))
5286   ]
5287   "TARGET_HAVE_MVE"
5288   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5289   [(set_attr "type" "mve_move")
5290    (set_attr "length""8")])
5291
5292 ;;
5293 ;; [vhsubq_m_s, vhsubq_m_u])
5294 ;;
5295 (define_insn "mve_vhsubq_m_<supf><mode>"
5296   [
5297    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5298         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5299                        (match_operand:MVE_2 2 "s_register_operand" "w")
5300                        (match_operand:MVE_2 3 "s_register_operand" "w")
5301                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5302          VHSUBQ_M))
5303   ]
5304   "TARGET_HAVE_MVE"
5305   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5306   [(set_attr "type" "mve_move")
5307    (set_attr "length""8")])
5308
5309 ;;
5310 ;; [vmaxq_m_s, vmaxq_m_u])
5311 ;;
5312 (define_insn "mve_vmaxq_m_<supf><mode>"
5313   [
5314    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5315         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5316                        (match_operand:MVE_2 2 "s_register_operand" "w")
5317                        (match_operand:MVE_2 3 "s_register_operand" "w")
5318                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5319          VMAXQ_M))
5320   ]
5321   "TARGET_HAVE_MVE"
5322   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5323   [(set_attr "type" "mve_move")
5324    (set_attr "length""8")])
5325
5326 ;;
5327 ;; [vminq_m_s, vminq_m_u])
5328 ;;
5329 (define_insn "mve_vminq_m_<supf><mode>"
5330   [
5331    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5332         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5333                        (match_operand:MVE_2 2 "s_register_operand" "w")
5334                        (match_operand:MVE_2 3 "s_register_operand" "w")
5335                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5336          VMINQ_M))
5337   ]
5338   "TARGET_HAVE_MVE"
5339   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5340   [(set_attr "type" "mve_move")
5341    (set_attr "length""8")])
5342
5343 ;;
5344 ;; [vmladavaq_p_u, vmladavaq_p_s])
5345 ;;
5346 (define_insn "mve_vmladavaq_p_<supf><mode>"
5347   [
5348    (set (match_operand:SI 0 "s_register_operand" "=Te")
5349         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5350                     (match_operand:MVE_2 2 "s_register_operand" "w")
5351                     (match_operand:MVE_2 3 "s_register_operand" "w")
5352                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5353          VMLADAVAQ_P))
5354   ]
5355   "TARGET_HAVE_MVE"
5356   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5357   [(set_attr "type" "mve_move")
5358    (set_attr "length""8")])
5359
5360 ;;
5361 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5362 ;;
5363 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5364   [
5365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5367                        (match_operand:MVE_2 2 "s_register_operand" "w")
5368                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5369                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5370          VMLAQ_M_N))
5371   ]
5372   "TARGET_HAVE_MVE"
5373   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5374   [(set_attr "type" "mve_move")
5375    (set_attr "length""8")])
5376
5377 ;;
5378 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5379 ;;
5380 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5381   [
5382    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5383         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5384                        (match_operand:MVE_2 2 "s_register_operand" "w")
5385                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5386                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5387          VMLASQ_M_N))
5388   ]
5389   "TARGET_HAVE_MVE"
5390   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5391   [(set_attr "type" "mve_move")
5392    (set_attr "length""8")])
5393
5394 ;;
5395 ;; [vmulhq_m_s, vmulhq_m_u])
5396 ;;
5397 (define_insn "mve_vmulhq_m_<supf><mode>"
5398   [
5399    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5400         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5401                        (match_operand:MVE_2 2 "s_register_operand" "w")
5402                        (match_operand:MVE_2 3 "s_register_operand" "w")
5403                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5404          VMULHQ_M))
5405   ]
5406   "TARGET_HAVE_MVE"
5407   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5408   [(set_attr "type" "mve_move")
5409    (set_attr "length""8")])
5410
5411 ;;
5412 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5413 ;;
5414 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5415   [
5416    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5417         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5418                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5419                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5420                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5421          VMULLBQ_INT_M))
5422   ]
5423   "TARGET_HAVE_MVE"
5424   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5425   [(set_attr "type" "mve_move")
5426    (set_attr "length""8")])
5427
5428 ;;
5429 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5430 ;;
5431 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5432   [
5433    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5434         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5435                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5436                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5437                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5438          VMULLTQ_INT_M))
5439   ]
5440   "TARGET_HAVE_MVE"
5441   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5442   [(set_attr "type" "mve_move")
5443    (set_attr "length""8")])
5444
5445 ;;
5446 ;; [vmulq_m_n_u, vmulq_m_n_s])
5447 ;;
5448 (define_insn "mve_vmulq_m_n_<supf><mode>"
5449   [
5450    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5451         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5452                        (match_operand:MVE_2 2 "s_register_operand" "w")
5453                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5455          VMULQ_M_N))
5456   ]
5457   "TARGET_HAVE_MVE"
5458   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5459   [(set_attr "type" "mve_move")
5460    (set_attr "length""8")])
5461
5462 ;;
5463 ;; [vmulq_m_s, vmulq_m_u])
5464 ;;
5465 (define_insn "mve_vmulq_m_<supf><mode>"
5466   [
5467    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5468         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5469                        (match_operand:MVE_2 2 "s_register_operand" "w")
5470                        (match_operand:MVE_2 3 "s_register_operand" "w")
5471                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5472          VMULQ_M))
5473   ]
5474   "TARGET_HAVE_MVE"
5475   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5476   [(set_attr "type" "mve_move")
5477    (set_attr "length""8")])
5478
5479 ;;
5480 ;; [vornq_m_u, vornq_m_s])
5481 ;;
5482 (define_insn "mve_vornq_m_<supf><mode>"
5483   [
5484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5486                        (match_operand:MVE_2 2 "s_register_operand" "w")
5487                        (match_operand:MVE_2 3 "s_register_operand" "w")
5488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5489          VORNQ_M))
5490   ]
5491   "TARGET_HAVE_MVE"
5492   "vpst\;vornt %q0, %q2, %q3"
5493   [(set_attr "type" "mve_move")
5494    (set_attr "length""8")])
5495
5496 ;;
5497 ;; [vorrq_m_s, vorrq_m_u])
5498 ;;
5499 (define_insn "mve_vorrq_m_<supf><mode>"
5500   [
5501    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5502         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5503                        (match_operand:MVE_2 2 "s_register_operand" "w")
5504                        (match_operand:MVE_2 3 "s_register_operand" "w")
5505                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5506          VORRQ_M))
5507   ]
5508   "TARGET_HAVE_MVE"
5509   "vpst\;vorrt %q0, %q2, %q3"
5510   [(set_attr "type" "mve_move")
5511    (set_attr "length""8")])
5512
5513 ;;
5514 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5515 ;;
5516 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5517   [
5518    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5519         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5520                        (match_operand:MVE_2 2 "s_register_operand" "w")
5521                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5522                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5523          VQADDQ_M_N))
5524   ]
5525   "TARGET_HAVE_MVE"
5526   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5527   [(set_attr "type" "mve_move")
5528    (set_attr "length""8")])
5529
5530 ;;
5531 ;; [vqaddq_m_u, vqaddq_m_s])
5532 ;;
5533 (define_insn "mve_vqaddq_m_<supf><mode>"
5534   [
5535    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5536         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5537                        (match_operand:MVE_2 2 "s_register_operand" "w")
5538                        (match_operand:MVE_2 3 "s_register_operand" "w")
5539                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5540          VQADDQ_M))
5541   ]
5542   "TARGET_HAVE_MVE"
5543   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5544   [(set_attr "type" "mve_move")
5545    (set_attr "length""8")])
5546
5547 ;;
5548 ;; [vqdmlahq_m_n_s])
5549 ;;
5550 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5551   [
5552    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5553         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5554                        (match_operand:MVE_2 2 "s_register_operand" "w")
5555                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5556                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5557          VQDMLAHQ_M_N_S))
5558   ]
5559   "TARGET_HAVE_MVE"
5560   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5561   [(set_attr "type" "mve_move")
5562    (set_attr "length""8")])
5563
5564 ;;
5565 ;; [vqdmlashq_m_n_s])
5566 ;;
5567 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5568   [
5569    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5570         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5571                        (match_operand:MVE_2 2 "s_register_operand" "w")
5572                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5573                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5574          VQDMLASHQ_M_N_S))
5575   ]
5576   "TARGET_HAVE_MVE"
5577   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5578   [(set_attr "type" "mve_move")
5579    (set_attr "length""8")])
5580
5581 ;;
5582 ;; [vqrdmlahq_m_n_s])
5583 ;;
5584 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5585   [
5586    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5587         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5588                        (match_operand:MVE_2 2 "s_register_operand" "w")
5589                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5590                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5591          VQRDMLAHQ_M_N_S))
5592   ]
5593   "TARGET_HAVE_MVE"
5594   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5595   [(set_attr "type" "mve_move")
5596    (set_attr "length""8")])
5597
5598 ;;
5599 ;; [vqrdmlashq_m_n_s])
5600 ;;
5601 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5602   [
5603    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5604         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5605                        (match_operand:MVE_2 2 "s_register_operand" "w")
5606                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5607                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5608          VQRDMLASHQ_M_N_S))
5609   ]
5610   "TARGET_HAVE_MVE"
5611   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5612   [(set_attr "type" "mve_move")
5613    (set_attr "length""8")])
5614
5615 ;;
5616 ;; [vqrshlq_m_u, vqrshlq_m_s])
5617 ;;
5618 (define_insn "mve_vqrshlq_m_<supf><mode>"
5619   [
5620    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5621         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5622                        (match_operand:MVE_2 2 "s_register_operand" "w")
5623                        (match_operand:MVE_2 3 "s_register_operand" "w")
5624                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5625          VQRSHLQ_M))
5626   ]
5627   "TARGET_HAVE_MVE"
5628   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5629   [(set_attr "type" "mve_move")
5630    (set_attr "length""8")])
5631
5632 ;;
5633 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5634 ;;
5635 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5636   [
5637    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5638         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5639                        (match_operand:MVE_2 2 "s_register_operand" "w")
5640                        (match_operand:SI 3 "immediate_operand" "i")
5641                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5642          VQSHLQ_M_N))
5643   ]
5644   "TARGET_HAVE_MVE"
5645   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5646   [(set_attr "type" "mve_move")
5647    (set_attr "length""8")])
5648
5649 ;;
5650 ;; [vqshlq_m_u, vqshlq_m_s])
5651 ;;
5652 (define_insn "mve_vqshlq_m_<supf><mode>"
5653   [
5654    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5655         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5656                        (match_operand:MVE_2 2 "s_register_operand" "w")
5657                        (match_operand:MVE_2 3 "s_register_operand" "w")
5658                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5659          VQSHLQ_M))
5660   ]
5661   "TARGET_HAVE_MVE"
5662   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5663   [(set_attr "type" "mve_move")
5664    (set_attr "length""8")])
5665
5666 ;;
5667 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
5668 ;;
5669 (define_insn "mve_vqsubq_m_n_<supf><mode>"
5670   [
5671    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5672         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5673                        (match_operand:MVE_2 2 "s_register_operand" "w")
5674                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5675                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5676          VQSUBQ_M_N))
5677   ]
5678   "TARGET_HAVE_MVE"
5679   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5680   [(set_attr "type" "mve_move")
5681    (set_attr "length""8")])
5682
5683 ;;
5684 ;; [vqsubq_m_u, vqsubq_m_s])
5685 ;;
5686 (define_insn "mve_vqsubq_m_<supf><mode>"
5687   [
5688    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5689         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5690                        (match_operand:MVE_2 2 "s_register_operand" "w")
5691                        (match_operand:MVE_2 3 "s_register_operand" "w")
5692                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5693          VQSUBQ_M))
5694   ]
5695   "TARGET_HAVE_MVE"
5696   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5697   [(set_attr "type" "mve_move")
5698    (set_attr "length""8")])
5699
5700 ;;
5701 ;; [vrhaddq_m_u, vrhaddq_m_s])
5702 ;;
5703 (define_insn "mve_vrhaddq_m_<supf><mode>"
5704   [
5705    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5706         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5707                        (match_operand:MVE_2 2 "s_register_operand" "w")
5708                        (match_operand:MVE_2 3 "s_register_operand" "w")
5709                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5710          VRHADDQ_M))
5711   ]
5712   "TARGET_HAVE_MVE"
5713   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5714   [(set_attr "type" "mve_move")
5715    (set_attr "length""8")])
5716
5717 ;;
5718 ;; [vrmulhq_m_u, vrmulhq_m_s])
5719 ;;
5720 (define_insn "mve_vrmulhq_m_<supf><mode>"
5721   [
5722    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5723         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5724                        (match_operand:MVE_2 2 "s_register_operand" "w")
5725                        (match_operand:MVE_2 3 "s_register_operand" "w")
5726                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5727          VRMULHQ_M))
5728   ]
5729   "TARGET_HAVE_MVE"
5730   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5731   [(set_attr "type" "mve_move")
5732    (set_attr "length""8")])
5733
5734 ;;
5735 ;; [vrshlq_m_s, vrshlq_m_u])
5736 ;;
5737 (define_insn "mve_vrshlq_m_<supf><mode>"
5738   [
5739    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5740         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5741                        (match_operand:MVE_2 2 "s_register_operand" "w")
5742                        (match_operand:MVE_2 3 "s_register_operand" "w")
5743                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5744          VRSHLQ_M))
5745   ]
5746   "TARGET_HAVE_MVE"
5747   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5748   [(set_attr "type" "mve_move")
5749    (set_attr "length""8")])
5750
5751 ;;
5752 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
5753 ;;
5754 (define_insn "mve_vrshrq_m_n_<supf><mode>"
5755   [
5756    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5757         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5758                        (match_operand:MVE_2 2 "s_register_operand" "w")
5759                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5760                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5761          VRSHRQ_M_N))
5762   ]
5763   "TARGET_HAVE_MVE"
5764   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5765   [(set_attr "type" "mve_move")
5766    (set_attr "length""8")])
5767
5768 ;;
5769 ;; [vshlq_m_n_s, vshlq_m_n_u])
5770 ;;
5771 (define_insn "mve_vshlq_m_n_<supf><mode>"
5772   [
5773    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5774         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5775                        (match_operand:MVE_2 2 "s_register_operand" "w")
5776                        (match_operand:SI 3 "immediate_operand" "i")
5777                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5778          VSHLQ_M_N))
5779   ]
5780   "TARGET_HAVE_MVE"
5781   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5782   [(set_attr "type" "mve_move")
5783    (set_attr "length""8")])
5784
5785 ;;
5786 ;; [vshrq_m_n_s, vshrq_m_n_u])
5787 ;;
5788 (define_insn "mve_vshrq_m_n_<supf><mode>"
5789   [
5790    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5791         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5792                        (match_operand:MVE_2 2 "s_register_operand" "w")
5793                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5794                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5795          VSHRQ_M_N))
5796   ]
5797   "TARGET_HAVE_MVE"
5798   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5799   [(set_attr "type" "mve_move")
5800    (set_attr "length""8")])
5801
5802 ;;
5803 ;; [vsliq_m_n_u, vsliq_m_n_s])
5804 ;;
5805 (define_insn "mve_vsliq_m_n_<supf><mode>"
5806    [
5807    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5808        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5809                        (match_operand:MVE_2 2 "s_register_operand" "w")
5810                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
5811                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5812          VSLIQ_M_N))
5813   ]
5814   "TARGET_HAVE_MVE"
5815   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
5816   [(set_attr "type" "mve_move")
5817    (set_attr "length""8")])
5818
5819 ;;
5820 ;; [vsubq_m_n_s, vsubq_m_n_u])
5821 ;;
5822 (define_insn "mve_vsubq_m_n_<supf><mode>"
5823   [
5824    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5825         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5826                        (match_operand:MVE_2 2 "s_register_operand" "w")
5827                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5828                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5829          VSUBQ_M_N))
5830   ]
5831   "TARGET_HAVE_MVE"
5832   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
5833   [(set_attr "type" "mve_move")
5834    (set_attr "length""8")])
5835
5836 ;;
5837 ;; [vhcaddq_rot270_m_s])
5838 ;;
5839 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
5840   [
5841    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5842         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5843                        (match_operand:MVE_2 2 "s_register_operand" "w")
5844                        (match_operand:MVE_2 3 "s_register_operand" "w")
5845                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5846          VHCADDQ_ROT270_M_S))
5847   ]
5848   "TARGET_HAVE_MVE"
5849   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
5850   [(set_attr "type" "mve_move")
5851    (set_attr "length""8")])
5852
5853 ;;
5854 ;; [vhcaddq_rot90_m_s])
5855 ;;
5856 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
5857   [
5858    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5859         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5860                        (match_operand:MVE_2 2 "s_register_operand" "w")
5861                        (match_operand:MVE_2 3 "s_register_operand" "w")
5862                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5863          VHCADDQ_ROT90_M_S))
5864   ]
5865   "TARGET_HAVE_MVE"
5866   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
5867   [(set_attr "type" "mve_move")
5868    (set_attr "length""8")])
5869
5870 ;;
5871 ;; [vmladavaxq_p_s])
5872 ;;
5873 (define_insn "mve_vmladavaxq_p_s<mode>"
5874   [
5875    (set (match_operand:SI 0 "s_register_operand" "=Te")
5876         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5877                        (match_operand:MVE_2 2 "s_register_operand" "w")
5878                        (match_operand:MVE_2 3 "s_register_operand" "w")
5879                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5880          VMLADAVAXQ_P_S))
5881   ]
5882   "TARGET_HAVE_MVE"
5883   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5884   [(set_attr "type" "mve_move")
5885    (set_attr "length""8")])
5886
5887 ;;
5888 ;; [vmlsdavaq_p_s])
5889 ;;
5890 (define_insn "mve_vmlsdavaq_p_s<mode>"
5891   [
5892    (set (match_operand:SI 0 "s_register_operand" "=Te")
5893         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5894                        (match_operand:MVE_2 2 "s_register_operand" "w")
5895                        (match_operand:MVE_2 3 "s_register_operand" "w")
5896                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5897          VMLSDAVAQ_P_S))
5898   ]
5899   "TARGET_HAVE_MVE"
5900   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
5901   [(set_attr "type" "mve_move")
5902    (set_attr "length""8")])
5903
5904 ;;
5905 ;; [vmlsdavaxq_p_s])
5906 ;;
5907 (define_insn "mve_vmlsdavaxq_p_s<mode>"
5908   [
5909    (set (match_operand:SI 0 "s_register_operand" "=Te")
5910         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5911                        (match_operand:MVE_2 2 "s_register_operand" "w")
5912                        (match_operand:MVE_2 3 "s_register_operand" "w")
5913                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5914          VMLSDAVAXQ_P_S))
5915   ]
5916   "TARGET_HAVE_MVE"
5917   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5918   [(set_attr "type" "mve_move")
5919    (set_attr "length""8")])
5920
5921 ;;
5922 ;; [vqdmladhq_m_s])
5923 ;;
5924 (define_insn "mve_vqdmladhq_m_s<mode>"
5925   [
5926    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5927         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5928                        (match_operand:MVE_2 2 "s_register_operand" "w")
5929                        (match_operand:MVE_2 3 "s_register_operand" "w")
5930                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5931          VQDMLADHQ_M_S))
5932   ]
5933   "TARGET_HAVE_MVE"
5934   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5935   [(set_attr "type" "mve_move")
5936    (set_attr "length""8")])
5937
5938 ;;
5939 ;; [vqdmladhxq_m_s])
5940 ;;
5941 (define_insn "mve_vqdmladhxq_m_s<mode>"
5942   [
5943    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5944         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5945                        (match_operand:MVE_2 2 "s_register_operand" "w")
5946                        (match_operand:MVE_2 3 "s_register_operand" "w")
5947                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5948          VQDMLADHXQ_M_S))
5949   ]
5950   "TARGET_HAVE_MVE"
5951   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5952   [(set_attr "type" "mve_move")
5953    (set_attr "length""8")])
5954
5955 ;;
5956 ;; [vqdmlsdhq_m_s])
5957 ;;
5958 (define_insn "mve_vqdmlsdhq_m_s<mode>"
5959   [
5960    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5961         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5962                        (match_operand:MVE_2 2 "s_register_operand" "w")
5963                        (match_operand:MVE_2 3 "s_register_operand" "w")
5964                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5965          VQDMLSDHQ_M_S))
5966   ]
5967   "TARGET_HAVE_MVE"
5968   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5969   [(set_attr "type" "mve_move")
5970    (set_attr "length""8")])
5971
5972 ;;
5973 ;; [vqdmlsdhxq_m_s])
5974 ;;
5975 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
5976   [
5977    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5978         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5979                        (match_operand:MVE_2 2 "s_register_operand" "w")
5980                        (match_operand:MVE_2 3 "s_register_operand" "w")
5981                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5982          VQDMLSDHXQ_M_S))
5983   ]
5984   "TARGET_HAVE_MVE"
5985   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5986   [(set_attr "type" "mve_move")
5987    (set_attr "length""8")])
5988
5989 ;;
5990 ;; [vqdmulhq_m_n_s])
5991 ;;
5992 (define_insn "mve_vqdmulhq_m_n_s<mode>"
5993   [
5994    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5995         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5996                        (match_operand:MVE_2 2 "s_register_operand" "w")
5997                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5998                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5999          VQDMULHQ_M_N_S))
6000   ]
6001   "TARGET_HAVE_MVE"
6002   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6003   [(set_attr "type" "mve_move")
6004    (set_attr "length""8")])
6005
6006 ;;
6007 ;; [vqdmulhq_m_s])
6008 ;;
6009 (define_insn "mve_vqdmulhq_m_s<mode>"
6010   [
6011    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6012         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6013                        (match_operand:MVE_2 2 "s_register_operand" "w")
6014                        (match_operand:MVE_2 3 "s_register_operand" "w")
6015                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6016          VQDMULHQ_M_S))
6017   ]
6018   "TARGET_HAVE_MVE"
6019   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6020   [(set_attr "type" "mve_move")
6021    (set_attr "length""8")])
6022
6023 ;;
6024 ;; [vqrdmladhq_m_s])
6025 ;;
6026 (define_insn "mve_vqrdmladhq_m_s<mode>"
6027   [
6028    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6029         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6030                        (match_operand:MVE_2 2 "s_register_operand" "w")
6031                        (match_operand:MVE_2 3 "s_register_operand" "w")
6032                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6033          VQRDMLADHQ_M_S))
6034   ]
6035   "TARGET_HAVE_MVE"
6036   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6037   [(set_attr "type" "mve_move")
6038    (set_attr "length""8")])
6039
6040 ;;
6041 ;; [vqrdmladhxq_m_s])
6042 ;;
6043 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6044   [
6045    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6046         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6047                        (match_operand:MVE_2 2 "s_register_operand" "w")
6048                        (match_operand:MVE_2 3 "s_register_operand" "w")
6049                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6050          VQRDMLADHXQ_M_S))
6051   ]
6052   "TARGET_HAVE_MVE"
6053   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6054   [(set_attr "type" "mve_move")
6055    (set_attr "length""8")])
6056
6057 ;;
6058 ;; [vqrdmlsdhq_m_s])
6059 ;;
6060 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6061   [
6062    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6063         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6064                        (match_operand:MVE_2 2 "s_register_operand" "w")
6065                        (match_operand:MVE_2 3 "s_register_operand" "w")
6066                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6067          VQRDMLSDHQ_M_S))
6068   ]
6069   "TARGET_HAVE_MVE"
6070   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6071   [(set_attr "type" "mve_move")
6072    (set_attr "length""8")])
6073
6074 ;;
6075 ;; [vqrdmlsdhxq_m_s])
6076 ;;
6077 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6078   [
6079    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6080         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6081                        (match_operand:MVE_2 2 "s_register_operand" "w")
6082                        (match_operand:MVE_2 3 "s_register_operand" "w")
6083                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6084          VQRDMLSDHXQ_M_S))
6085   ]
6086   "TARGET_HAVE_MVE"
6087   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6088   [(set_attr "type" "mve_move")
6089    (set_attr "length""8")])
6090
6091 ;;
6092 ;; [vqrdmulhq_m_n_s])
6093 ;;
6094 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6095   [
6096    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6097         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6098                        (match_operand:MVE_2 2 "s_register_operand" "w")
6099                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6100                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6101          VQRDMULHQ_M_N_S))
6102   ]
6103   "TARGET_HAVE_MVE"
6104   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6105   [(set_attr "type" "mve_move")
6106    (set_attr "length""8")])
6107
6108 ;;
6109 ;; [vqrdmulhq_m_s])
6110 ;;
6111 (define_insn "mve_vqrdmulhq_m_s<mode>"
6112   [
6113    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6114         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6115                        (match_operand:MVE_2 2 "s_register_operand" "w")
6116                        (match_operand:MVE_2 3 "s_register_operand" "w")
6117                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6118          VQRDMULHQ_M_S))
6119   ]
6120   "TARGET_HAVE_MVE"
6121   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6122   [(set_attr "type" "mve_move")
6123    (set_attr "length""8")])
6124
6125 ;;
6126 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6127 ;;
6128 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6129   [
6130    (set (match_operand:DI 0 "s_register_operand" "=r")
6131         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6132                        (match_operand:MVE_5 2 "s_register_operand" "w")
6133                        (match_operand:MVE_5 3 "s_register_operand" "w")
6134                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6135          VMLALDAVAQ_P))
6136   ]
6137   "TARGET_HAVE_MVE"
6138   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6139   [(set_attr "type" "mve_move")
6140    (set_attr "length""8")])
6141
6142 ;;
6143 ;; [vmlaldavaxq_p_s])
6144 ;;
6145 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6146   [
6147    (set (match_operand:DI 0 "s_register_operand" "=r")
6148         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6149                        (match_operand:MVE_5 2 "s_register_operand" "w")
6150                        (match_operand:MVE_5 3 "s_register_operand" "w")
6151                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6152          VMLALDAVAXQ_P))
6153   ]
6154   "TARGET_HAVE_MVE"
6155   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6156   [(set_attr "type" "mve_move")
6157    (set_attr "length""8")])
6158
6159 ;;
6160 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6161 ;;
6162 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6163   [
6164    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6165         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6166                        (match_operand:MVE_5 2 "s_register_operand" "w")
6167                        (match_operand:SI 3 "mve_imm_8" "Rb")
6168                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6169          VQRSHRNBQ_M_N))
6170   ]
6171   "TARGET_HAVE_MVE"
6172   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6173   [(set_attr "type" "mve_move")
6174    (set_attr "length""8")])
6175
6176 ;;
6177 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6178 ;;
6179 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6180   [
6181    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6182         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6183                        (match_operand:MVE_5 2 "s_register_operand" "w")
6184                        (match_operand:SI 3 "mve_imm_8" "Rb")
6185                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6186          VQRSHRNTQ_M_N))
6187   ]
6188   "TARGET_HAVE_MVE"
6189   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6190   [(set_attr "type" "mve_move")
6191    (set_attr "length""8")])
6192
6193 ;;
6194 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6195 ;;
6196 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6197   [
6198    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6199         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6200                        (match_operand:MVE_5 2 "s_register_operand" "w")
6201                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6202                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6203          VQSHRNBQ_M_N))
6204   ]
6205   "TARGET_HAVE_MVE"
6206   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6207   [(set_attr "type" "mve_move")
6208    (set_attr "length""8")])
6209
6210 ;;
6211 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6212 ;;
6213 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6214   [
6215    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6216         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6217                        (match_operand:MVE_5 2 "s_register_operand" "w")
6218                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6219                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6220          VQSHRNTQ_M_N))
6221   ]
6222   "TARGET_HAVE_MVE"
6223   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6224   [(set_attr "type" "mve_move")
6225    (set_attr "length""8")])
6226
6227 ;;
6228 ;; [vrmlaldavhaq_p_s])
6229 ;;
6230 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6231   [
6232    (set (match_operand:DI 0 "s_register_operand" "=r")
6233         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6234                        (match_operand:V4SI 2 "s_register_operand" "w")
6235                        (match_operand:V4SI 3 "s_register_operand" "w")
6236                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6237          VRMLALDAVHAQ_P_S))
6238   ]
6239   "TARGET_HAVE_MVE"
6240   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6241   [(set_attr "type" "mve_move")
6242    (set_attr "length""8")])
6243
6244 ;;
6245 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6246 ;;
6247 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6248   [
6249    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6250         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6251                        (match_operand:MVE_5 2 "s_register_operand" "w")
6252                        (match_operand:SI 3 "mve_imm_8" "Rb")
6253                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6254          VRSHRNBQ_M_N))
6255   ]
6256   "TARGET_HAVE_MVE"
6257   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6258   [(set_attr "type" "mve_move")
6259    (set_attr "length""8")])
6260
6261 ;;
6262 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6263 ;;
6264 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6265   [
6266    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6267         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6268                        (match_operand:MVE_5 2 "s_register_operand" "w")
6269                        (match_operand:SI 3 "mve_imm_8" "Rb")
6270                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6271          VRSHRNTQ_M_N))
6272   ]
6273   "TARGET_HAVE_MVE"
6274   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6275   [(set_attr "type" "mve_move")
6276    (set_attr "length""8")])
6277
6278 ;;
6279 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6280 ;;
6281 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6282   [
6283    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6284         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6285                        (match_operand:MVE_3 2 "s_register_operand" "w")
6286                        (match_operand:SI 3 "immediate_operand" "i")
6287                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6288          VSHLLBQ_M_N))
6289   ]
6290   "TARGET_HAVE_MVE"
6291   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6292   [(set_attr "type" "mve_move")
6293    (set_attr "length""8")])
6294
6295 ;;
6296 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6297 ;;
6298 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6299   [
6300    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6301         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6302                        (match_operand:MVE_3 2 "s_register_operand" "w")
6303                        (match_operand:SI 3 "immediate_operand" "i")
6304                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6305          VSHLLTQ_M_N))
6306   ]
6307   "TARGET_HAVE_MVE"
6308   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6309   [(set_attr "type" "mve_move")
6310    (set_attr "length""8")])
6311
6312 ;;
6313 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6314 ;;
6315 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6316   [
6317    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6318         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6319                        (match_operand:MVE_5 2 "s_register_operand" "w")
6320                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6321                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6322          VSHRNBQ_M_N))
6323   ]
6324   "TARGET_HAVE_MVE"
6325   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6326   [(set_attr "type" "mve_move")
6327    (set_attr "length""8")])
6328
6329 ;;
6330 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6331 ;;
6332 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6333   [
6334    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6335         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6336                        (match_operand:MVE_5 2 "s_register_operand" "w")
6337                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6338                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6339          VSHRNTQ_M_N))
6340   ]
6341   "TARGET_HAVE_MVE"
6342   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6343   [(set_attr "type" "mve_move")
6344    (set_attr "length""8")])
6345
6346 ;;
6347 ;; [vmlsldavaq_p_s])
6348 ;;
6349 (define_insn "mve_vmlsldavaq_p_s<mode>"
6350   [
6351    (set (match_operand:DI 0 "s_register_operand" "=r")
6352         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6353                        (match_operand:MVE_5 2 "s_register_operand" "w")
6354                        (match_operand:MVE_5 3 "s_register_operand" "w")
6355                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6356          VMLSLDAVAQ_P_S))
6357   ]
6358   "TARGET_HAVE_MVE"
6359   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6360   [(set_attr "type" "mve_move")
6361    (set_attr "length""8")])
6362
6363 ;;
6364 ;; [vmlsldavaxq_p_s])
6365 ;;
6366 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6367   [
6368    (set (match_operand:DI 0 "s_register_operand" "=r")
6369         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6370                        (match_operand:MVE_5 2 "s_register_operand" "w")
6371                        (match_operand:MVE_5 3 "s_register_operand" "w")
6372                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6373          VMLSLDAVAXQ_P_S))
6374   ]
6375   "TARGET_HAVE_MVE"
6376   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6377   [(set_attr "type" "mve_move")
6378    (set_attr "length""8")])
6379
6380 ;;
6381 ;; [vmullbq_poly_m_p])
6382 ;;
6383 (define_insn "mve_vmullbq_poly_m_p<mode>"
6384   [
6385    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6386         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6387                        (match_operand:MVE_3 2 "s_register_operand" "w")
6388                        (match_operand:MVE_3 3 "s_register_operand" "w")
6389                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6390          VMULLBQ_POLY_M_P))
6391   ]
6392   "TARGET_HAVE_MVE"
6393   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6394   [(set_attr "type" "mve_move")
6395    (set_attr "length""8")])
6396
6397 ;;
6398 ;; [vmulltq_poly_m_p])
6399 ;;
6400 (define_insn "mve_vmulltq_poly_m_p<mode>"
6401   [
6402    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6403         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6404                        (match_operand:MVE_3 2 "s_register_operand" "w")
6405                        (match_operand:MVE_3 3 "s_register_operand" "w")
6406                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6407          VMULLTQ_POLY_M_P))
6408   ]
6409   "TARGET_HAVE_MVE"
6410   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6411   [(set_attr "type" "mve_move")
6412    (set_attr "length""8")])
6413
6414 ;;
6415 ;; [vqdmullbq_m_n_s])
6416 ;;
6417 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6418   [
6419    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6420         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6421                        (match_operand:MVE_5 2 "s_register_operand" "w")
6422                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6423                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6424          VQDMULLBQ_M_N_S))
6425   ]
6426   "TARGET_HAVE_MVE"
6427   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6428   [(set_attr "type" "mve_move")
6429    (set_attr "length""8")])
6430
6431 ;;
6432 ;; [vqdmullbq_m_s])
6433 ;;
6434 (define_insn "mve_vqdmullbq_m_s<mode>"
6435   [
6436    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6437         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6438                        (match_operand:MVE_5 2 "s_register_operand" "w")
6439                        (match_operand:MVE_5 3 "s_register_operand" "w")
6440                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6441          VQDMULLBQ_M_S))
6442   ]
6443   "TARGET_HAVE_MVE"
6444   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6445   [(set_attr "type" "mve_move")
6446    (set_attr "length""8")])
6447
6448 ;;
6449 ;; [vqdmulltq_m_n_s])
6450 ;;
6451 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6452   [
6453    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6454         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6455                        (match_operand:MVE_5 2 "s_register_operand" "w")
6456                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6457                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6458          VQDMULLTQ_M_N_S))
6459   ]
6460   "TARGET_HAVE_MVE"
6461   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6462   [(set_attr "type" "mve_move")
6463    (set_attr "length""8")])
6464
6465 ;;
6466 ;; [vqdmulltq_m_s])
6467 ;;
6468 (define_insn "mve_vqdmulltq_m_s<mode>"
6469   [
6470    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6471         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6472                        (match_operand:MVE_5 2 "s_register_operand" "w")
6473                        (match_operand:MVE_5 3 "s_register_operand" "w")
6474                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6475          VQDMULLTQ_M_S))
6476   ]
6477   "TARGET_HAVE_MVE"
6478   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6479   [(set_attr "type" "mve_move")
6480    (set_attr "length""8")])
6481
6482 ;;
6483 ;; [vqrshrunbq_m_n_s])
6484 ;;
6485 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6486   [
6487    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6488         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6489                        (match_operand:MVE_5 2 "s_register_operand" "w")
6490                        (match_operand:SI 3 "mve_imm_8" "Rb")
6491                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6492          VQRSHRUNBQ_M_N_S))
6493   ]
6494   "TARGET_HAVE_MVE"
6495   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6496   [(set_attr "type" "mve_move")
6497    (set_attr "length""8")])
6498
6499 ;;
6500 ;; [vqrshruntq_m_n_s])
6501 ;;
6502 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6503   [
6504    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6505         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6506                        (match_operand:MVE_5 2 "s_register_operand" "w")
6507                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6508                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6509          VQRSHRUNTQ_M_N_S))
6510   ]
6511   "TARGET_HAVE_MVE"
6512   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6513   [(set_attr "type" "mve_move")
6514    (set_attr "length""8")])
6515
6516 ;;
6517 ;; [vqshrunbq_m_n_s])
6518 ;;
6519 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6520   [
6521    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6522         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6523                        (match_operand:MVE_5 2 "s_register_operand" "w")
6524                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6525                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6526          VQSHRUNBQ_M_N_S))
6527   ]
6528   "TARGET_HAVE_MVE"
6529   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6530   [(set_attr "type" "mve_move")
6531    (set_attr "length""8")])
6532
6533 ;;
6534 ;; [vqshruntq_m_n_s])
6535 ;;
6536 (define_insn "mve_vqshruntq_m_n_s<mode>"
6537   [
6538    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6539         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6540                        (match_operand:MVE_5 2 "s_register_operand" "w")
6541                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6542                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6543          VQSHRUNTQ_M_N_S))
6544   ]
6545   "TARGET_HAVE_MVE"
6546   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6547   [(set_attr "type" "mve_move")
6548    (set_attr "length""8")])
6549
6550 ;;
6551 ;; [vrmlaldavhaq_p_u])
6552 ;;
6553 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6554   [
6555    (set (match_operand:DI 0 "s_register_operand" "=r")
6556         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6557                        (match_operand:V4SI 2 "s_register_operand" "w")
6558                        (match_operand:V4SI 3 "s_register_operand" "w")
6559                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6560          VRMLALDAVHAQ_P_U))
6561   ]
6562   "TARGET_HAVE_MVE"
6563   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6564   [(set_attr "type" "mve_move")
6565    (set_attr "length""8")])
6566
6567 ;;
6568 ;; [vrmlaldavhaxq_p_s])
6569 ;;
6570 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6571   [
6572    (set (match_operand:DI 0 "s_register_operand" "=r")
6573         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6574                        (match_operand:V4SI 2 "s_register_operand" "w")
6575                        (match_operand:V4SI 3 "s_register_operand" "w")
6576                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6577          VRMLALDAVHAXQ_P_S))
6578   ]
6579   "TARGET_HAVE_MVE"
6580   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6581   [(set_attr "type" "mve_move")
6582    (set_attr "length""8")])
6583
6584 ;;
6585 ;; [vrmlsldavhaq_p_s])
6586 ;;
6587 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6588   [
6589    (set (match_operand:DI 0 "s_register_operand" "=r")
6590         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6591                        (match_operand:V4SI 2 "s_register_operand" "w")
6592                        (match_operand:V4SI 3 "s_register_operand" "w")
6593                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6594          VRMLSLDAVHAQ_P_S))
6595   ]
6596   "TARGET_HAVE_MVE"
6597   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6598   [(set_attr "type" "mve_move")
6599    (set_attr "length""8")])
6600
6601 ;;
6602 ;; [vrmlsldavhaxq_p_s])
6603 ;;
6604 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6605   [
6606    (set (match_operand:DI 0 "s_register_operand" "=r")
6607         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6608                        (match_operand:V4SI 2 "s_register_operand" "w")
6609                        (match_operand:V4SI 3 "s_register_operand" "w")
6610                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6611          VRMLSLDAVHAXQ_P_S))
6612   ]
6613   "TARGET_HAVE_MVE"
6614   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6615   [(set_attr "type" "mve_move")
6616    (set_attr "length""8")])
6617 ;;
6618 ;; [vabdq_m_f])
6619 ;;
6620 (define_insn "mve_vabdq_m_f<mode>"
6621   [
6622    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6623         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6624                        (match_operand:MVE_0 2 "s_register_operand" "w")
6625                        (match_operand:MVE_0 3 "s_register_operand" "w")
6626                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6627          VABDQ_M_F))
6628   ]
6629   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6630   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6631   [(set_attr "type" "mve_move")
6632    (set_attr "length""8")])
6633
6634 ;;
6635 ;; [vaddq_m_f])
6636 ;;
6637 (define_insn "mve_vaddq_m_f<mode>"
6638   [
6639    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6640         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6641                        (match_operand:MVE_0 2 "s_register_operand" "w")
6642                        (match_operand:MVE_0 3 "s_register_operand" "w")
6643                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6644          VADDQ_M_F))
6645   ]
6646   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6647   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6648   [(set_attr "type" "mve_move")
6649    (set_attr "length""8")])
6650
6651 ;;
6652 ;; [vaddq_m_n_f])
6653 ;;
6654 (define_insn "mve_vaddq_m_n_f<mode>"
6655   [
6656    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6657         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6658                        (match_operand:MVE_0 2 "s_register_operand" "w")
6659                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6660                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6661          VADDQ_M_N_F))
6662   ]
6663   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6664   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
6665   [(set_attr "type" "mve_move")
6666    (set_attr "length""8")])
6667
6668 ;;
6669 ;; [vandq_m_f])
6670 ;;
6671 (define_insn "mve_vandq_m_f<mode>"
6672   [
6673    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6674         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6675                        (match_operand:MVE_0 2 "s_register_operand" "w")
6676                        (match_operand:MVE_0 3 "s_register_operand" "w")
6677                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6678          VANDQ_M_F))
6679   ]
6680   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6681   "vpst\;vandt %q0, %q2, %q3"
6682   [(set_attr "type" "mve_move")
6683    (set_attr "length""8")])
6684
6685 ;;
6686 ;; [vbicq_m_f])
6687 ;;
6688 (define_insn "mve_vbicq_m_f<mode>"
6689   [
6690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6691         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6692                        (match_operand:MVE_0 2 "s_register_operand" "w")
6693                        (match_operand:MVE_0 3 "s_register_operand" "w")
6694                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6695          VBICQ_M_F))
6696   ]
6697   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6698   "vpst\;vbict %q0, %q2, %q3"
6699   [(set_attr "type" "mve_move")
6700    (set_attr "length""8")])
6701
6702 ;;
6703 ;; [vbrsrq_m_n_f])
6704 ;;
6705 (define_insn "mve_vbrsrq_m_n_f<mode>"
6706   [
6707    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6708         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6709                        (match_operand:MVE_0 2 "s_register_operand" "w")
6710                        (match_operand:SI 3 "s_register_operand" "r")
6711                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6712          VBRSRQ_M_N_F))
6713   ]
6714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6715   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6716   [(set_attr "type" "mve_move")
6717    (set_attr "length""8")])
6718
6719 ;;
6720 ;; [vcaddq_rot270_m_f])
6721 ;;
6722 (define_insn "mve_vcaddq_rot270_m_f<mode>"
6723   [
6724    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6725         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6726                        (match_operand:MVE_0 2 "s_register_operand" "w")
6727                        (match_operand:MVE_0 3 "s_register_operand" "w")
6728                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6729          VCADDQ_ROT270_M_F))
6730   ]
6731   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6732   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6733   [(set_attr "type" "mve_move")
6734    (set_attr "length""8")])
6735
6736 ;;
6737 ;; [vcaddq_rot90_m_f])
6738 ;;
6739 (define_insn "mve_vcaddq_rot90_m_f<mode>"
6740   [
6741    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6742         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6743                        (match_operand:MVE_0 2 "s_register_operand" "w")
6744                        (match_operand:MVE_0 3 "s_register_operand" "w")
6745                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6746          VCADDQ_ROT90_M_F))
6747   ]
6748   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6749   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6750   [(set_attr "type" "mve_move")
6751    (set_attr "length""8")])
6752
6753 ;;
6754 ;; [vcmlaq_m_f])
6755 ;;
6756 (define_insn "mve_vcmlaq_m_f<mode>"
6757   [
6758    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6759         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6760                        (match_operand:MVE_0 2 "s_register_operand" "w")
6761                        (match_operand:MVE_0 3 "s_register_operand" "w")
6762                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6763          VCMLAQ_M_F))
6764   ]
6765   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6766   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6767   [(set_attr "type" "mve_move")
6768    (set_attr "length""8")])
6769
6770 ;;
6771 ;; [vcmlaq_rot180_m_f])
6772 ;;
6773 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
6774   [
6775    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6776         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6777                        (match_operand:MVE_0 2 "s_register_operand" "w")
6778                        (match_operand:MVE_0 3 "s_register_operand" "w")
6779                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6780          VCMLAQ_ROT180_M_F))
6781   ]
6782   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6783   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6784   [(set_attr "type" "mve_move")
6785    (set_attr "length""8")])
6786
6787 ;;
6788 ;; [vcmlaq_rot270_m_f])
6789 ;;
6790 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
6791   [
6792    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6793         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6794                        (match_operand:MVE_0 2 "s_register_operand" "w")
6795                        (match_operand:MVE_0 3 "s_register_operand" "w")
6796                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6797          VCMLAQ_ROT270_M_F))
6798   ]
6799   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6800   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6801   [(set_attr "type" "mve_move")
6802    (set_attr "length""8")])
6803
6804 ;;
6805 ;; [vcmlaq_rot90_m_f])
6806 ;;
6807 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
6808   [
6809    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6810         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6811                        (match_operand:MVE_0 2 "s_register_operand" "w")
6812                        (match_operand:MVE_0 3 "s_register_operand" "w")
6813                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6814          VCMLAQ_ROT90_M_F))
6815   ]
6816   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6817   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6818   [(set_attr "type" "mve_move")
6819    (set_attr "length""8")])
6820
6821 ;;
6822 ;; [vcmulq_m_f])
6823 ;;
6824 (define_insn "mve_vcmulq_m_f<mode>"
6825   [
6826    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6827         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6828                        (match_operand:MVE_0 2 "s_register_operand" "w")
6829                        (match_operand:MVE_0 3 "s_register_operand" "w")
6830                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6831          VCMULQ_M_F))
6832   ]
6833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6834   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6835   [(set_attr "type" "mve_move")
6836    (set_attr "length""8")])
6837
6838 ;;
6839 ;; [vcmulq_rot180_m_f])
6840 ;;
6841 (define_insn "mve_vcmulq_rot180_m_f<mode>"
6842   [
6843    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6844         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6845                        (match_operand:MVE_0 2 "s_register_operand" "w")
6846                        (match_operand:MVE_0 3 "s_register_operand" "w")
6847                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6848          VCMULQ_ROT180_M_F))
6849   ]
6850   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6851   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6852   [(set_attr "type" "mve_move")
6853    (set_attr "length""8")])
6854
6855 ;;
6856 ;; [vcmulq_rot270_m_f])
6857 ;;
6858 (define_insn "mve_vcmulq_rot270_m_f<mode>"
6859   [
6860    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6861         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6862                        (match_operand:MVE_0 2 "s_register_operand" "w")
6863                        (match_operand:MVE_0 3 "s_register_operand" "w")
6864                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6865          VCMULQ_ROT270_M_F))
6866   ]
6867   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6868   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6869   [(set_attr "type" "mve_move")
6870    (set_attr "length""8")])
6871
6872 ;;
6873 ;; [vcmulq_rot90_m_f])
6874 ;;
6875 (define_insn "mve_vcmulq_rot90_m_f<mode>"
6876   [
6877    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6878         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6879                        (match_operand:MVE_0 2 "s_register_operand" "w")
6880                        (match_operand:MVE_0 3 "s_register_operand" "w")
6881                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6882          VCMULQ_ROT90_M_F))
6883   ]
6884   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6885   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6886   [(set_attr "type" "mve_move")
6887    (set_attr "length""8")])
6888
6889 ;;
6890 ;; [veorq_m_f])
6891 ;;
6892 (define_insn "mve_veorq_m_f<mode>"
6893   [
6894    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6895         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6896                        (match_operand:MVE_0 2 "s_register_operand" "w")
6897                        (match_operand:MVE_0 3 "s_register_operand" "w")
6898                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6899          VEORQ_M_F))
6900   ]
6901   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6902   "vpst\;veort %q0, %q2, %q3"
6903   [(set_attr "type" "mve_move")
6904    (set_attr "length""8")])
6905
6906 ;;
6907 ;; [vfmaq_m_f])
6908 ;;
6909 (define_insn "mve_vfmaq_m_f<mode>"
6910   [
6911    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6912         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6913                        (match_operand:MVE_0 2 "s_register_operand" "w")
6914                        (match_operand:MVE_0 3 "s_register_operand" "w")
6915                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6916          VFMAQ_M_F))
6917   ]
6918   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6919   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
6920   [(set_attr "type" "mve_move")
6921    (set_attr "length""8")])
6922
6923 ;;
6924 ;; [vfmaq_m_n_f])
6925 ;;
6926 (define_insn "mve_vfmaq_m_n_f<mode>"
6927   [
6928    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6929         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6930                        (match_operand:MVE_0 2 "s_register_operand" "w")
6931                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6932                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6933          VFMAQ_M_N_F))
6934   ]
6935   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6936   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
6937   [(set_attr "type" "mve_move")
6938    (set_attr "length""8")])
6939
6940 ;;
6941 ;; [vfmasq_m_n_f])
6942 ;;
6943 (define_insn "mve_vfmasq_m_n_f<mode>"
6944   [
6945    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6946         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6947                        (match_operand:MVE_0 2 "s_register_operand" "w")
6948                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6949                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6950          VFMASQ_M_N_F))
6951   ]
6952   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6953   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
6954   [(set_attr "type" "mve_move")
6955    (set_attr "length""8")])
6956
6957 ;;
6958 ;; [vfmsq_m_f])
6959 ;;
6960 (define_insn "mve_vfmsq_m_f<mode>"
6961   [
6962    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6963         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6964                        (match_operand:MVE_0 2 "s_register_operand" "w")
6965                        (match_operand:MVE_0 3 "s_register_operand" "w")
6966                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6967          VFMSQ_M_F))
6968   ]
6969   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6970   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
6971   [(set_attr "type" "mve_move")
6972    (set_attr "length""8")])
6973
6974 ;;
6975 ;; [vmaxnmq_m_f])
6976 ;;
6977 (define_insn "mve_vmaxnmq_m_f<mode>"
6978   [
6979    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6980         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6981                        (match_operand:MVE_0 2 "s_register_operand" "w")
6982                        (match_operand:MVE_0 3 "s_register_operand" "w")
6983                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6984          VMAXNMQ_M_F))
6985   ]
6986   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6987   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6988   [(set_attr "type" "mve_move")
6989    (set_attr "length""8")])
6990
6991 ;;
6992 ;; [vminnmq_m_f])
6993 ;;
6994 (define_insn "mve_vminnmq_m_f<mode>"
6995   [
6996    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6997         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6998                        (match_operand:MVE_0 2 "s_register_operand" "w")
6999                        (match_operand:MVE_0 3 "s_register_operand" "w")
7000                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7001          VMINNMQ_M_F))
7002   ]
7003   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7004   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7005   [(set_attr "type" "mve_move")
7006    (set_attr "length""8")])
7007
7008 ;;
7009 ;; [vmulq_m_f])
7010 ;;
7011 (define_insn "mve_vmulq_m_f<mode>"
7012   [
7013    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7014         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7015                        (match_operand:MVE_0 2 "s_register_operand" "w")
7016                        (match_operand:MVE_0 3 "s_register_operand" "w")
7017                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7018          VMULQ_M_F))
7019   ]
7020   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7021   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7022   [(set_attr "type" "mve_move")
7023    (set_attr "length""8")])
7024
7025 ;;
7026 ;; [vmulq_m_n_f])
7027 ;;
7028 (define_insn "mve_vmulq_m_n_f<mode>"
7029   [
7030    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7031         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7032                        (match_operand:MVE_0 2 "s_register_operand" "w")
7033                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7034                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7035          VMULQ_M_N_F))
7036   ]
7037   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7038   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7039   [(set_attr "type" "mve_move")
7040    (set_attr "length""8")])
7041
7042 ;;
7043 ;; [vornq_m_f])
7044 ;;
7045 (define_insn "mve_vornq_m_f<mode>"
7046   [
7047    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7048         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7049                        (match_operand:MVE_0 2 "s_register_operand" "w")
7050                        (match_operand:MVE_0 3 "s_register_operand" "w")
7051                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7052          VORNQ_M_F))
7053   ]
7054   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7055   "vpst\;vornt %q0, %q2, %q3"
7056   [(set_attr "type" "mve_move")
7057    (set_attr "length""8")])
7058
7059 ;;
7060 ;; [vorrq_m_f])
7061 ;;
7062 (define_insn "mve_vorrq_m_f<mode>"
7063   [
7064    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7065         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7066                        (match_operand:MVE_0 2 "s_register_operand" "w")
7067                        (match_operand:MVE_0 3 "s_register_operand" "w")
7068                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7069          VORRQ_M_F))
7070   ]
7071   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7072   "vpst\;vorrt %q0, %q2, %q3"
7073   [(set_attr "type" "mve_move")
7074    (set_attr "length""8")])
7075
7076 ;;
7077 ;; [vsubq_m_f])
7078 ;;
7079 (define_insn "mve_vsubq_m_f<mode>"
7080   [
7081    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7082         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7083                        (match_operand:MVE_0 2 "s_register_operand" "w")
7084                        (match_operand:MVE_0 3 "s_register_operand" "w")
7085                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7086          VSUBQ_M_F))
7087   ]
7088   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7089   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7090   [(set_attr "type" "mve_move")
7091    (set_attr "length""8")])
7092
7093 ;;
7094 ;; [vsubq_m_n_f])
7095 ;;
7096 (define_insn "mve_vsubq_m_n_f<mode>"
7097   [
7098    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7099         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7100                        (match_operand:MVE_0 2 "s_register_operand" "w")
7101                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7102                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7103          VSUBQ_M_N_F))
7104   ]
7105   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7106   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7107   [(set_attr "type" "mve_move")
7108    (set_attr "length""8")])
7109
7110 ;;
7111 ;; [vstrbq_s vstrbq_u]
7112 ;;
7113 (define_insn "mve_vstrbq_<supf><mode>"
7114   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7115         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7116          VSTRBQ))
7117   ]
7118   "TARGET_HAVE_MVE"
7119 {
7120    rtx ops[2];
7121    int regno = REGNO (operands[1]);
7122    ops[1] = gen_rtx_REG (TImode, regno);
7123    ops[0]  = operands[0];
7124    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7125    return "";
7126 }
7127   [(set_attr "length" "4")])
7128
7129 ;;
7130 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7131 ;;
7132 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7133   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7134    (match_operand:MVE_2 1 "s_register_operand")
7135    (match_operand:MVE_2 2 "s_register_operand")
7136    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7137   "TARGET_HAVE_MVE"
7138 {
7139   rtx ind = XEXP (operands[0], 0);
7140   gcc_assert (REG_P (ind));
7141   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7142                                                               operands[2]));
7143   DONE;
7144 })
7145
7146 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7147   [(set (mem:BLK (scratch))
7148         (unspec:BLK
7149           [(match_operand:SI 0 "register_operand" "r")
7150            (match_operand:MVE_2 1 "s_register_operand" "w")
7151            (match_operand:MVE_2 2 "s_register_operand" "w")]
7152           VSTRBSOQ))]
7153   "TARGET_HAVE_MVE"
7154   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7155   [(set_attr "length" "4")])
7156
7157 ;;
7158 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7159 ;;
7160 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7161   [(set (mem:BLK (scratch))
7162         (unspec:BLK
7163                 [(match_operand:V4SI 0 "s_register_operand" "w")
7164                  (match_operand:SI 1 "immediate_operand" "i")
7165                  (match_operand:V4SI 2 "s_register_operand" "w")]
7166          VSTRWSBQ))
7167   ]
7168   "TARGET_HAVE_MVE"
7169 {
7170    rtx ops[3];
7171    ops[0] = operands[0];
7172    ops[1] = operands[1];
7173    ops[2] = operands[2];
7174    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7175    return "";
7176 }
7177   [(set_attr "length" "4")])
7178
7179 ;;
7180 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7181 ;;
7182 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7183   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7184         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7185                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7186          VLDRBGOQ))
7187   ]
7188   "TARGET_HAVE_MVE"
7189 {
7190    rtx ops[3];
7191    ops[0] = operands[0];
7192    ops[1] = operands[1];
7193    ops[2] = operands[2];
7194    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7195      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7196    else
7197      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7198    return "";
7199 }
7200   [(set_attr "length" "4")])
7201
7202 ;;
7203 ;; [vldrbq_s vldrbq_u]
7204 ;;
7205 (define_insn "mve_vldrbq_<supf><mode>"
7206   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7207         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7208          VLDRBQ))
7209   ]
7210   "TARGET_HAVE_MVE"
7211 {
7212    rtx ops[2];
7213    int regno = REGNO (operands[0]);
7214    ops[0] = gen_rtx_REG (TImode, regno);
7215    ops[1]  = operands[1];
7216    if (<V_sz_elem> == 8)
7217      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7218    else
7219      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7220    return "";
7221 }
7222   [(set_attr "length" "4")])
7223
7224 ;;
7225 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7226 ;;
7227 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7228   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7229         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7230                       (match_operand:SI 2 "immediate_operand" "i")]
7231          VLDRWGBQ))
7232   ]
7233   "TARGET_HAVE_MVE"
7234 {
7235    rtx ops[3];
7236    ops[0] = operands[0];
7237    ops[1] = operands[1];
7238    ops[2] = operands[2];
7239    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7240    return "";
7241 }
7242   [(set_attr "length" "4")])
7243
7244 ;;
7245 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7246 ;;
7247 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7248   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7249    (match_operand:MVE_2 1 "s_register_operand")
7250    (match_operand:MVE_2 2 "s_register_operand")
7251    (match_operand:HI 3 "vpr_register_operand" "Up")
7252    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7253   "TARGET_HAVE_MVE"
7254 {
7255   rtx ind = XEXP (operands[0], 0);
7256   gcc_assert (REG_P (ind));
7257   emit_insn (
7258     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7259                                                        operands[2],
7260                                                        operands[3]));
7261   DONE;
7262 })
7263
7264 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7265   [(set (mem:BLK (scratch))
7266         (unspec:BLK
7267           [(match_operand:SI 0 "register_operand" "r")
7268            (match_operand:MVE_2 1 "s_register_operand" "w")
7269            (match_operand:MVE_2 2 "s_register_operand" "w")
7270            (match_operand:HI 3 "vpr_register_operand" "Up")]
7271           VSTRBSOQ))]
7272   "TARGET_HAVE_MVE"
7273   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7274   [(set_attr "length" "8")])
7275
7276 ;;
7277 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7278 ;;
7279 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7280   [(set (mem:BLK (scratch))
7281         (unspec:BLK
7282                 [(match_operand:V4SI 0 "s_register_operand" "w")
7283                  (match_operand:SI 1 "immediate_operand" "i")
7284                  (match_operand:V4SI 2 "s_register_operand" "w")
7285                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7286          VSTRWSBQ))
7287   ]
7288   "TARGET_HAVE_MVE"
7289 {
7290    rtx ops[3];
7291    ops[0] = operands[0];
7292    ops[1] = operands[1];
7293    ops[2] = operands[2];
7294    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7295    return "";
7296 }
7297   [(set_attr "length" "8")])
7298
7299 ;;
7300 ;; [vstrbq_p_s vstrbq_p_u]
7301 ;;
7302 (define_insn "mve_vstrbq_p_<supf><mode>"
7303   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7304         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7305                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7306          VSTRBQ))
7307   ]
7308   "TARGET_HAVE_MVE"
7309 {
7310    rtx ops[2];
7311    int regno = REGNO (operands[1]);
7312    ops[1] = gen_rtx_REG (TImode, regno);
7313    ops[0]  = operands[0];
7314    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7315    return "";
7316 }
7317   [(set_attr "length" "8")])
7318
7319 ;;
7320 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7321 ;;
7322 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7323   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7324         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7325                        (match_operand:MVE_2 2 "s_register_operand" "w")
7326                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7327          VLDRBGOQ))
7328   ]
7329   "TARGET_HAVE_MVE"
7330 {
7331    rtx ops[4];
7332    ops[0] = operands[0];
7333    ops[1] = operands[1];
7334    ops[2] = operands[2];
7335    ops[3] = operands[3];
7336    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7337      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7338    else
7339      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7340    return "";
7341 }
7342   [(set_attr "length" "8")])
7343
7344 ;;
7345 ;; [vldrbq_z_s vldrbq_z_u]
7346 ;;
7347 (define_insn "mve_vldrbq_z_<supf><mode>"
7348   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7349         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7350                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7351          VLDRBQ))
7352   ]
7353   "TARGET_HAVE_MVE"
7354 {
7355    rtx ops[2];
7356    int regno = REGNO (operands[0]);
7357    ops[0] = gen_rtx_REG (TImode, regno);
7358    ops[1]  = operands[1];
7359    if (<V_sz_elem> == 8)
7360      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7361    else
7362      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7363    return "";
7364 }
7365   [(set_attr "length" "8")])
7366
7367 ;;
7368 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7369 ;;
7370 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7371   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7372         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7373                       (match_operand:SI 2 "immediate_operand" "i")
7374                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7375          VLDRWGBQ))
7376   ]
7377   "TARGET_HAVE_MVE"
7378 {
7379    rtx ops[3];
7380    ops[0] = operands[0];
7381    ops[1] = operands[1];
7382    ops[2] = operands[2];
7383    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7384    return "";
7385 }
7386   [(set_attr "length" "8")])
7387
7388 ;;
7389 ;; [vldrhq_f]
7390 ;;
7391 (define_insn "mve_vldrhq_fv8hf"
7392   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7393         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7394          VLDRHQ_F))
7395   ]
7396   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7397 {
7398    rtx ops[2];
7399    int regno = REGNO (operands[0]);
7400    ops[0] = gen_rtx_REG (TImode, regno);
7401    ops[1]  = operands[1];
7402    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7403    return "";
7404 }
7405   [(set_attr "length" "4")])
7406
7407 ;;
7408 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7409 ;;
7410 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7411   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7412         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7413                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7414         VLDRHGOQ))
7415   ]
7416   "TARGET_HAVE_MVE"
7417 {
7418    rtx ops[3];
7419    ops[0] = operands[0];
7420    ops[1] = operands[1];
7421    ops[2] = operands[2];
7422    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7423      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7424    else
7425      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7426    return "";
7427 }
7428   [(set_attr "length" "4")])
7429
7430 ;;
7431 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7432 ;;
7433 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7434   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7435         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7436                        (match_operand:MVE_6 2 "s_register_operand" "w")
7437                        (match_operand:HI 3 "vpr_register_operand" "Up")
7438         ]VLDRHGOQ))
7439   ]
7440   "TARGET_HAVE_MVE"
7441 {
7442    rtx ops[4];
7443    ops[0] = operands[0];
7444    ops[1] = operands[1];
7445    ops[2] = operands[2];
7446    ops[3] = operands[3];
7447    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7448      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7449    else
7450      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7451    return "";
7452 }
7453  [(set_attr "length" "8")])
7454
7455 ;;
7456 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7457 ;;
7458 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7459   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7460         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7461                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7462         VLDRHGSOQ))
7463   ]
7464   "TARGET_HAVE_MVE"
7465 {
7466    rtx ops[3];
7467    ops[0] = operands[0];
7468    ops[1] = operands[1];
7469    ops[2] = operands[2];
7470       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7471      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7472    else
7473      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7474    return "";
7475 }
7476   [(set_attr "length" "4")])
7477
7478 ;;
7479 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7480 ;;
7481 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7482   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7483         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7484                        (match_operand:MVE_6 2 "s_register_operand" "w")
7485                        (match_operand:HI 3 "vpr_register_operand" "Up")
7486         ]VLDRHGSOQ))
7487   ]
7488   "TARGET_HAVE_MVE"
7489 {
7490    rtx ops[4];
7491    ops[0] = operands[0];
7492    ops[1] = operands[1];
7493    ops[2] = operands[2];
7494    ops[3] = operands[3];
7495    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7496      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7497    else
7498      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7499    return "";
7500 }
7501   [(set_attr "length" "8")])
7502
7503 ;;
7504 ;; [vldrhq_s, vldrhq_u]
7505 ;;
7506 (define_insn "mve_vldrhq_<supf><mode>"
7507   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7508         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7509          VLDRHQ))
7510   ]
7511   "TARGET_HAVE_MVE"
7512 {
7513    rtx ops[2];
7514    int regno = REGNO (operands[0]);
7515    ops[0] = gen_rtx_REG (TImode, regno);
7516    ops[1]  = operands[1];
7517    if (<V_sz_elem> == 16)
7518      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7519    else
7520      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7521    return "";
7522 }
7523   [(set_attr "length" "4")])
7524
7525 ;;
7526 ;; [vldrhq_z_f]
7527 ;;
7528 (define_insn "mve_vldrhq_z_fv8hf"
7529   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7530         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7531         (match_operand:HI 2 "vpr_register_operand" "Up")]
7532          VLDRHQ_F))
7533   ]
7534   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7535 {
7536    rtx ops[2];
7537    int regno = REGNO (operands[0]);
7538    ops[0] = gen_rtx_REG (TImode, regno);
7539    ops[1]  = operands[1];
7540    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7541    return "";
7542 }
7543   [(set_attr "length" "8")])
7544
7545 ;;
7546 ;; [vldrhq_z_s vldrhq_z_u]
7547 ;;
7548 (define_insn "mve_vldrhq_z_<supf><mode>"
7549   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7550         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7551         (match_operand:HI 2 "vpr_register_operand" "Up")]
7552          VLDRHQ))
7553   ]
7554   "TARGET_HAVE_MVE"
7555 {
7556    rtx ops[2];
7557    int regno = REGNO (operands[0]);
7558    ops[0] = gen_rtx_REG (TImode, regno);
7559    ops[1]  = operands[1];
7560    if (<V_sz_elem> == 16)
7561      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7562    else
7563      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7564    return "";
7565 }
7566   [(set_attr "length" "8")])
7567
7568 ;;
7569 ;; [vldrwq_f]
7570 ;;
7571 (define_insn "mve_vldrwq_fv4sf"
7572   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7573         (unspec:V4SF [(match_operand:V4SI 1 "mve_memory_operand" "Ux")]
7574          VLDRWQ_F))
7575   ]
7576   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7577 {
7578    rtx ops[2];
7579    int regno = REGNO (operands[0]);
7580    ops[0] = gen_rtx_REG (TImode, regno);
7581    ops[1]  = operands[1];
7582    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7583    return "";
7584 }
7585   [(set_attr "length" "4")])
7586
7587 ;;
7588 ;; [vldrwq_s vldrwq_u]
7589 ;;
7590 (define_insn "mve_vldrwq_<supf>v4si"
7591   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7592         (unspec:V4SI [(match_operand:V4SI 1 "mve_memory_operand" "Ux")]
7593          VLDRWQ))
7594   ]
7595   "TARGET_HAVE_MVE"
7596 {
7597    rtx ops[2];
7598    int regno = REGNO (operands[0]);
7599    ops[0] = gen_rtx_REG (TImode, regno);
7600    ops[1]  = operands[1];
7601    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7602    return "";
7603 }
7604   [(set_attr "length" "4")])
7605
7606 ;;
7607 ;; [vldrwq_z_f]
7608 ;;
7609 (define_insn "mve_vldrwq_z_fv4sf"
7610   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7611         (unspec:V4SF [(match_operand:V4SI 1 "mve_memory_operand" "Ux")
7612         (match_operand:HI 2 "vpr_register_operand" "Up")]
7613          VLDRWQ_F))
7614   ]
7615   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7616 {
7617    rtx ops[2];
7618    int regno = REGNO (operands[0]);
7619    ops[0] = gen_rtx_REG (TImode, regno);
7620    ops[1]  = operands[1];
7621    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7622    return "";
7623 }
7624   [(set_attr "length" "8")])
7625
7626 ;;
7627 ;; [vldrwq_z_s vldrwq_z_u]
7628 ;;
7629 (define_insn "mve_vldrwq_z_<supf>v4si"
7630   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7631         (unspec:V4SI [(match_operand:V4SI 1 "mve_memory_operand" "Ux")
7632         (match_operand:HI 2 "vpr_register_operand" "Up")]
7633          VLDRWQ))
7634   ]
7635   "TARGET_HAVE_MVE"
7636 {
7637    rtx ops[2];
7638    int regno = REGNO (operands[0]);
7639    ops[0] = gen_rtx_REG (TImode, regno);
7640    ops[1]  = operands[1];
7641    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7642    return "";
7643 }
7644   [(set_attr "length" "8")])
7645
7646 (define_expand "mve_vld1q_f<mode>"
7647   [(match_operand:MVE_0 0 "s_register_operand")
7648    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7649   ]
7650   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
7651 {
7652   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
7653   DONE;
7654 })
7655
7656 (define_expand "mve_vld1q_<supf><mode>"
7657   [(match_operand:MVE_2 0 "s_register_operand")
7658    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
7659   ]
7660   "TARGET_HAVE_MVE"
7661 {
7662   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
7663   DONE;
7664 })
7665
7666 ;;
7667 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
7668 ;;
7669 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
7670   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7671         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7672                       (match_operand:SI 2 "immediate_operand" "i")]
7673          VLDRDGBQ))
7674   ]
7675   "TARGET_HAVE_MVE"
7676 {
7677    rtx ops[3];
7678    ops[0] = operands[0];
7679    ops[1] = operands[1];
7680    ops[2] = operands[2];
7681    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
7682    return "";
7683 }
7684   [(set_attr "length" "4")])
7685
7686 ;;
7687 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
7688 ;;
7689 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
7690   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7691         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7692                       (match_operand:SI 2 "immediate_operand" "i")
7693                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7694          VLDRDGBQ))
7695   ]
7696   "TARGET_HAVE_MVE"
7697 {
7698    rtx ops[3];
7699    ops[0] = operands[0];
7700    ops[1] = operands[1];
7701    ops[2] = operands[2];
7702    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
7703    return "";
7704 }
7705   [(set_attr "length" "8")])
7706
7707 ;;
7708 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
7709 ;;
7710 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
7711  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7712        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7713                      (match_operand:V2DI 2 "s_register_operand" "w")]
7714         VLDRDGOQ))
7715  ]
7716  "TARGET_HAVE_MVE"
7717 {
7718   rtx ops[3];
7719   ops[0] = operands[0];
7720   ops[1] = operands[1];
7721   ops[2] = operands[2];
7722   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
7723   return "";
7724 }
7725  [(set_attr "length" "4")])
7726
7727 ;;
7728 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
7729 ;;
7730 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
7731  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7732        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7733                      (match_operand:V2DI 2 "s_register_operand" "w")
7734                      (match_operand:HI 3 "vpr_register_operand" "Up")]
7735         VLDRDGOQ))
7736  ]
7737  "TARGET_HAVE_MVE"
7738 {
7739   rtx ops[3];
7740   ops[0] = operands[0];
7741   ops[1] = operands[1];
7742   ops[2] = operands[2];
7743   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
7744   return "";
7745 }
7746  [(set_attr "length" "8")])
7747
7748 ;;
7749 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
7750 ;;
7751 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
7752   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7753         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7754                       (match_operand:V2DI 2 "s_register_operand" "w")]
7755          VLDRDGSOQ))
7756   ]
7757   "TARGET_HAVE_MVE"
7758 {
7759    rtx ops[3];
7760    ops[0] = operands[0];
7761    ops[1] = operands[1];
7762    ops[2] = operands[2];
7763    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7764    return "";
7765 }
7766   [(set_attr "length" "4")])
7767
7768 ;;
7769 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
7770 ;;
7771 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
7772   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7773         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7774                       (match_operand:V2DI 2 "s_register_operand" "w")
7775                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7776          VLDRDGSOQ))
7777   ]
7778   "TARGET_HAVE_MVE"
7779 {
7780    rtx ops[3];
7781    ops[0] = operands[0];
7782    ops[1] = operands[1];
7783    ops[2] = operands[2];
7784    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7785    return "";
7786 }
7787   [(set_attr "length" "8")])
7788
7789 ;;
7790 ;; [vldrhq_gather_offset_f]
7791 ;;
7792 (define_insn "mve_vldrhq_gather_offset_fv8hf"
7793   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7794         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7795                       (match_operand:V8HI 2 "s_register_operand" "w")]
7796          VLDRHQGO_F))
7797   ]
7798   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7799 {
7800    rtx ops[3];
7801    ops[0] = operands[0];
7802    ops[1] = operands[1];
7803    ops[2] = operands[2];
7804    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
7805    return "";
7806 }
7807   [(set_attr "length" "4")])
7808
7809 ;;
7810 ;; [vldrhq_gather_offset_z_f]
7811 ;;
7812 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
7813   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7814         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7815                       (match_operand:V8HI 2 "s_register_operand" "w")
7816                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7817          VLDRHQGO_F))
7818   ]
7819   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7820 {
7821    rtx ops[4];
7822    ops[0] = operands[0];
7823    ops[1] = operands[1];
7824    ops[2] = operands[2];
7825    ops[3] = operands[3];
7826    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
7827    return "";
7828 }
7829   [(set_attr "length" "8")])
7830
7831 ;;
7832 ;; [vldrhq_gather_shifted_offset_f]
7833 ;;
7834 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
7835   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7836         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7837                       (match_operand:V8HI 2 "s_register_operand" "w")]
7838          VLDRHQGSO_F))
7839   ]
7840   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7841 {
7842    rtx ops[3];
7843    ops[0] = operands[0];
7844    ops[1] = operands[1];
7845    ops[2] = operands[2];
7846    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7847    return "";
7848 }
7849   [(set_attr "length" "4")])
7850
7851 ;;
7852 ;; [vldrhq_gather_shifted_offset_z_f]
7853 ;;
7854 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
7855   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7856         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7857                       (match_operand:V8HI 2 "s_register_operand" "w")
7858                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7859          VLDRHQGSO_F))
7860   ]
7861   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7862 {
7863    rtx ops[4];
7864    ops[0] = operands[0];
7865    ops[1] = operands[1];
7866    ops[2] = operands[2];
7867    ops[3] = operands[3];
7868    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7869    return "";
7870 }
7871   [(set_attr "length" "8")])
7872
7873 ;;
7874 ;; [vldrwq_gather_base_f]
7875 ;;
7876 (define_insn "mve_vldrwq_gather_base_fv4sf"
7877   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7878         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7879                       (match_operand:SI 2 "immediate_operand" "i")]
7880          VLDRWQGB_F))
7881   ]
7882   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7883 {
7884    rtx ops[3];
7885    ops[0] = operands[0];
7886    ops[1] = operands[1];
7887    ops[2] = operands[2];
7888    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7889    return "";
7890 }
7891   [(set_attr "length" "4")])
7892
7893 ;;
7894 ;; [vldrwq_gather_base_z_f]
7895 ;;
7896 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
7897   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7898         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7899                       (match_operand:SI 2 "immediate_operand" "i")
7900                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7901          VLDRWQGB_F))
7902   ]
7903   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7904 {
7905    rtx ops[3];
7906    ops[0] = operands[0];
7907    ops[1] = operands[1];
7908    ops[2] = operands[2];
7909    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7910    return "";
7911 }
7912   [(set_attr "length" "8")])
7913
7914 ;;
7915 ;; [vldrwq_gather_offset_f]
7916 ;;
7917 (define_insn "mve_vldrwq_gather_offset_fv4sf"
7918   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7919         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7920                        (match_operand:V4SI 2 "s_register_operand" "w")]
7921          VLDRWQGO_F))
7922   ]
7923   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7924 {
7925    rtx ops[3];
7926    ops[0] = operands[0];
7927    ops[1] = operands[1];
7928    ops[2] = operands[2];
7929    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7930    return "";
7931 }
7932   [(set_attr "length" "4")])
7933
7934 ;;
7935 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
7936 ;;
7937 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
7938   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7939         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7940                        (match_operand:V4SI 2 "s_register_operand" "w")]
7941          VLDRWGOQ))
7942   ]
7943   "TARGET_HAVE_MVE"
7944 {
7945    rtx ops[3];
7946    ops[0] = operands[0];
7947    ops[1] = operands[1];
7948    ops[2] = operands[2];
7949    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7950    return "";
7951 }
7952   [(set_attr "length" "4")])
7953
7954 ;;
7955 ;; [vldrwq_gather_offset_z_f]
7956 ;;
7957 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
7958   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7959         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7960                       (match_operand:V4SI 2 "s_register_operand" "w")
7961                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7962          VLDRWQGO_F))
7963   ]
7964   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7965 {
7966    rtx ops[4];
7967    ops[0] = operands[0];
7968    ops[1] = operands[1];
7969    ops[2] = operands[2];
7970    ops[3] = operands[3];
7971    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7972    return "";
7973 }
7974   [(set_attr "length" "8")])
7975
7976 ;;
7977 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
7978 ;;
7979 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
7980   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7981         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7982                       (match_operand:V4SI 2 "s_register_operand" "w")
7983                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7984          VLDRWGOQ))
7985   ]
7986   "TARGET_HAVE_MVE"
7987 {
7988    rtx ops[4];
7989    ops[0] = operands[0];
7990    ops[1] = operands[1];
7991    ops[2] = operands[2];
7992    ops[3] = operands[3];
7993    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7994    return "";
7995 }
7996   [(set_attr "length" "8")])
7997
7998 ;;
7999 ;; [vldrwq_gather_shifted_offset_f]
8000 ;;
8001 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8002   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8003         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8004                       (match_operand:V4SI 2 "s_register_operand" "w")]
8005          VLDRWQGSO_F))
8006   ]
8007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8008 {
8009    rtx ops[3];
8010    ops[0] = operands[0];
8011    ops[1] = operands[1];
8012    ops[2] = operands[2];
8013    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8014    return "";
8015 }
8016   [(set_attr "length" "4")])
8017
8018 ;;
8019 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8020 ;;
8021 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8022   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8023         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8024                       (match_operand:V4SI 2 "s_register_operand" "w")]
8025          VLDRWGSOQ))
8026   ]
8027   "TARGET_HAVE_MVE"
8028 {
8029    rtx ops[3];
8030    ops[0] = operands[0];
8031    ops[1] = operands[1];
8032    ops[2] = operands[2];
8033    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8034    return "";
8035 }
8036   [(set_attr "length" "4")])
8037
8038 ;;
8039 ;; [vldrwq_gather_shifted_offset_z_f]
8040 ;;
8041 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8042   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8043         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8044                       (match_operand:V4SI 2 "s_register_operand" "w")
8045                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8046          VLDRWQGSO_F))
8047   ]
8048   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8049 {
8050    rtx ops[4];
8051    ops[0] = operands[0];
8052    ops[1] = operands[1];
8053    ops[2] = operands[2];
8054    ops[3] = operands[3];
8055    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8056    return "";
8057 }
8058   [(set_attr "length" "8")])
8059
8060 ;;
8061 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8062 ;;
8063 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8064   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8065         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8066                       (match_operand:V4SI 2 "s_register_operand" "w")
8067                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8068          VLDRWGSOQ))
8069   ]
8070   "TARGET_HAVE_MVE"
8071 {
8072    rtx ops[4];
8073    ops[0] = operands[0];
8074    ops[1] = operands[1];
8075    ops[2] = operands[2];
8076    ops[3] = operands[3];
8077    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8078    return "";
8079 }
8080   [(set_attr "length" "8")])
8081
8082 ;;
8083 ;; [vstrhq_f]
8084 ;;
8085 (define_insn "mve_vstrhq_fv8hf"
8086   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8087         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8088          VSTRHQ_F))
8089   ]
8090   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8091 {
8092    rtx ops[2];
8093    int regno = REGNO (operands[1]);
8094    ops[1] = gen_rtx_REG (TImode, regno);
8095    ops[0]  = operands[0];
8096    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8097    return "";
8098 }
8099   [(set_attr "length" "4")])
8100
8101 ;;
8102 ;; [vstrhq_p_f]
8103 ;;
8104 (define_insn "mve_vstrhq_p_fv8hf"
8105   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8106         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8107                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8108          VSTRHQ_F))
8109   ]
8110   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8111 {
8112    rtx ops[2];
8113    int regno = REGNO (operands[1]);
8114    ops[1] = gen_rtx_REG (TImode, regno);
8115    ops[0]  = operands[0];
8116    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8117    return "";
8118 }
8119   [(set_attr "length" "8")])
8120
8121 ;;
8122 ;; [vstrhq_p_s vstrhq_p_u]
8123 ;;
8124 (define_insn "mve_vstrhq_p_<supf><mode>"
8125   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8126         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8127                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8128          VSTRHQ))
8129   ]
8130   "TARGET_HAVE_MVE"
8131 {
8132    rtx ops[2];
8133    int regno = REGNO (operands[1]);
8134    ops[1] = gen_rtx_REG (TImode, regno);
8135    ops[0]  = operands[0];
8136    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8137    return "";
8138 }
8139   [(set_attr "length" "8")])
8140
8141 ;;
8142 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8143 ;;
8144 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8145   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8146    (match_operand:MVE_6 1 "s_register_operand")
8147    (match_operand:MVE_6 2 "s_register_operand")
8148    (match_operand:HI 3 "vpr_register_operand")
8149    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8150   "TARGET_HAVE_MVE"
8151 {
8152   rtx ind = XEXP (operands[0], 0);
8153   gcc_assert (REG_P (ind));
8154   emit_insn (
8155     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8156                                                        operands[2],
8157                                                        operands[3]));
8158   DONE;
8159 })
8160
8161 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8162   [(set (mem:BLK (scratch))
8163         (unspec:BLK
8164           [(match_operand:SI 0 "register_operand" "r")
8165            (match_operand:MVE_6 1 "s_register_operand" "w")
8166            (match_operand:MVE_6 2 "s_register_operand" "w")
8167            (match_operand:HI 3 "vpr_register_operand" "Up")]
8168           VSTRHSOQ))]
8169   "TARGET_HAVE_MVE"
8170   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8171   [(set_attr "length" "8")])
8172
8173 ;;
8174 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8175 ;;
8176 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8177   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8178    (match_operand:MVE_6 1 "s_register_operand")
8179    (match_operand:MVE_6 2 "s_register_operand")
8180    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8181   "TARGET_HAVE_MVE"
8182 {
8183   rtx ind = XEXP (operands[0], 0);
8184   gcc_assert (REG_P (ind));
8185   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8186                                                               operands[2]));
8187   DONE;
8188 })
8189
8190 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8191   [(set (mem:BLK (scratch))
8192         (unspec:BLK
8193           [(match_operand:SI 0 "register_operand" "r")
8194            (match_operand:MVE_6 1 "s_register_operand" "w")
8195            (match_operand:MVE_6 2 "s_register_operand" "w")]
8196           VSTRHSOQ))]
8197   "TARGET_HAVE_MVE"
8198   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8199   [(set_attr "length" "4")])
8200
8201 ;;
8202 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8203 ;;
8204 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8205   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8206    (match_operand:MVE_6 1 "s_register_operand")
8207    (match_operand:MVE_6 2 "s_register_operand")
8208    (match_operand:HI 3 "vpr_register_operand")
8209    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8210   "TARGET_HAVE_MVE"
8211 {
8212   rtx ind = XEXP (operands[0], 0);
8213   gcc_assert (REG_P (ind));
8214   emit_insn (
8215     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8216                                                                operands[2],
8217                                                                operands[3]));
8218   DONE;
8219 })
8220
8221 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8222   [(set (mem:BLK (scratch))
8223         (unspec:BLK
8224           [(match_operand:SI 0 "register_operand" "r")
8225            (match_operand:MVE_6 1 "s_register_operand" "w")
8226            (match_operand:MVE_6 2 "s_register_operand" "w")
8227            (match_operand:HI 3 "vpr_register_operand" "Up")]
8228           VSTRHSSOQ))]
8229   "TARGET_HAVE_MVE"
8230   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8231   [(set_attr "length" "8")])
8232
8233 ;;
8234 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8235 ;;
8236 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8237   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8238    (match_operand:MVE_6 1 "s_register_operand")
8239    (match_operand:MVE_6 2 "s_register_operand")
8240    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8241   "TARGET_HAVE_MVE"
8242 {
8243   rtx ind = XEXP (operands[0], 0);
8244   gcc_assert (REG_P (ind));
8245   emit_insn (
8246     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8247                                                              operands[2]));
8248   DONE;
8249 })
8250
8251 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8252   [(set (mem:BLK (scratch))
8253         (unspec:BLK
8254           [(match_operand:SI 0 "register_operand" "r")
8255            (match_operand:MVE_6 1 "s_register_operand" "w")
8256            (match_operand:MVE_6 2 "s_register_operand" "w")]
8257           VSTRHSSOQ))]
8258   "TARGET_HAVE_MVE"
8259   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8260   [(set_attr "length" "4")])
8261
8262 ;;
8263 ;; [vstrhq_s, vstrhq_u]
8264 ;;
8265 (define_insn "mve_vstrhq_<supf><mode>"
8266   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8267         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8268          VSTRHQ))
8269   ]
8270   "TARGET_HAVE_MVE"
8271 {
8272    rtx ops[2];
8273    int regno = REGNO (operands[1]);
8274    ops[1] = gen_rtx_REG (TImode, regno);
8275    ops[0]  = operands[0];
8276    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8277    return "";
8278 }
8279   [(set_attr "length" "4")])
8280
8281 ;;
8282 ;; [vstrwq_f]
8283 ;;
8284 (define_insn "mve_vstrwq_fv4sf"
8285   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
8286         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8287          VSTRWQ_F))
8288   ]
8289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8290 {
8291    rtx ops[2];
8292    int regno = REGNO (operands[1]);
8293    ops[1] = gen_rtx_REG (TImode, regno);
8294    ops[0]  = operands[0];
8295    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8296    return "";
8297 }
8298   [(set_attr "length" "4")])
8299
8300 ;;
8301 ;; [vstrwq_p_f]
8302 ;;
8303 (define_insn "mve_vstrwq_p_fv4sf"
8304   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
8305         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8306                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8307          VSTRWQ_F))
8308   ]
8309   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8310 {
8311    rtx ops[2];
8312    int regno = REGNO (operands[1]);
8313    ops[1] = gen_rtx_REG (TImode, regno);
8314    ops[0]  = operands[0];
8315    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8316    return "";
8317 }
8318   [(set_attr "length" "8")])
8319
8320 ;;
8321 ;; [vstrwq_p_s vstrwq_p_u]
8322 ;;
8323 (define_insn "mve_vstrwq_p_<supf>v4si"
8324   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
8325         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8326                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8327          VSTRWQ))
8328   ]
8329   "TARGET_HAVE_MVE"
8330 {
8331    rtx ops[2];
8332    int regno = REGNO (operands[1]);
8333    ops[1] = gen_rtx_REG (TImode, regno);
8334    ops[0]  = operands[0];
8335    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8336    return "";
8337 }
8338   [(set_attr "length" "8")])
8339
8340 ;;
8341 ;; [vstrwq_s vstrwq_u]
8342 ;;
8343 (define_insn "mve_vstrwq_<supf>v4si"
8344   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
8345         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8346          VSTRWQ))
8347   ]
8348   "TARGET_HAVE_MVE"
8349 {
8350    rtx ops[2];
8351    int regno = REGNO (operands[1]);
8352    ops[1] = gen_rtx_REG (TImode, regno);
8353    ops[0]  = operands[0];
8354    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8355    return "";
8356 }
8357   [(set_attr "length" "4")])
8358
8359 (define_expand "mve_vst1q_f<mode>"
8360   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8361    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8362   ]
8363   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8364 {
8365   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8366   DONE;
8367 })
8368
8369 (define_expand "mve_vst1q_<supf><mode>"
8370   [(match_operand:MVE_2 0 "mve_memory_operand")
8371    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8372   ]
8373   "TARGET_HAVE_MVE"
8374 {
8375   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8376   DONE;
8377 })
8378
8379 ;;
8380 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8381 ;;
8382 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8383   [(set (mem:BLK (scratch))
8384         (unspec:BLK
8385                 [(match_operand:V2DI 0 "s_register_operand" "w")
8386                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8387                  (match_operand:V2DI 2 "s_register_operand" "w")
8388                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8389          VSTRDSBQ))
8390   ]
8391   "TARGET_HAVE_MVE"
8392 {
8393    rtx ops[3];
8394    ops[0] = operands[0];
8395    ops[1] = operands[1];
8396    ops[2] = operands[2];
8397    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8398    return "";
8399 }
8400   [(set_attr "length" "8")])
8401
8402 ;;
8403 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8404 ;;
8405 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8406   [(set (mem:BLK (scratch))
8407         (unspec:BLK
8408                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8409                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8410                  (match_operand:V2DI 2 "s_register_operand" "w")]
8411          VSTRDSBQ))
8412   ]
8413   "TARGET_HAVE_MVE"
8414 {
8415    rtx ops[3];
8416    ops[0] = operands[0];
8417    ops[1] = operands[1];
8418    ops[2] = operands[2];
8419    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8420    return "";
8421 }
8422   [(set_attr "length" "4")])
8423
8424 ;;
8425 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8426 ;;
8427 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8428   [(match_operand:V2DI 0 "mve_scatter_memory")
8429    (match_operand:V2DI 1 "s_register_operand")
8430    (match_operand:V2DI 2 "s_register_operand")
8431    (match_operand:HI 3 "vpr_register_operand")
8432    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8433   "TARGET_HAVE_MVE"
8434 {
8435   rtx ind = XEXP (operands[0], 0);
8436   gcc_assert (REG_P (ind));
8437   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8438                                                               operands[2],
8439                                                               operands[3]));
8440   DONE;
8441 })
8442
8443 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8444   [(set (mem:BLK (scratch))
8445         (unspec:BLK
8446           [(match_operand:SI 0 "register_operand" "r")
8447            (match_operand:V2DI 1 "s_register_operand" "w")
8448            (match_operand:V2DI 2 "s_register_operand" "w")
8449            (match_operand:HI 3 "vpr_register_operand" "Up")]
8450           VSTRDSOQ))]
8451   "TARGET_HAVE_MVE"
8452   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8453   [(set_attr "length" "8")])
8454
8455 ;;
8456 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8457 ;;
8458 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8459   [(match_operand:V2DI 0 "mve_scatter_memory")
8460    (match_operand:V2DI 1 "s_register_operand")
8461    (match_operand:V2DI 2 "s_register_operand")
8462    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8463   "TARGET_HAVE_MVE"
8464 {
8465   rtx ind = XEXP (operands[0], 0);
8466   gcc_assert (REG_P (ind));
8467   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8468                                                             operands[2]));
8469   DONE;
8470 })
8471
8472 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8473   [(set (mem:BLK (scratch))
8474         (unspec:BLK
8475           [(match_operand:SI 0 "register_operand" "r")
8476            (match_operand:V2DI 1 "s_register_operand" "w")
8477            (match_operand:V2DI 2 "s_register_operand" "w")]
8478           VSTRDSOQ))]
8479   "TARGET_HAVE_MVE"
8480   "vstrd.64\t%q2, [%0, %q1]"
8481   [(set_attr "length" "4")])
8482
8483 ;;
8484 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8485 ;;
8486 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8487   [(match_operand:V2DI 0 "mve_scatter_memory")
8488    (match_operand:V2DI 1 "s_register_operand")
8489    (match_operand:V2DI 2 "s_register_operand")
8490    (match_operand:HI 3 "vpr_register_operand")
8491    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8492   "TARGET_HAVE_MVE"
8493 {
8494   rtx ind = XEXP (operands[0], 0);
8495   gcc_assert (REG_P (ind));
8496   emit_insn (
8497     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8498                                                              operands[2],
8499                                                              operands[3]));
8500   DONE;
8501 })
8502
8503 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8504   [(set (mem:BLK (scratch))
8505         (unspec:BLK
8506           [(match_operand:SI 0 "register_operand" "r")
8507            (match_operand:V2DI 1 "s_register_operand" "w")
8508            (match_operand:V2DI 2 "s_register_operand" "w")
8509            (match_operand:HI 3 "vpr_register_operand" "Up")]
8510           VSTRDSSOQ))]
8511   "TARGET_HAVE_MVE"
8512   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8513   [(set_attr "length" "8")])
8514
8515 ;;
8516 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8517 ;;
8518 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8519   [(match_operand:V2DI 0 "mve_scatter_memory")
8520    (match_operand:V2DI 1 "s_register_operand")
8521    (match_operand:V2DI 2 "s_register_operand")
8522    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8523   "TARGET_HAVE_MVE"
8524 {
8525   rtx ind = XEXP (operands[0], 0);
8526   gcc_assert (REG_P (ind));
8527   emit_insn (
8528     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8529                                                            operands[2]));
8530   DONE;
8531 })
8532
8533 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8534   [(set (mem:BLK (scratch))
8535         (unspec:BLK
8536           [(match_operand:SI 0 "register_operand" "r")
8537            (match_operand:V2DI 1 "s_register_operand" "w")
8538            (match_operand:V2DI 2 "s_register_operand" "w")]
8539           VSTRDSSOQ))]
8540   "TARGET_HAVE_MVE"
8541   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8542   [(set_attr "length" "4")])
8543
8544 ;;
8545 ;; [vstrhq_scatter_offset_f]
8546 ;;
8547 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8548   [(match_operand:V8HI 0 "mve_scatter_memory")
8549    (match_operand:V8HI 1 "s_register_operand")
8550    (match_operand:V8HF 2 "s_register_operand")
8551    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8552   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8553 {
8554   rtx ind = XEXP (operands[0], 0);
8555   gcc_assert (REG_P (ind));
8556   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8557                                                        operands[2]));
8558   DONE;
8559 })
8560
8561 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8562   [(set (mem:BLK (scratch))
8563         (unspec:BLK
8564           [(match_operand:SI 0 "register_operand" "r")
8565            (match_operand:V8HI 1 "s_register_operand" "w")
8566            (match_operand:V8HF 2 "s_register_operand" "w")]
8567           VSTRHQSO_F))]
8568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8569   "vstrh.16\t%q2, [%0, %q1]"
8570   [(set_attr "length" "4")])
8571
8572 ;;
8573 ;; [vstrhq_scatter_offset_p_f]
8574 ;;
8575 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8576   [(match_operand:V8HI 0 "mve_scatter_memory")
8577    (match_operand:V8HI 1 "s_register_operand")
8578    (match_operand:V8HF 2 "s_register_operand")
8579    (match_operand:HI 3 "vpr_register_operand")
8580    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8581   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8582 {
8583   rtx ind = XEXP (operands[0], 0);
8584   gcc_assert (REG_P (ind));
8585   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8586                                                          operands[2],
8587                                                          operands[3]));
8588   DONE;
8589 })
8590
8591 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8592   [(set (mem:BLK (scratch))
8593         (unspec:BLK
8594           [(match_operand:SI 0 "register_operand" "r")
8595            (match_operand:V8HI 1 "s_register_operand" "w")
8596            (match_operand:V8HF 2 "s_register_operand" "w")
8597            (match_operand:HI 3 "vpr_register_operand" "Up")]
8598           VSTRHQSO_F))]
8599   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8600   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8601   [(set_attr "length" "8")])
8602
8603 ;;
8604 ;; [vstrhq_scatter_shifted_offset_f]
8605 ;;
8606 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8607   [(match_operand:V8HI 0 "memory_operand" "=Us")
8608    (match_operand:V8HI 1 "s_register_operand" "w")
8609    (match_operand:V8HF 2 "s_register_operand" "w")
8610    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8611   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8612 {
8613   rtx ind = XEXP (operands[0], 0);
8614   gcc_assert (REG_P (ind));
8615   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8616                                                                operands[2]));
8617   DONE;
8618 })
8619
8620 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8621   [(set (mem:BLK (scratch))
8622         (unspec:BLK
8623           [(match_operand:SI 0 "register_operand" "r")
8624            (match_operand:V8HI 1 "s_register_operand" "w")
8625            (match_operand:V8HF 2 "s_register_operand" "w")]
8626           VSTRHQSSO_F))]
8627   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8628   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8629   [(set_attr "length" "4")])
8630
8631 ;;
8632 ;; [vstrhq_scatter_shifted_offset_p_f]
8633 ;;
8634 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8635   [(match_operand:V8HI 0 "memory_operand" "=Us")
8636    (match_operand:V8HI 1 "s_register_operand" "w")
8637    (match_operand:V8HF 2 "s_register_operand" "w")
8638    (match_operand:HI 3 "vpr_register_operand" "Up")
8639    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8640   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8641 {
8642   rtx ind = XEXP (operands[0], 0);
8643   gcc_assert (REG_P (ind));
8644   emit_insn (
8645     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8646                                                         operands[2],
8647                                                         operands[3]));
8648   DONE;
8649 })
8650
8651 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
8652   [(set (mem:BLK (scratch))
8653         (unspec:BLK
8654           [(match_operand:SI 0 "register_operand" "r")
8655            (match_operand:V8HI 1 "s_register_operand" "w")
8656            (match_operand:V8HF 2 "s_register_operand" "w")
8657            (match_operand:HI 3 "vpr_register_operand" "Up")]
8658           VSTRHQSSO_F))]
8659   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8660   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
8661   [(set_attr "length" "8")])
8662
8663 ;;
8664 ;; [vstrwq_scatter_base_f]
8665 ;;
8666 (define_insn "mve_vstrwq_scatter_base_fv4sf"
8667   [(set (mem:BLK (scratch))
8668         (unspec:BLK
8669                 [(match_operand:V4SI 0 "s_register_operand" "w")
8670                  (match_operand:SI 1 "immediate_operand" "i")
8671                  (match_operand:V4SF 2 "s_register_operand" "w")]
8672          VSTRWQSB_F))
8673   ]
8674   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8675 {
8676    rtx ops[3];
8677    ops[0] = operands[0];
8678    ops[1] = operands[1];
8679    ops[2] = operands[2];
8680    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
8681    return "";
8682 }
8683   [(set_attr "length" "4")])
8684
8685 ;;
8686 ;; [vstrwq_scatter_base_p_f]
8687 ;;
8688 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
8689   [(set (mem:BLK (scratch))
8690         (unspec:BLK
8691                 [(match_operand:V4SI 0 "s_register_operand" "w")
8692                  (match_operand:SI 1 "immediate_operand" "i")
8693                  (match_operand:V4SF 2 "s_register_operand" "w")
8694                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8695          VSTRWQSB_F))
8696   ]
8697   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8698 {
8699    rtx ops[3];
8700    ops[0] = operands[0];
8701    ops[1] = operands[1];
8702    ops[2] = operands[2];
8703    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8704    return "";
8705 }
8706   [(set_attr "length" "8")])
8707
8708 ;;
8709 ;; [vstrwq_scatter_offset_f]
8710 ;;
8711 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
8712   [(match_operand:V4SI 0 "mve_scatter_memory")
8713    (match_operand:V4SI 1 "s_register_operand")
8714    (match_operand:V4SF 2 "s_register_operand")
8715    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8716   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8717 {
8718   rtx ind = XEXP (operands[0], 0);
8719   gcc_assert (REG_P (ind));
8720   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
8721                                                        operands[2]));
8722   DONE;
8723 })
8724
8725 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
8726   [(set (mem:BLK (scratch))
8727         (unspec:BLK
8728           [(match_operand:SI 0 "register_operand" "r")
8729            (match_operand:V4SI 1 "s_register_operand" "w")
8730            (match_operand:V4SF 2 "s_register_operand" "w")]
8731           VSTRWQSO_F))]
8732   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8733   "vstrw.32\t%q2, [%0, %q1]"
8734   [(set_attr "length" "4")])
8735
8736 ;;
8737 ;; [vstrwq_scatter_offset_p_f]
8738 ;;
8739 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
8740   [(match_operand:V4SI 0 "mve_scatter_memory")
8741    (match_operand:V4SI 1 "s_register_operand")
8742    (match_operand:V4SF 2 "s_register_operand")
8743    (match_operand:HI 3 "vpr_register_operand")
8744    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8746 {
8747   rtx ind = XEXP (operands[0], 0);
8748   gcc_assert (REG_P (ind));
8749   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
8750                                                          operands[2],
8751                                                          operands[3]));
8752   DONE;
8753 })
8754
8755 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
8756   [(set (mem:BLK (scratch))
8757         (unspec:BLK
8758           [(match_operand:SI 0 "register_operand" "r")
8759            (match_operand:V4SI 1 "s_register_operand" "w")
8760            (match_operand:V4SF 2 "s_register_operand" "w")
8761            (match_operand:HI 3 "vpr_register_operand" "Up")]
8762           VSTRWQSO_F))]
8763   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8764   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8765   [(set_attr "length" "8")])
8766
8767 ;;
8768 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8769 ;;
8770 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
8771   [(match_operand:V4SI 0 "mve_scatter_memory")
8772    (match_operand:V4SI 1 "s_register_operand")
8773    (match_operand:V4SI 2 "s_register_operand")
8774    (match_operand:HI 3 "vpr_register_operand")
8775    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8776   "TARGET_HAVE_MVE"
8777 {
8778   rtx ind = XEXP (operands[0], 0);
8779   gcc_assert (REG_P (ind));
8780   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
8781                                                               operands[2],
8782                                                               operands[3]));
8783   DONE;
8784 })
8785
8786 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
8787   [(set (mem:BLK (scratch))
8788         (unspec:BLK
8789           [(match_operand:SI 0 "register_operand" "r")
8790            (match_operand:V4SI 1 "s_register_operand" "w")
8791            (match_operand:V4SI 2 "s_register_operand" "w")
8792            (match_operand:HI 3 "vpr_register_operand" "Up")]
8793           VSTRWSOQ))]
8794   "TARGET_HAVE_MVE"
8795   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8796   [(set_attr "length" "8")])
8797
8798 ;;
8799 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8800 ;;
8801 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
8802   [(match_operand:V4SI 0 "mve_scatter_memory")
8803    (match_operand:V4SI 1 "s_register_operand")
8804    (match_operand:V4SI 2 "s_register_operand")
8805    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8806   "TARGET_HAVE_MVE"
8807 {
8808   rtx ind = XEXP (operands[0], 0);
8809   gcc_assert (REG_P (ind));
8810   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
8811                                                             operands[2]));
8812   DONE;
8813 })
8814
8815 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
8816   [(set (mem:BLK (scratch))
8817         (unspec:BLK
8818           [(match_operand:SI 0 "register_operand" "r")
8819            (match_operand:V4SI 1 "s_register_operand" "w")
8820            (match_operand:V4SI 2 "s_register_operand" "w")]
8821           VSTRWSOQ))]
8822   "TARGET_HAVE_MVE"
8823   "vstrw.32\t%q2, [%0, %q1]"
8824   [(set_attr "length" "4")])
8825
8826 ;;
8827 ;; [vstrwq_scatter_shifted_offset_f]
8828 ;;
8829 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
8830   [(match_operand:V4SI 0 "mve_scatter_memory")
8831    (match_operand:V4SI 1 "s_register_operand")
8832    (match_operand:V4SF 2 "s_register_operand")
8833    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8835 {
8836   rtx ind = XEXP (operands[0], 0);
8837   gcc_assert (REG_P (ind));
8838   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
8839                                                                operands[2]));
8840   DONE;
8841 })
8842
8843 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
8844   [(set (mem:BLK (scratch))
8845         (unspec:BLK
8846           [(match_operand:SI 0 "register_operand" "r")
8847            (match_operand:V4SI 1 "s_register_operand" "w")
8848            (match_operand:V4SF 2 "s_register_operand" "w")]
8849          VSTRWQSSO_F))]
8850   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8851   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8852   [(set_attr "length" "8")])
8853
8854 ;;
8855 ;; [vstrwq_scatter_shifted_offset_p_f]
8856 ;;
8857 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
8858   [(match_operand:V4SI 0 "mve_scatter_memory")
8859    (match_operand:V4SI 1 "s_register_operand")
8860    (match_operand:V4SF 2 "s_register_operand")
8861    (match_operand:HI 3 "vpr_register_operand")
8862    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8863   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8864 {
8865   rtx ind = XEXP (operands[0], 0);
8866   gcc_assert (REG_P (ind));
8867   emit_insn (
8868     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
8869                                                         operands[2],
8870                                                         operands[3]));
8871   DONE;
8872 })
8873
8874 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
8875   [(set (mem:BLK (scratch))
8876         (unspec:BLK
8877           [(match_operand:SI 0 "register_operand" "r")
8878            (match_operand:V4SI 1 "s_register_operand" "w")
8879            (match_operand:V4SF 2 "s_register_operand" "w")
8880            (match_operand:HI 3 "vpr_register_operand" "Up")]
8881           VSTRWQSSO_F))]
8882   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8883   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8884   [(set_attr "length" "8")])
8885
8886 ;;
8887 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
8888 ;;
8889 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
8890   [(match_operand:V4SI 0 "mve_scatter_memory")
8891    (match_operand:V4SI 1 "s_register_operand")
8892    (match_operand:V4SI 2 "s_register_operand")
8893    (match_operand:HI 3 "vpr_register_operand")
8894    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8895   "TARGET_HAVE_MVE"
8896 {
8897   rtx ind = XEXP (operands[0], 0);
8898   gcc_assert (REG_P (ind));
8899   emit_insn (
8900     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
8901                                                              operands[2],
8902                                                              operands[3]));
8903   DONE;
8904 })
8905
8906 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
8907   [(set (mem:BLK (scratch))
8908         (unspec:BLK
8909           [(match_operand:SI 0 "register_operand" "r")
8910            (match_operand:V4SI 1 "s_register_operand" "w")
8911            (match_operand:V4SI 2 "s_register_operand" "w")
8912            (match_operand:HI 3 "vpr_register_operand" "Up")]
8913           VSTRWSSOQ))]
8914   "TARGET_HAVE_MVE"
8915   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8916   [(set_attr "length" "8")])
8917
8918 ;;
8919 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
8920 ;;
8921 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
8922   [(match_operand:V4SI 0 "mve_scatter_memory")
8923    (match_operand:V4SI 1 "s_register_operand")
8924    (match_operand:V4SI 2 "s_register_operand")
8925    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8926   "TARGET_HAVE_MVE"
8927 {
8928   rtx ind = XEXP (operands[0], 0);
8929   gcc_assert (REG_P (ind));
8930   emit_insn (
8931     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
8932                                                            operands[2]));
8933   DONE;
8934 })
8935
8936 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
8937   [(set (mem:BLK (scratch))
8938         (unspec:BLK
8939           [(match_operand:SI 0 "register_operand" "r")
8940            (match_operand:V4SI 1 "s_register_operand" "w")
8941            (match_operand:V4SI 2 "s_register_operand" "w")]
8942           VSTRWSSOQ))]
8943   "TARGET_HAVE_MVE"
8944   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8945   [(set_attr "length" "4")])
8946
8947 ;;
8948 ;; [vaddq_s, vaddq_u])
8949 ;;
8950 (define_insn "mve_vaddq<mode>"
8951   [
8952    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
8953         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
8954                     (match_operand:MVE_2 2 "s_register_operand" "w")))
8955   ]
8956   "TARGET_HAVE_MVE"
8957   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
8958   [(set_attr "type" "mve_move")
8959 ])
8960
8961 ;;
8962 ;; [vaddq_f])
8963 ;;
8964 (define_insn "mve_vaddq_f<mode>"
8965   [
8966    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8967         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
8968                     (match_operand:MVE_0 2 "s_register_operand" "w")))
8969   ]
8970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8971   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
8972   [(set_attr "type" "mve_move")
8973 ])
8974
8975 ;;
8976 ;; [vidupq_n_u])
8977 ;;
8978 (define_expand "mve_vidupq_n_u<mode>"
8979  [(match_operand:MVE_2 0 "s_register_operand")
8980   (match_operand:SI 1 "s_register_operand")
8981   (match_operand:SI 2 "mve_imm_selective_upto_8")]
8982  "TARGET_HAVE_MVE"
8983 {
8984   rtx temp = gen_reg_rtx (SImode);
8985   emit_move_insn (temp, operands[1]);
8986   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
8987   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
8988                                           operands[2], inc));
8989   DONE;
8990 })
8991
8992 ;;
8993 ;; [vidupq_u_insn])
8994 ;;
8995 (define_insn "mve_vidupq_u<mode>_insn"
8996  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8997        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
8998                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
8999          VIDUPQ))
9000   (set (match_operand:SI 1 "s_register_operand" "=Te")
9001        (plus:SI (match_dup 2)
9002                 (match_operand:SI 4 "immediate_operand" "i")))]
9003  "TARGET_HAVE_MVE"
9004  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9005
9006 ;;
9007 ;; [vidupq_m_n_u])
9008 ;;
9009 (define_expand "mve_vidupq_m_n_u<mode>"
9010   [(match_operand:MVE_2 0 "s_register_operand")
9011    (match_operand:MVE_2 1 "s_register_operand")
9012    (match_operand:SI 2 "s_register_operand")
9013    (match_operand:SI 3 "mve_imm_selective_upto_8")
9014    (match_operand:HI 4 "vpr_register_operand")]
9015   "TARGET_HAVE_MVE"
9016 {
9017   rtx temp = gen_reg_rtx (SImode);
9018   emit_move_insn (temp, operands[2]);
9019   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9020   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9021                                              operands[2], operands[3],
9022                                              operands[4], inc));
9023   DONE;
9024 })
9025
9026 ;;
9027 ;; [vidupq_m_wb_u_insn])
9028 ;;
9029 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9030  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9031        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9032                       (match_operand:SI 3 "s_register_operand" "2")
9033                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9034                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9035         VIDUPQ_M))
9036   (set (match_operand:SI 2 "s_register_operand" "=Te")
9037        (plus:SI (match_dup 3)
9038                 (match_operand:SI 6 "immediate_operand" "i")))]
9039  "TARGET_HAVE_MVE"
9040  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9041  [(set_attr "length""8")])
9042
9043 ;;
9044 ;; [vddupq_n_u])
9045 ;;
9046 (define_expand "mve_vddupq_n_u<mode>"
9047  [(match_operand:MVE_2 0 "s_register_operand")
9048   (match_operand:SI 1 "s_register_operand")
9049   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9050  "TARGET_HAVE_MVE"
9051 {
9052   rtx temp = gen_reg_rtx (SImode);
9053   emit_move_insn (temp, operands[1]);
9054   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9055   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9056                                           operands[2], inc));
9057   DONE;
9058 })
9059
9060 ;;
9061 ;; [vddupq_u_insn])
9062 ;;
9063 (define_insn "mve_vddupq_u<mode>_insn"
9064  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9065        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9066                       (match_operand:SI 3 "immediate_operand" "i")]
9067         VDDUPQ))
9068   (set (match_operand:SI 1 "s_register_operand" "=Te")
9069        (minus:SI (match_dup 2)
9070                  (match_operand:SI 4 "immediate_operand" "i")))]
9071  "TARGET_HAVE_MVE"
9072  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9073
9074 ;;
9075 ;; [vddupq_m_n_u])
9076 ;;
9077 (define_expand "mve_vddupq_m_n_u<mode>"
9078   [(match_operand:MVE_2 0 "s_register_operand")
9079    (match_operand:MVE_2 1 "s_register_operand")
9080    (match_operand:SI 2 "s_register_operand")
9081    (match_operand:SI 3 "mve_imm_selective_upto_8")
9082    (match_operand:HI 4 "vpr_register_operand")]
9083   "TARGET_HAVE_MVE"
9084 {
9085   rtx temp = gen_reg_rtx (SImode);
9086   emit_move_insn (temp, operands[2]);
9087   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9088   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9089                                              operands[2], operands[3],
9090                                              operands[4], inc));
9091   DONE;
9092 })
9093
9094 ;;
9095 ;; [vddupq_m_wb_u_insn])
9096 ;;
9097 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9098  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9099        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9100                       (match_operand:SI 3 "s_register_operand" "2")
9101                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9102                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9103         VDDUPQ_M))
9104   (set (match_operand:SI 2 "s_register_operand" "=Te")
9105        (minus:SI (match_dup 3)
9106                  (match_operand:SI 6 "immediate_operand" "i")))]
9107  "TARGET_HAVE_MVE"
9108  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9109  [(set_attr "length""8")])
9110
9111 ;;
9112 ;; [vdwdupq_n_u])
9113 ;;
9114 (define_expand "mve_vdwdupq_n_u<mode>"
9115  [(match_operand:MVE_2 0 "s_register_operand")
9116   (match_operand:SI 1 "s_register_operand")
9117   (match_operand:DI 2 "s_register_operand")
9118   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9119  "TARGET_HAVE_MVE"
9120 {
9121   rtx ignore_wb = gen_reg_rtx (SImode);
9122   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9123                                               operands[1], operands[2],
9124                                               operands[3]));
9125   DONE;
9126 })
9127
9128 ;;
9129 ;; [vdwdupq_wb_u])
9130 ;;
9131 (define_expand "mve_vdwdupq_wb_u<mode>"
9132  [(match_operand:SI 0 "s_register_operand")
9133   (match_operand:SI 1 "s_register_operand")
9134   (match_operand:DI 2 "s_register_operand")
9135   (match_operand:SI 3 "mve_imm_selective_upto_8")
9136   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9137  "TARGET_HAVE_MVE"
9138 {
9139   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9140   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9141                                               operands[1], operands[2],
9142                                               operands[3]));
9143   DONE;
9144 })
9145
9146 ;;
9147 ;; [vdwdupq_wb_u_insn])
9148 ;;
9149 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9150   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9151         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9152                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9153                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9154          VDWDUPQ))
9155    (set (match_operand:SI 1 "s_register_operand" "=Te")
9156         (unspec:SI [(match_dup 2)
9157                     (subreg:SI (match_dup 3) 4)
9158                     (match_dup 4)]
9159          VDWDUPQ))]
9160   "TARGET_HAVE_MVE"
9161   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9162 )
9163
9164 ;;
9165 ;; [vdwdupq_m_n_u])
9166 ;;
9167 (define_expand "mve_vdwdupq_m_n_u<mode>"
9168  [(match_operand:MVE_2 0 "s_register_operand")
9169   (match_operand:MVE_2 1 "s_register_operand")
9170   (match_operand:SI 2 "s_register_operand")
9171   (match_operand:DI 3 "s_register_operand")
9172   (match_operand:SI 4 "mve_imm_selective_upto_8")
9173   (match_operand:HI 5 "vpr_register_operand")]
9174  "TARGET_HAVE_MVE"
9175 {
9176   rtx ignore_wb = gen_reg_rtx (SImode);
9177   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9178                                                 operands[1], operands[2],
9179                                                 operands[3], operands[4],
9180                                                 operands[5]));
9181   DONE;
9182 })
9183
9184 ;;
9185 ;; [vdwdupq_m_wb_u])
9186 ;;
9187 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9188  [(match_operand:SI 0 "s_register_operand")
9189   (match_operand:MVE_2 1 "s_register_operand")
9190   (match_operand:SI 2 "s_register_operand")
9191   (match_operand:DI 3 "s_register_operand")
9192   (match_operand:SI 4 "mve_imm_selective_upto_8")
9193   (match_operand:HI 5 "vpr_register_operand")]
9194  "TARGET_HAVE_MVE"
9195 {
9196   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9197   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9198                                                 operands[1], operands[2],
9199                                                 operands[3], operands[4],
9200                                                 operands[5]));
9201   DONE;
9202 })
9203
9204 ;;
9205 ;; [vdwdupq_m_wb_u_insn])
9206 ;;
9207 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9208   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9209         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9210                        (match_operand:SI 3 "s_register_operand" "1")
9211                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9212                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9213                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9214          VDWDUPQ_M))
9215    (set (match_operand:SI 1 "s_register_operand" "=Te")
9216         (unspec:SI [(match_dup 2)
9217                     (match_dup 3)
9218                     (subreg:SI (match_dup 4) 4)
9219                     (match_dup 5)
9220                     (match_dup 6)]
9221          VDWDUPQ_M))
9222   ]
9223   "TARGET_HAVE_MVE"
9224   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9225   [(set_attr "type" "mve_move")
9226    (set_attr "length""8")])
9227
9228 ;;
9229 ;; [viwdupq_n_u])
9230 ;;
9231 (define_expand "mve_viwdupq_n_u<mode>"
9232  [(match_operand:MVE_2 0 "s_register_operand")
9233   (match_operand:SI 1 "s_register_operand")
9234   (match_operand:DI 2 "s_register_operand")
9235   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9236  "TARGET_HAVE_MVE"
9237 {
9238   rtx ignore_wb = gen_reg_rtx (SImode);
9239   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9240                                               operands[1], operands[2],
9241                                               operands[3]));
9242   DONE;
9243 })
9244
9245 ;;
9246 ;; [viwdupq_wb_u])
9247 ;;
9248 (define_expand "mve_viwdupq_wb_u<mode>"
9249  [(match_operand:SI 0 "s_register_operand")
9250   (match_operand:SI 1 "s_register_operand")
9251   (match_operand:DI 2 "s_register_operand")
9252   (match_operand:SI 3 "mve_imm_selective_upto_8")
9253   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9254  "TARGET_HAVE_MVE"
9255 {
9256   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9257   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9258                                               operands[1], operands[2],
9259                                               operands[3]));
9260   DONE;
9261 })
9262
9263 ;;
9264 ;; [viwdupq_wb_u_insn])
9265 ;;
9266 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9267   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9268         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9269                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9270                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9271          VIWDUPQ))
9272    (set (match_operand:SI 1 "s_register_operand" "=Te")
9273         (unspec:SI [(match_dup 2)
9274                     (subreg:SI (match_dup 3) 4)
9275                     (match_dup 4)]
9276          VIWDUPQ))]
9277   "TARGET_HAVE_MVE"
9278   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9279 )
9280
9281 ;;
9282 ;; [viwdupq_m_n_u])
9283 ;;
9284 (define_expand "mve_viwdupq_m_n_u<mode>"
9285  [(match_operand:MVE_2 0 "s_register_operand")
9286   (match_operand:MVE_2 1 "s_register_operand")
9287   (match_operand:SI 2 "s_register_operand")
9288   (match_operand:DI 3 "s_register_operand")
9289   (match_operand:SI 4 "mve_imm_selective_upto_8")
9290   (match_operand:HI 5 "vpr_register_operand")]
9291  "TARGET_HAVE_MVE"
9292 {
9293   rtx ignore_wb = gen_reg_rtx (SImode);
9294   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9295                                                 operands[1], operands[2],
9296                                                 operands[3], operands[4],
9297                                                 operands[5]));
9298   DONE;
9299 })
9300
9301 ;;
9302 ;; [viwdupq_m_wb_u])
9303 ;;
9304 (define_expand "mve_viwdupq_m_wb_u<mode>"
9305  [(match_operand:SI 0 "s_register_operand")
9306   (match_operand:MVE_2 1 "s_register_operand")
9307   (match_operand:SI 2 "s_register_operand")
9308   (match_operand:DI 3 "s_register_operand")
9309   (match_operand:SI 4 "mve_imm_selective_upto_8")
9310   (match_operand:HI 5 "vpr_register_operand")]
9311  "TARGET_HAVE_MVE"
9312 {
9313   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9314   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9315                                                 operands[1], operands[2],
9316                                                 operands[3], operands[4],
9317                                                 operands[5]));
9318   DONE;
9319 })
9320
9321 ;;
9322 ;; [viwdupq_m_wb_u_insn])
9323 ;;
9324 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9325   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9326         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9327                        (match_operand:SI 3 "s_register_operand" "1")
9328                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9329                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9330                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9331          VIWDUPQ_M))
9332    (set (match_operand:SI 1 "s_register_operand" "=Te")
9333         (unspec:SI [(match_dup 2)
9334                     (match_dup 3)
9335                     (subreg:SI (match_dup 4) 4)
9336                     (match_dup 5)
9337                     (match_dup 6)]
9338          VIWDUPQ_M))
9339   ]
9340   "TARGET_HAVE_MVE"
9341   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9342   [(set_attr "type" "mve_move")
9343    (set_attr "length""8")])
9344
9345 ;;
9346 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9347 ;;
9348 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9349   [(set (mem:BLK (scratch))
9350         (unspec:BLK
9351                 [(match_operand:V4SI 1 "s_register_operand" "0")
9352                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9353                  (match_operand:V4SI 3 "s_register_operand" "w")]
9354          VSTRWSBWBQ))
9355    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9356         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9357          VSTRWSBWBQ))
9358   ]
9359   "TARGET_HAVE_MVE"
9360 {
9361    rtx ops[3];
9362    ops[0] = operands[1];
9363    ops[1] = operands[2];
9364    ops[2] = operands[3];
9365    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9366    return "";
9367 }
9368   [(set_attr "length" "4")])
9369
9370 ;;
9371 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9372 ;;
9373 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9374  [(set (mem:BLK (scratch))
9375        (unspec:BLK
9376                 [(match_operand:V4SI 1 "s_register_operand" "0")
9377                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9378                  (match_operand:V4SI 3 "s_register_operand" "w")
9379                  (match_operand:HI 4 "vpr_register_operand")]
9380         VSTRWSBWBQ))
9381    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9382         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9383          VSTRWSBWBQ))
9384   ]
9385   "TARGET_HAVE_MVE"
9386 {
9387    rtx ops[3];
9388    ops[0] = operands[1];
9389    ops[1] = operands[2];
9390    ops[2] = operands[3];
9391    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9392    return "";
9393 }
9394   [(set_attr "length" "8")])
9395
9396 ;;
9397 ;; [vstrwq_scatter_base_wb_f]
9398 ;;
9399 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9400  [(set (mem:BLK (scratch))
9401        (unspec:BLK
9402                 [(match_operand:V4SI 1 "s_register_operand" "0")
9403                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9404                  (match_operand:V4SF 3 "s_register_operand" "w")]
9405          VSTRWQSBWB_F))
9406    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9407         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9408          VSTRWQSBWB_F))
9409   ]
9410   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9411 {
9412    rtx ops[3];
9413    ops[0] = operands[1];
9414    ops[1] = operands[2];
9415    ops[2] = operands[3];
9416    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9417    return "";
9418 }
9419   [(set_attr "length" "4")])
9420
9421 ;;
9422 ;; [vstrwq_scatter_base_wb_p_f]
9423 ;;
9424 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9425  [(set (mem:BLK (scratch))
9426        (unspec:BLK
9427                 [(match_operand:V4SI 1 "s_register_operand" "0")
9428                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9429                  (match_operand:V4SF 3 "s_register_operand" "w")
9430                  (match_operand:HI 4 "vpr_register_operand")]
9431         VSTRWQSBWB_F))
9432    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9433         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9434          VSTRWQSBWB_F))
9435   ]
9436   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9437 {
9438    rtx ops[3];
9439    ops[0] = operands[1];
9440    ops[1] = operands[2];
9441    ops[2] = operands[3];
9442    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9443    return "";
9444 }
9445   [(set_attr "length" "8")])
9446
9447 ;;
9448 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9449 ;;
9450 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9451   [(set (mem:BLK (scratch))
9452         (unspec:BLK
9453                 [(match_operand:V2DI 1 "s_register_operand" "0")
9454                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9455                  (match_operand:V2DI 3 "s_register_operand" "w")]
9456          VSTRDSBWBQ))
9457    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9458         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9459          VSTRDSBWBQ))
9460   ]
9461   "TARGET_HAVE_MVE"
9462 {
9463    rtx ops[3];
9464    ops[0] = operands[1];
9465    ops[1] = operands[2];
9466    ops[2] = operands[3];
9467    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9468    return "";
9469 }
9470   [(set_attr "length" "4")])
9471
9472 ;;
9473 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9474 ;;
9475 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9476   [(set (mem:BLK (scratch))
9477         (unspec:BLK
9478                 [(match_operand:V2DI 1 "s_register_operand" "0")
9479                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9480                  (match_operand:V2DI 3 "s_register_operand" "w")
9481                  (match_operand:HI 4 "vpr_register_operand")]
9482          VSTRDSBWBQ))
9483    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9484         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9485          VSTRDSBWBQ))
9486   ]
9487   "TARGET_HAVE_MVE"
9488 {
9489    rtx ops[3];
9490    ops[0] = operands[1];
9491    ops[1] = operands[2];
9492    ops[2] = operands[3];
9493    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9494    return "";
9495 }
9496   [(set_attr "length" "8")])
9497
9498 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9499   [(match_operand:V4SI 0 "s_register_operand")
9500    (match_operand:V4SI 1 "s_register_operand")
9501    (match_operand:SI 2 "mve_vldrd_immediate")
9502    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9503   "TARGET_HAVE_MVE"
9504 {
9505   rtx ignore_result = gen_reg_rtx (V4SImode);
9506   emit_insn (
9507   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9508                                                  operands[1], operands[2]));
9509   DONE;
9510 })
9511
9512 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9513   [(match_operand:V4SI 0 "s_register_operand")
9514    (match_operand:V4SI 1 "s_register_operand")
9515    (match_operand:SI 2 "mve_vldrd_immediate")
9516    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9517   "TARGET_HAVE_MVE"
9518 {
9519   rtx ignore_wb = gen_reg_rtx (V4SImode);
9520   emit_insn (
9521   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9522                                                  operands[1], operands[2]));
9523   DONE;
9524 })
9525
9526 ;;
9527 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9528 ;;
9529 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9530   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9531         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9532                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9533                       (mem:BLK (scratch))]
9534          VLDRWGBWBQ))
9535    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9536         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9537          VLDRWGBWBQ))
9538   ]
9539   "TARGET_HAVE_MVE"
9540 {
9541    rtx ops[3];
9542    ops[0] = operands[0];
9543    ops[1] = operands[2];
9544    ops[2] = operands[3];
9545    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9546    return "";
9547 }
9548   [(set_attr "length" "4")])
9549
9550 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9551   [(match_operand:V4SI 0 "s_register_operand")
9552    (match_operand:V4SI 1 "s_register_operand")
9553    (match_operand:SI 2 "mve_vldrd_immediate")
9554    (match_operand:HI 3 "vpr_register_operand")
9555    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9556   "TARGET_HAVE_MVE"
9557 {
9558   rtx ignore_result = gen_reg_rtx (V4SImode);
9559   emit_insn (
9560   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9561                                                    operands[1], operands[2],
9562                                                    operands[3]));
9563   DONE;
9564 })
9565 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9566   [(match_operand:V4SI 0 "s_register_operand")
9567    (match_operand:V4SI 1 "s_register_operand")
9568    (match_operand:SI 2 "mve_vldrd_immediate")
9569    (match_operand:HI 3 "vpr_register_operand")
9570    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9571   "TARGET_HAVE_MVE"
9572 {
9573   rtx ignore_wb = gen_reg_rtx (V4SImode);
9574   emit_insn (
9575   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9576                                                    operands[1], operands[2],
9577                                                    operands[3]));
9578   DONE;
9579 })
9580
9581 ;;
9582 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9583 ;;
9584 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9585   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9586         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9587                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9588                       (match_operand:HI 4 "vpr_register_operand" "Up")
9589                       (mem:BLK (scratch))]
9590          VLDRWGBWBQ))
9591    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9592         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9593          VLDRWGBWBQ))
9594   ]
9595   "TARGET_HAVE_MVE"
9596 {
9597    rtx ops[3];
9598    ops[0] = operands[0];
9599    ops[1] = operands[2];
9600    ops[2] = operands[3];
9601    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9602    return "";
9603 }
9604   [(set_attr "length" "8")])
9605
9606 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9607   [(match_operand:V4SI 0 "s_register_operand")
9608    (match_operand:V4SI 1 "s_register_operand")
9609    (match_operand:SI 2 "mve_vldrd_immediate")
9610    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9611   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9612 {
9613   rtx ignore_result = gen_reg_rtx (V4SFmode);
9614   emit_insn (
9615   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9616                                             operands[1], operands[2]));
9617   DONE;
9618 })
9619
9620 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9621   [(match_operand:V4SF 0 "s_register_operand")
9622    (match_operand:V4SI 1 "s_register_operand")
9623    (match_operand:SI 2 "mve_vldrd_immediate")
9624    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9625   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9626 {
9627   rtx ignore_wb = gen_reg_rtx (V4SImode);
9628   emit_insn (
9629   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9630                                             operands[1], operands[2]));
9631   DONE;
9632 })
9633
9634 ;;
9635 ;; [vldrwq_gather_base_wb_f]
9636 ;;
9637 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9638   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9639         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9640                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9641                       (mem:BLK (scratch))]
9642          VLDRWQGBWB_F))
9643    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9644         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9645          VLDRWQGBWB_F))
9646   ]
9647   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9648 {
9649    rtx ops[3];
9650    ops[0] = operands[0];
9651    ops[1] = operands[2];
9652    ops[2] = operands[3];
9653    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9654    return "";
9655 }
9656   [(set_attr "length" "4")])
9657
9658 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
9659   [(match_operand:V4SI 0 "s_register_operand")
9660    (match_operand:V4SI 1 "s_register_operand")
9661    (match_operand:SI 2 "mve_vldrd_immediate")
9662    (match_operand:HI 3 "vpr_register_operand")
9663    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9664   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9665 {
9666   rtx ignore_result = gen_reg_rtx (V4SFmode);
9667   emit_insn (
9668   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
9669                                               operands[1], operands[2],
9670                                               operands[3]));
9671   DONE;
9672 })
9673
9674 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
9675   [(match_operand:V4SF 0 "s_register_operand")
9676    (match_operand:V4SI 1 "s_register_operand")
9677    (match_operand:SI 2 "mve_vldrd_immediate")
9678    (match_operand:HI 3 "vpr_register_operand")
9679    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9680   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9681 {
9682   rtx ignore_wb = gen_reg_rtx (V4SImode);
9683   emit_insn (
9684   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
9685                                               operands[1], operands[2],
9686                                               operands[3]));
9687   DONE;
9688 })
9689
9690 ;;
9691 ;; [vldrwq_gather_base_wb_z_f]
9692 ;;
9693 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
9694   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9695         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9696                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9697                       (match_operand:HI 4 "vpr_register_operand" "Up")
9698                       (mem:BLK (scratch))]
9699          VLDRWQGBWB_F))
9700    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9701         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9702          VLDRWQGBWB_F))
9703   ]
9704   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9705 {
9706    rtx ops[3];
9707    ops[0] = operands[0];
9708    ops[1] = operands[2];
9709    ops[2] = operands[3];
9710    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9711    return "";
9712 }
9713   [(set_attr "length" "8")])
9714
9715 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
9716   [(match_operand:V2DI 0 "s_register_operand")
9717    (match_operand:V2DI 1 "s_register_operand")
9718    (match_operand:SI 2 "mve_vldrd_immediate")
9719    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9720   "TARGET_HAVE_MVE"
9721 {
9722   rtx ignore_result = gen_reg_rtx (V2DImode);
9723   emit_insn (
9724   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
9725                                                  operands[1], operands[2]));
9726   DONE;
9727 })
9728
9729 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
9730   [(match_operand:V2DI 0 "s_register_operand")
9731    (match_operand:V2DI 1 "s_register_operand")
9732    (match_operand:SI 2 "mve_vldrd_immediate")
9733    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9734   "TARGET_HAVE_MVE"
9735 {
9736   rtx ignore_wb = gen_reg_rtx (V2DImode);
9737   emit_insn (
9738   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
9739                                                  operands[1], operands[2]));
9740   DONE;
9741 })
9742
9743
9744 ;;
9745 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
9746 ;;
9747 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
9748   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9749         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9750                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9751                       (mem:BLK (scratch))]
9752          VLDRDGBWBQ))
9753    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9754         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9755          VLDRDGBWBQ))
9756   ]
9757   "TARGET_HAVE_MVE"
9758 {
9759    rtx ops[3];
9760    ops[0] = operands[0];
9761    ops[1] = operands[2];
9762    ops[2] = operands[3];
9763    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
9764    return "";
9765 }
9766   [(set_attr "length" "4")])
9767
9768 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
9769   [(match_operand:V2DI 0 "s_register_operand")
9770    (match_operand:V2DI 1 "s_register_operand")
9771    (match_operand:SI 2 "mve_vldrd_immediate")
9772    (match_operand:HI 3 "vpr_register_operand")
9773    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9774   "TARGET_HAVE_MVE"
9775 {
9776   rtx ignore_result = gen_reg_rtx (V2DImode);
9777   emit_insn (
9778   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
9779                                                    operands[1], operands[2],
9780                                                    operands[3]));
9781   DONE;
9782 })
9783
9784 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
9785   [(match_operand:V2DI 0 "s_register_operand")
9786    (match_operand:V2DI 1 "s_register_operand")
9787    (match_operand:SI 2 "mve_vldrd_immediate")
9788    (match_operand:HI 3 "vpr_register_operand")
9789    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9790   "TARGET_HAVE_MVE"
9791 {
9792   rtx ignore_wb = gen_reg_rtx (V2DImode);
9793   emit_insn (
9794   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
9795                                                    operands[1], operands[2],
9796                                                    operands[3]));
9797   DONE;
9798 })
9799
9800 (define_insn "get_fpscr_nzcvqc"
9801  [(set (match_operand:SI 0 "register_operand" "=r")
9802    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
9803  "TARGET_HAVE_MVE"
9804  "vmrs\\t%0, FPSCR_nzcvqc"
9805   [(set_attr "type" "mve_move")])
9806
9807 (define_insn "set_fpscr_nzcvqc"
9808  [(set (reg:SI VFPCC_REGNUM)
9809    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
9810     VUNSPEC_SET_FPSCR_NZCVQC))]
9811  "TARGET_HAVE_MVE"
9812  "vmsr\\tFPSCR_nzcvqc, %0"
9813   [(set_attr "type" "mve_move")])
9814
9815 ;;
9816 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
9817 ;;
9818 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
9819   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9820         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9821                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9822                       (match_operand:HI 4 "vpr_register_operand" "Up")
9823                       (mem:BLK (scratch))]
9824          VLDRDGBWBQ))
9825    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9826         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9827          VLDRDGBWBQ))
9828   ]
9829   "TARGET_HAVE_MVE"
9830 {
9831    rtx ops[3];
9832    ops[0] = operands[0];
9833    ops[1] = operands[2];
9834    ops[2] = operands[3];
9835    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
9836    return "";
9837 }
9838   [(set_attr "length" "8")])
9839 ;;
9840 ;; [vadciq_m_s, vadciq_m_u])
9841 ;;
9842 (define_insn "mve_vadciq_m_<supf>v4si"
9843   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9844         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9845                       (match_operand:V4SI 2 "s_register_operand" "w")
9846                       (match_operand:V4SI 3 "s_register_operand" "w")
9847                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9848          VADCIQ_M))
9849    (set (reg:SI VFPCC_REGNUM)
9850         (unspec:SI [(const_int 0)]
9851          VADCIQ_M))
9852   ]
9853   "TARGET_HAVE_MVE"
9854   "vpst\;vadcit.i32\t%q0, %q2, %q3"
9855   [(set_attr "type" "mve_move")
9856    (set_attr "length" "8")])
9857
9858 ;;
9859 ;; [vadciq_u, vadciq_s])
9860 ;;
9861 (define_insn "mve_vadciq_<supf>v4si"
9862   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9863         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9864                       (match_operand:V4SI 2 "s_register_operand" "w")]
9865          VADCIQ))
9866    (set (reg:SI VFPCC_REGNUM)
9867         (unspec:SI [(const_int 0)]
9868          VADCIQ))
9869   ]
9870   "TARGET_HAVE_MVE"
9871   "vadci.i32\t%q0, %q1, %q2"
9872   [(set_attr "type" "mve_move")
9873    (set_attr "length" "4")])
9874
9875 ;;
9876 ;; [vadcq_m_s, vadcq_m_u])
9877 ;;
9878 (define_insn "mve_vadcq_m_<supf>v4si"
9879   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9880         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9881                       (match_operand:V4SI 2 "s_register_operand" "w")
9882                       (match_operand:V4SI 3 "s_register_operand" "w")
9883                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9884          VADCQ_M))
9885    (set (reg:SI VFPCC_REGNUM)
9886         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9887          VADCQ_M))
9888   ]
9889   "TARGET_HAVE_MVE"
9890   "vpst\;vadct.i32\t%q0, %q2, %q3"
9891   [(set_attr "type" "mve_move")
9892    (set_attr "length" "8")])
9893
9894 ;;
9895 ;; [vadcq_u, vadcq_s])
9896 ;;
9897 (define_insn "mve_vadcq_<supf>v4si"
9898   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9899         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9900                        (match_operand:V4SI 2 "s_register_operand" "w")]
9901          VADCQ))
9902    (set (reg:SI VFPCC_REGNUM)
9903         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9904          VADCQ))
9905   ]
9906   "TARGET_HAVE_MVE"
9907   "vadc.i32\t%q0, %q1, %q2"
9908   [(set_attr "type" "mve_move")
9909    (set_attr "length" "4")
9910    (set_attr "conds" "set")])
9911
9912 ;;
9913 ;; [vsbciq_m_u, vsbciq_m_s])
9914 ;;
9915 (define_insn "mve_vsbciq_m_<supf>v4si"
9916   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9917         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9918                       (match_operand:V4SI 2 "s_register_operand" "w")
9919                       (match_operand:V4SI 3 "s_register_operand" "w")
9920                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9921          VSBCIQ_M))
9922    (set (reg:SI VFPCC_REGNUM)
9923         (unspec:SI [(const_int 0)]
9924          VSBCIQ_M))
9925   ]
9926   "TARGET_HAVE_MVE"
9927   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
9928   [(set_attr "type" "mve_move")
9929    (set_attr "length" "8")])
9930
9931 ;;
9932 ;; [vsbciq_s, vsbciq_u])
9933 ;;
9934 (define_insn "mve_vsbciq_<supf>v4si"
9935   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9936         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9937                       (match_operand:V4SI 2 "s_register_operand" "w")]
9938          VSBCIQ))
9939    (set (reg:SI VFPCC_REGNUM)
9940         (unspec:SI [(const_int 0)]
9941          VSBCIQ))
9942   ]
9943   "TARGET_HAVE_MVE"
9944   "vsbci.i32\t%q0, %q1, %q2"
9945   [(set_attr "type" "mve_move")
9946    (set_attr "length" "4")])
9947
9948 ;;
9949 ;; [vsbcq_m_u, vsbcq_m_s])
9950 ;;
9951 (define_insn "mve_vsbcq_m_<supf>v4si"
9952   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9953         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9954                       (match_operand:V4SI 2 "s_register_operand" "w")
9955                       (match_operand:V4SI 3 "s_register_operand" "w")
9956                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9957          VSBCQ_M))
9958    (set (reg:SI VFPCC_REGNUM)
9959         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9960          VSBCQ_M))
9961   ]
9962   "TARGET_HAVE_MVE"
9963   "vpst\;vsbct.i32\t%q0, %q2, %q3"
9964   [(set_attr "type" "mve_move")
9965    (set_attr "length" "8")])
9966
9967 ;;
9968 ;; [vsbcq_s, vsbcq_u])
9969 ;;
9970 (define_insn "mve_vsbcq_<supf>v4si"
9971   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9972         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9973                       (match_operand:V4SI 2 "s_register_operand" "w")]
9974          VSBCQ))
9975    (set (reg:SI VFPCC_REGNUM)
9976         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9977          VSBCQ))
9978   ]
9979   "TARGET_HAVE_MVE"
9980   "vsbc.i32\t%q0, %q1, %q2"
9981   [(set_attr "type" "mve_move")
9982    (set_attr "length" "4")])
9983
9984 ;;
9985 ;; [vst2q])
9986 ;;
9987 (define_insn "mve_vst2q<mode>"
9988   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
9989         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
9990                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9991          VST2Q))
9992   ]
9993   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9994    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9995 {
9996    rtx ops[4];
9997    int regno = REGNO (operands[1]);
9998    ops[0] = gen_rtx_REG (TImode, regno);
9999    ops[1] = gen_rtx_REG (TImode, regno + 4);
10000    rtx reg  = operands[0];
10001    while (reg && !REG_P (reg))
10002     reg = XEXP (reg, 0);
10003    gcc_assert (REG_P (reg));
10004    ops[2] = reg;
10005    ops[3] = operands[0];
10006    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10007                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10008    return "";
10009 }
10010   [(set_attr "length" "8")])
10011
10012 ;;
10013 ;; [vld2q])
10014 ;;
10015 (define_insn "mve_vld2q<mode>"
10016   [(set (match_operand:OI 0 "s_register_operand" "=w")
10017         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10018                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10019          VLD2Q))
10020   ]
10021   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10022    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10023 {
10024    rtx ops[4];
10025    int regno = REGNO (operands[0]);
10026    ops[0] = gen_rtx_REG (TImode, regno);
10027    ops[1] = gen_rtx_REG (TImode, regno + 4);
10028    rtx reg  = operands[1];
10029    while (reg && !REG_P (reg))
10030     reg = XEXP (reg, 0);
10031    gcc_assert (REG_P (reg));
10032    ops[2] = reg;
10033    ops[3] = operands[1];
10034    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10035                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10036    return "";
10037 }
10038   [(set_attr "length" "8")])
10039
10040 ;;
10041 ;; [vld4q])
10042 ;;
10043 (define_insn "mve_vld4q<mode>"
10044   [(set (match_operand:XI 0 "s_register_operand" "=w")
10045         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10046                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10047          VLD4Q))
10048   ]
10049   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10050    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10051 {
10052    rtx ops[6];
10053    int regno = REGNO (operands[0]);
10054    ops[0] = gen_rtx_REG (TImode, regno);
10055    ops[1] = gen_rtx_REG (TImode, regno+4);
10056    ops[2] = gen_rtx_REG (TImode, regno+8);
10057    ops[3] = gen_rtx_REG (TImode, regno + 12);
10058    rtx reg  = operands[1];
10059    while (reg && !REG_P (reg))
10060     reg = XEXP (reg, 0);
10061    gcc_assert (REG_P (reg));
10062    ops[4] = reg;
10063    ops[5] = operands[1];
10064    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10065                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10066                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10067                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10068    return "";
10069 }
10070   [(set_attr "length" "16")])
10071 ;;
10072 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10073 ;;
10074 (define_insn "mve_vec_extract<mode><V_elem_l>"
10075  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10076    (vec_select:<V_elem>
10077     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10078     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10079   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10080    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10081 {
10082   if (BYTES_BIG_ENDIAN)
10083     {
10084       int elt = INTVAL (operands[2]);
10085       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10086       operands[2] = GEN_INT (elt);
10087     }
10088   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10089 }
10090  [(set_attr "type" "mve_move")])
10091
10092 (define_insn "mve_vec_extractv2didi"
10093  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10094    (vec_select:DI
10095     (match_operand:V2DI 1 "s_register_operand" "w")
10096     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10097   "TARGET_HAVE_MVE"
10098 {
10099   int elt = INTVAL (operands[2]);
10100   if (BYTES_BIG_ENDIAN)
10101     elt = 1 - elt;
10102
10103   if (elt == 0)
10104    return "vmov\t%Q0, %R0, %e1";
10105   else
10106    return "vmov\t%Q0, %R0, %f1";
10107 }
10108  [(set_attr "type" "mve_move")])
10109
10110 (define_insn "*mve_vec_extract_sext_internal<mode>"
10111  [(set (match_operand:SI 0 "s_register_operand" "=r")
10112    (sign_extend:SI
10113     (vec_select:<V_elem>
10114      (match_operand:MVE_2 1 "s_register_operand" "w")
10115      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10116   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10117    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10118 {
10119   if (BYTES_BIG_ENDIAN)
10120     {
10121       int elt = INTVAL (operands[2]);
10122       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10123       operands[2] = GEN_INT (elt);
10124     }
10125   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10126 }
10127  [(set_attr "type" "mve_move")])
10128
10129 (define_insn "*mve_vec_extract_zext_internal<mode>"
10130  [(set (match_operand:SI 0 "s_register_operand" "=r")
10131    (zero_extend:SI
10132     (vec_select:<V_elem>
10133      (match_operand:MVE_2 1 "s_register_operand" "w")
10134      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10135   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10136    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10137 {
10138   if (BYTES_BIG_ENDIAN)
10139     {
10140       int elt = INTVAL (operands[2]);
10141       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10142       operands[2] = GEN_INT (elt);
10143     }
10144   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10145 }
10146  [(set_attr "type" "mve_move")])
10147
10148 ;;
10149 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10150 ;;
10151 (define_insn "mve_vec_set<mode>_internal"
10152  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10153        (vec_merge:VQ2
10154         (vec_duplicate:VQ2
10155           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10156         (match_operand:VQ2 3 "s_register_operand" "0")
10157         (match_operand:SI 2 "immediate_operand" "i")))]
10158   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10159    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10160 {
10161   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10162   if (BYTES_BIG_ENDIAN)
10163     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10164   operands[2] = GEN_INT (elt);
10165
10166   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10167 }
10168  [(set_attr "type" "mve_move")])
10169
10170 (define_insn "mve_vec_setv2di_internal"
10171  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10172        (vec_merge:V2DI
10173         (vec_duplicate:V2DI
10174           (match_operand:DI 1 "nonimmediate_operand" "r"))
10175         (match_operand:V2DI 3 "s_register_operand" "0")
10176         (match_operand:SI 2 "immediate_operand" "i")))]
10177  "TARGET_HAVE_MVE"
10178 {
10179   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10180   if (BYTES_BIG_ENDIAN)
10181     elt = 1 - elt;
10182
10183   if (elt == 0)
10184    return "vmov\t%e0, %Q1, %R1";
10185   else
10186    return "vmov\t%f0, %J1, %K1";
10187 }
10188  [(set_attr "type" "mve_move")])
10189
10190 ;;
10191 ;; [uqrshll_di]
10192 ;;
10193 (define_insn "mve_uqrshll_sat<supf>_di"
10194   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10195         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10196                     (match_operand:SI 2 "register_operand" "r")]
10197          UQRSHLLQ))]
10198   "TARGET_HAVE_MVE"
10199   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10200   [(set_attr "predicable" "yes")])
10201
10202 ;;
10203 ;; [sqrshrl_di]
10204 ;;
10205 (define_insn "mve_sqrshrl_sat<supf>_di"
10206   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10207         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10208                     (match_operand:SI 2 "register_operand" "r")]
10209          SQRSHRLQ))]
10210   "TARGET_HAVE_MVE"
10211   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10212   [(set_attr "predicable" "yes")])
10213
10214 ;;
10215 ;; [uqrshl_si]
10216 ;;
10217 (define_insn "mve_uqrshl_si"
10218   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10219         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10220                     (match_operand:SI 2 "register_operand" "r")]
10221          UQRSHL))]
10222   "TARGET_HAVE_MVE"
10223   "uqrshl%?\\t%1, %2"
10224   [(set_attr "predicable" "yes")])
10225
10226 ;;
10227 ;; [sqrshr_si]
10228 ;;
10229 (define_insn "mve_sqrshr_si"
10230   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10231         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10232                     (match_operand:SI 2 "register_operand" "r")]
10233          SQRSHR))]
10234   "TARGET_HAVE_MVE"
10235   "sqrshr%?\\t%1, %2"
10236   [(set_attr "predicable" "yes")])
10237
10238 ;;
10239 ;; [uqshll_di]
10240 ;;
10241 (define_insn "mve_uqshll_di"
10242   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10243         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10244                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10245   "TARGET_HAVE_MVE"
10246   "uqshll%?\\t%Q1, %R1, %2"
10247   [(set_attr "predicable" "yes")])
10248
10249 ;;
10250 ;; [urshrl_di]
10251 ;;
10252 (define_insn "mve_urshrl_di"
10253   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10254         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10255                     (match_operand:SI 2 "immediate_operand" "Pg")]
10256          URSHRL))]
10257   "TARGET_HAVE_MVE"
10258   "urshrl%?\\t%Q1, %R1, %2"
10259   [(set_attr "predicable" "yes")])
10260
10261 ;;
10262 ;; [uqshl_si]
10263 ;;
10264 (define_insn "mve_uqshl_si"
10265   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10266         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10267                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10268   "TARGET_HAVE_MVE"
10269   "uqshl%?\\t%1, %2"
10270   [(set_attr "predicable" "yes")])
10271
10272 ;;
10273 ;; [urshr_si]
10274 ;;
10275 (define_insn "mve_urshr_si"
10276   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10277         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10278                     (match_operand:SI 2 "immediate_operand" "Pg")]
10279          URSHR))]
10280   "TARGET_HAVE_MVE"
10281   "urshr%?\\t%1, %2"
10282   [(set_attr "predicable" "yes")])
10283
10284 ;;
10285 ;; [sqshl_si]
10286 ;;
10287 (define_insn "mve_sqshl_si"
10288   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10289         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10290                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10291   "TARGET_HAVE_MVE"
10292   "sqshl%?\\t%1, %2"
10293   [(set_attr "predicable" "yes")])
10294
10295 ;;
10296 ;; [srshr_si]
10297 ;;
10298 (define_insn "mve_srshr_si"
10299   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10300         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10301                     (match_operand:SI 2 "immediate_operand" "Pg")]
10302          SRSHR))]
10303   "TARGET_HAVE_MVE"
10304   "srshr%?\\t%1, %2"
10305   [(set_attr "predicable" "yes")])
10306
10307 ;;
10308 ;; [srshrl_di]
10309 ;;
10310 (define_insn "mve_srshrl_di"
10311   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10312         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10313                     (match_operand:SI 2 "immediate_operand" "Pg")]
10314          SRSHRL))]
10315   "TARGET_HAVE_MVE"
10316   "srshrl%?\\t%Q1, %R1, %2"
10317   [(set_attr "predicable" "yes")])
10318
10319 ;;
10320 ;; [sqshll_di]
10321 ;;
10322 (define_insn "mve_sqshll_di"
10323   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10324         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10325                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10326   "TARGET_HAVE_MVE"
10327   "sqshll%?\\t%Q1, %R1, %2"
10328   [(set_attr "predicable" "yes")])
10329
10330 ;;
10331 ;; [vshlcq_m_u vshlcq_m_s]
10332 ;;
10333 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10334  [(match_operand:MVE_2 0 "s_register_operand")
10335   (match_operand:MVE_2 1 "s_register_operand")
10336   (match_operand:SI 2 "s_register_operand")
10337   (match_operand:SI 3 "mve_imm_32")
10338   (match_operand:HI 4 "vpr_register_operand")
10339   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10340  "TARGET_HAVE_MVE"
10341 {
10342   rtx ignore_wb = gen_reg_rtx (SImode);
10343   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10344                                             operands[2], operands[3],
10345                                             operands[4]));
10346   DONE;
10347 })
10348
10349 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10350  [(match_operand:SI 0 "s_register_operand")
10351   (match_operand:MVE_2 1 "s_register_operand")
10352   (match_operand:SI 2 "s_register_operand")
10353   (match_operand:SI 3 "mve_imm_32")
10354   (match_operand:HI 4 "vpr_register_operand")
10355   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10356  "TARGET_HAVE_MVE"
10357 {
10358   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10359   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10360                                             operands[1], operands[2],
10361                                             operands[3], operands[4]));
10362   DONE;
10363 })
10364
10365 (define_insn "mve_vshlcq_m_<supf><mode>"
10366  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10367        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10368                       (match_operand:SI 3 "s_register_operand" "1")
10369                       (match_operand:SI 4 "mve_imm_32" "Rf")
10370                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10371         VSHLCQ_M))
10372   (set (match_operand:SI  1 "s_register_operand" "=r")
10373        (unspec:SI [(match_dup 2)
10374                    (match_dup 3)
10375                    (match_dup 4)
10376                    (match_dup 5)]
10377         VSHLCQ_M))
10378  ]
10379  "TARGET_HAVE_MVE"
10380  "vpst\;vshlct\t%q0, %1, %4"
10381  [(set_attr "type" "mve_move")
10382   (set_attr "length" "8")])
10383
10384 ;; CDE instructions on MVE registers.
10385
10386 (define_insn "arm_vcx1qv16qi"
10387   [(set (match_operand:V16QI 0 "register_operand" "=t")
10388         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10389                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10390          UNSPEC_VCDE))]
10391   "TARGET_CDE && TARGET_HAVE_MVE"
10392   "vcx1\\tp%c1, %q0, #%c2"
10393   [(set_attr "type" "coproc")]
10394 )
10395
10396 (define_insn "arm_vcx1qav16qi"
10397   [(set (match_operand:V16QI 0 "register_operand" "=t")
10398         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10399                             (match_operand:V16QI 2 "register_operand" "0")
10400                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10401          UNSPEC_VCDEA))]
10402   "TARGET_CDE && TARGET_HAVE_MVE"
10403   "vcx1a\\tp%c1, %q0, #%c3"
10404   [(set_attr "type" "coproc")]
10405 )
10406
10407 (define_insn "arm_vcx2qv16qi"
10408   [(set (match_operand:V16QI 0 "register_operand" "=t")
10409         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10410                           (match_operand:V16QI 2 "register_operand" "t")
10411                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10412          UNSPEC_VCDE))]
10413   "TARGET_CDE && TARGET_HAVE_MVE"
10414   "vcx2\\tp%c1, %q0, %q2, #%c3"
10415   [(set_attr "type" "coproc")]
10416 )
10417
10418 (define_insn "arm_vcx2qav16qi"
10419   [(set (match_operand:V16QI 0 "register_operand" "=t")
10420         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10421                           (match_operand:V16QI 2 "register_operand" "0")
10422                           (match_operand:V16QI 3 "register_operand" "t")
10423                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10424          UNSPEC_VCDEA))]
10425   "TARGET_CDE && TARGET_HAVE_MVE"
10426   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10427   [(set_attr "type" "coproc")]
10428 )
10429
10430 (define_insn "arm_vcx3qv16qi"
10431   [(set (match_operand:V16QI 0 "register_operand" "=t")
10432         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10433                           (match_operand:V16QI 2 "register_operand" "t")
10434                           (match_operand:V16QI 3 "register_operand" "t")
10435                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10436          UNSPEC_VCDE))]
10437   "TARGET_CDE && TARGET_HAVE_MVE"
10438   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10439   [(set_attr "type" "coproc")]
10440 )
10441
10442 (define_insn "arm_vcx3qav16qi"
10443   [(set (match_operand:V16QI 0 "register_operand" "=t")
10444         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10445                           (match_operand:V16QI 2 "register_operand" "0")
10446                           (match_operand:V16QI 3 "register_operand" "t")
10447                           (match_operand:V16QI 4 "register_operand" "t")
10448                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10449          UNSPEC_VCDEA))]
10450   "TARGET_CDE && TARGET_HAVE_MVE"
10451   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10452   [(set_attr "type" "coproc")]
10453 )
10454
10455 (define_insn "arm_vcx1q<a>_p_v16qi"
10456   [(set (match_operand:V16QI 0 "register_operand" "=t")
10457         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10458                            (match_operand:V16QI 2 "register_operand" "0")
10459                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10460                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10461          CDE_VCX))]
10462   "TARGET_CDE && TARGET_HAVE_MVE"
10463   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10464   [(set_attr "type" "coproc")
10465    (set_attr "length" "8")]
10466 )
10467
10468 (define_insn "arm_vcx2q<a>_p_v16qi"
10469   [(set (match_operand:V16QI 0 "register_operand" "=t")
10470         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10471                           (match_operand:V16QI 2 "register_operand" "0")
10472                           (match_operand:V16QI 3 "register_operand" "t")
10473                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10474                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10475          CDE_VCX))]
10476   "TARGET_CDE && TARGET_HAVE_MVE"
10477   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10478   [(set_attr "type" "coproc")
10479    (set_attr "length" "8")]
10480 )
10481
10482 (define_insn "arm_vcx3q<a>_p_v16qi"
10483   [(set (match_operand:V16QI 0 "register_operand" "=t")
10484         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10485                           (match_operand:V16QI 2 "register_operand" "0")
10486                           (match_operand:V16QI 3 "register_operand" "t")
10487                           (match_operand:V16QI 4 "register_operand" "t")
10488                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10489                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10490          CDE_VCX))]
10491   "TARGET_CDE && TARGET_HAVE_MVE"
10492   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10493   [(set_attr "type" "coproc")
10494    (set_attr "length" "8")]
10495 )
10496
10497 (define_insn "*movmisalign<mode>_mve_store"
10498   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Ux")
10499         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10500          UNSPEC_MISALIGNED_ACCESS))]
10501   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10502     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10503    && !BYTES_BIG_ENDIAN && unaligned_access"
10504   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10505   [(set_attr "type" "mve_store")]
10506 )
10507
10508
10509 (define_insn "*movmisalign<mode>_mve_load"
10510   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10511         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Ux")]
10512          UNSPEC_MISALIGNED_ACCESS))]
10513   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10514     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10515    && !BYTES_BIG_ENDIAN && unaligned_access"
10516   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10517   [(set_attr "type" "mve_load")]
10518 )