gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2023 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w   , w,   r,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand"      " w,r,w,DnDm,UxUi,r,w, Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   switch (which_alternative)
  26     {
  27     case 0:  /* [w,w].  */
  28       return "vmov\t%q0, %q1";
  29
  30     case 1:  /* [w,r].  */
  31       return "vmov\t%e0, %Q1, %R1  %@ <mode>\;vmov\t%f0, %J1, %K1";
  32
  33     case 2:  /* [r,w].  */
  34       return "vmov\t%Q0, %R0, %e1  %@ <mode>\;vmov\t%J0, %K0, %f1";
  35
  36     case 3:  /* [w,DnDm].  */
  37       {
  38         int width, is_valid;
  39
  40         is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  41                                                   &operands[1], &width);
  42
  43         gcc_assert (is_valid);
  44
  45         if (width == 0)
  46           return "vmov.f32\t%q0, %1  %@ <mode>";
  47         else
  48           {
  49             const int templ_size = 40;
  50             static char templ[templ_size];
  51             if (snprintf (templ, templ_size,
  52                           "vmov.i%d\t%%q0, %%x1  %%@ <mode>", width)
  53                 > templ_size)
  54               abort ();
  55             return templ;
  56           }
  57       }
  58
  59     case 4:  /* [w,UxUi].  */
  60       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode
  61           || <MODE>mode == TImode)
  62         return "vldrw.u32\t%q0, %E1";
  63       else
  64         return "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1";
  65
  66     case 5:  /* [r,r].  */
  67       return output_move_quad (operands);
  68
  69     case 6:  /* [Ux,w].  */
  70       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode
  71           || <MODE>mode == TImode)
  72         return "vstrw.32\t%q1, %E0";
  73       else
  74         return "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0";
  75
  76     case 7:  /* [w,Ul].  */
  77         return output_move_neon (operands);
  78
  79     default:
  80       gcc_unreachable ();
  81       return "";
  82     }
  83 }
  84   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_store,mve_load")
  85    (set_attr "length" "4,8,8,4,4,8,4,8")
  86    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*")
  87    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*")])
  88
  89 (define_insn "*mve_vdup<mode>"
  90   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  91         (vec_duplicate:MVE_vecs
  92           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  93   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  94   "vdup.<V_sz_elem>\t%q0, %1"
  95   [(set_attr "length" "4")
  96    (set_attr "type" "mve_move")])
  97
  98 ;;
  99 ;; [vst4q])
 100 ;;
 101 (define_insn "mve_vst4q<mode>"
 102   [(set (match_operand:XI 0 "mve_struct_operand" "=Ug")
 103         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 104                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 105          VST4Q))
 106   ]
 107   "TARGET_HAVE_MVE"
 108 {
 109    rtx ops[6];
 110    int regno = REGNO (operands[1]);
 111    ops[0] = gen_rtx_REG (TImode, regno);
 112    ops[1] = gen_rtx_REG (TImode, regno+4);
 113    ops[2] = gen_rtx_REG (TImode, regno+8);
 114    ops[3] = gen_rtx_REG (TImode, regno+12);
 115    rtx reg  = operands[0];
 116    while (reg && !REG_P (reg))
 117     reg = XEXP (reg, 0);
 118    gcc_assert (REG_P (reg));
 119    ops[4] = reg;
 120    ops[5] = operands[0];
 121    /* Here in first three instructions data is stored to ops[4]'s location but
 122       in the fourth instruction data is stored to operands[0], this is to
 123       support the writeback.  */
 124    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 125                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 126                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 127                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 128    return "";
 129 }
 130   [(set_attr "length" "16")])
 131
 132 ;;
 133 ;; [vrndaq_f]
 134 ;; [vrndmq_f]
 135 ;; [vrndnq_f]
 136 ;; [vrndpq_f]
 137 ;; [vrndq_f]
 138 ;; [vrndxq_f]
 139 ;;
 140 (define_insn "@mve_<mve_insn>q_f<mode>"
 141   [
 142    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 143         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 144          MVE_FP_UNARY))
 145   ]
 146   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 147   "<mve_mnemo>.f%#<V_sz_elem>\t%q0, %q1"
 148   [(set_attr "type" "mve_move")
 149 ])
 150
 151 ;;
 152 ;; [vrev64q_f])
 153 ;;
 154 (define_insn "@mve_<mve_insn>q_f<mode>"
 155   [
 156    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 157         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 158          MVE_FP_VREV64Q_ONLY))
 159   ]
 160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 161   "<mve_insn>.%#<V_sz_elem>\t%q0, %q1"
 162   [(set_attr "type" "mve_move")
 163 ])
 164
 165 ;;
 166 ;; [vabsq_f]
 167 ;; [vnegq_f]
 168 ;;
 169 (define_insn "mve_v<absneg_str>q_f<mode>"
 170   [
 171    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 172         (ABSNEG:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 173   ]
 174   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 175   "v<absneg_str>.f%#<V_sz_elem>\t%q0, %q1"
 176   [(set_attr "type" "mve_move")
 177 ])
 178
 179 ;;
 180 ;; [vdupq_n_f])
 181 ;;
 182 (define_insn "@mve_<mve_insn>q_n_f<mode>"
 183   [
 184    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 185         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 186          MVE_FP_N_VDUPQ_ONLY))
 187   ]
 188   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 189   "<mve_insn>.%#<V_sz_elem>\t%q0, %1"
 190   [(set_attr "type" "mve_move")
 191 ])
 192
 193 ;;
 194 ;; [vrev32q_f])
 195 ;;
 196 (define_insn "@mve_<mve_insn>q_f<mode>"
 197   [
 198    (set (match_operand:MVE_V8HF 0 "s_register_operand" "=w")
 199         (unspec:MVE_V8HF [(match_operand:MVE_V8HF 1 "s_register_operand" "w")]
 200          MVE_FP_VREV32Q_ONLY))
 201   ]
 202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 203   "<mve_insn>.<V_sz_elem>\t%q0, %q1"
 204   [(set_attr "type" "mve_move")
 205 ])
 206 ;;
 207 ;; [vcvttq_f32_f16])
 208 ;;
 209 (define_insn "mve_vcvttq_f32_f16v4sf"
 210   [
 211    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 212         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 213          VCVTTQ_F32_F16))
 214   ]
 215   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 216   "vcvtt.f32.f16 %q0, %q1"
 217   [(set_attr "type" "mve_move")
 218 ])
 219
 220 ;;
 221 ;; [vcvtbq_f32_f16])
 222 ;;
 223 (define_insn "mve_vcvtbq_f32_f16v4sf"
 224   [
 225    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 226         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 227          VCVTBQ_F32_F16))
 228   ]
 229   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 230   "vcvtb.f32.f16 %q0, %q1"
 231   [(set_attr "type" "mve_move")
 232 ])
 233
 234 ;;
 235 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 236 ;;
 237 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 238   [
 239    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 240         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 241          VCVTQ_TO_F))
 242   ]
 243   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 244   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 245   [(set_attr "type" "mve_move")
 246 ])
 247
 248 ;;
 249 ;; [vrev64q_u, vrev64q_s])
 250 ;;
 251 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 252   [
 253    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 254         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 255          VREV64Q))
 256   ]
 257   "TARGET_HAVE_MVE"
 258   "<mve_insn>.%#<V_sz_elem>\t%q0, %q1"
 259   [(set_attr "type" "mve_move")
 260 ])
 261
 262 ;;
 263 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 264 ;;
 265 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 266   [
 267    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 268         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 269          VCVTQ_FROM_F))
 270   ]
 271   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 272   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 273   [(set_attr "type" "mve_move")
 274 ])
 275
 276 ;;
 277 ;; [vabsq_s]
 278 ;; [vnegq_s]
 279 ;;
 280 (define_insn "mve_v<absneg_str>q_s<mode>"
 281   [
 282    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 283         (ABSNEG:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 284   ]
 285   "TARGET_HAVE_MVE"
 286   "v<absneg_str>.s%#<V_sz_elem>\t%q0, %q1"
 287   [(set_attr "type" "mve_move")
 288 ])
 289
 290 ;;
 291 ;; [vmvnq_u, vmvnq_s])
 292 ;;
 293 (define_insn "mve_vmvnq_u<mode>"
 294   [
 295    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 296         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 297   ]
 298   "TARGET_HAVE_MVE"
 299   "vmvn\t%q0, %q1"
 300   [(set_attr "type" "mve_move")
 301 ])
 302 (define_expand "mve_vmvnq_s<mode>"
 303   [
 304    (set (match_operand:MVE_2 0 "s_register_operand")
 305         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 306   ]
 307   "TARGET_HAVE_MVE"
 308 )
 309
 310 ;;
 311 ;; [vdupq_n_u, vdupq_n_s])
 312 ;;
 313 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
 314   [
 315    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 316         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 317          VDUPQ_N))
 318   ]
 319   "TARGET_HAVE_MVE"
 320   "<mve_insn>.%#<V_sz_elem>\t%q0, %1"
 321   [(set_attr "type" "mve_move")
 322 ])
 323
 324 ;;
 325 ;; [vclzq_u, vclzq_s])
 326 ;;
 327 (define_insn "@mve_vclzq_s<mode>"
 328   [
 329    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 330         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 331   ]
 332   "TARGET_HAVE_MVE"
 333   "vclz.i%#<V_sz_elem>\t%q0, %q1"
 334   [(set_attr "type" "mve_move")
 335 ])
 336 (define_expand "mve_vclzq_u<mode>"
 337   [
 338    (set (match_operand:MVE_2 0 "s_register_operand")
 339         (clz:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 340   ]
 341   "TARGET_HAVE_MVE"
 342 )
 343
 344 ;;
 345 ;; [vclsq_s]
 346 ;; [vqabsq_s]
 347 ;; [vqnegq_s]
 348 ;;
 349 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 350   [
 351    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 352         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 353          MVE_INT_UNARY))
 354   ]
 355   "TARGET_HAVE_MVE"
 356   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1"
 357   [(set_attr "type" "mve_move")
 358 ])
 359
 360 ;;
 361 ;; [vaddvq_s, vaddvq_u])
 362 ;;
 363 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 364   [
 365    (set (match_operand:SI 0 "s_register_operand" "=Te")
 366         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 367          VADDVQ))
 368   ]
 369   "TARGET_HAVE_MVE"
 370   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q1"
 371   [(set_attr "type" "mve_move")
 372 ])
 373
 374 ;;
 375 ;; [vrev32q_u, vrev32q_s])
 376 ;;
 377 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 378   [
 379    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 380         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 381          VREV32Q))
 382   ]
 383   "TARGET_HAVE_MVE"
 384   "<mve_insn>.%#<V_sz_elem>\t%q0, %q1"
 385   [(set_attr "type" "mve_move")
 386 ])
 387
 388 ;;
 389 ;; [vmovlbq_s, vmovlbq_u]
 390 ;; [vmovltq_u, vmovltq_s]
 391 ;;
 392 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 393   [
 394    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 395         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 396          VMOVLxQ))
 397   ]
 398   "TARGET_HAVE_MVE"
 399   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1"
 400   [(set_attr "type" "mve_move")
 401 ])
 402
 403 ;;
 404 ;; [vcvtpq_s, vcvtpq_u])
 405 ;;
 406 (define_insn "mve_vcvtpq_<supf><mode>"
 407   [
 408    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 409         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 410          VCVTPQ))
 411   ]
 412   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 413   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 414   [(set_attr "type" "mve_move")
 415 ])
 416
 417 ;;
 418 ;; [vcvtnq_s, vcvtnq_u])
 419 ;;
 420 (define_insn "mve_vcvtnq_<supf><mode>"
 421   [
 422    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 423         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 424          VCVTNQ))
 425   ]
 426   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 427   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 428   [(set_attr "type" "mve_move")
 429 ])
 430
 431 ;;
 432 ;; [vcvtmq_s, vcvtmq_u])
 433 ;;
 434 (define_insn "mve_vcvtmq_<supf><mode>"
 435   [
 436    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 437         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 438          VCVTMQ))
 439   ]
 440   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 441   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 442   [(set_attr "type" "mve_move")
 443 ])
 444
 445 ;;
 446 ;; [vcvtaq_u, vcvtaq_s])
 447 ;;
 448 (define_insn "mve_vcvtaq_<supf><mode>"
 449   [
 450    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 451         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 452          VCVTAQ))
 453   ]
 454   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 455   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 456   [(set_attr "type" "mve_move")
 457 ])
 458
 459 ;;
 460 ;; [vmvnq_n_u, vmvnq_n_s])
 461 ;;
 462 (define_insn "mve_vmvnq_n_<supf><mode>"
 463   [
 464    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 465         (unspec:MVE_5 [(match_operand:<V_elem> 1 "immediate_operand" "i")]
 466          VMVNQ_N))
 467   ]
 468   "TARGET_HAVE_MVE"
 469   "vmvn.i%#<V_sz_elem>  %q0, %1"
 470   [(set_attr "type" "mve_move")
 471 ])
 472
 473 ;;
 474 ;; [vrev16q_u, vrev16q_s])
 475 ;;
 476 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 477   [
 478    (set (match_operand:MVE_V16QI 0 "s_register_operand" "=w")
 479         (unspec:MVE_V16QI [(match_operand:MVE_V16QI 1 "s_register_operand" "w")]
 480          VREV16Q))
 481   ]
 482   "TARGET_HAVE_MVE"
 483   "<mve_insn>.<V_sz_elem>\t%q0, %q1"
 484   [(set_attr "type" "mve_move")
 485 ])
 486
 487 ;;
 488 ;; [vaddlvq_s vaddlvq_u])
 489 ;;
 490 (define_insn "@mve_<mve_insn>q_<supf>v4si"
 491   [
 492    (set (match_operand:DI 0 "s_register_operand" "=r")
 493         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 494          VADDLVQ))
 495   ]
 496   "TARGET_HAVE_MVE"
 497   "<mve_insn>.<supf>32\t%Q0, %R0, %q1"
 498   [(set_attr "type" "mve_move")
 499 ])
 500
 501 ;;
 502 ;; [vctp8q vctp16q vctp32q vctp64q])
 503 ;;
 504 (define_insn "mve_vctp<MVE_vctp>q<MVE_vpred>"
 505   [
 506    (set (match_operand:MVE_7 0 "vpr_register_operand" "=Up")
 507         (unspec:MVE_7 [(match_operand:SI 1 "s_register_operand" "r")]
 508         VCTP))
 509   ]
 510   "TARGET_HAVE_MVE"
 511   "vctp.<MVE_vctp> %1"
 512   [(set_attr "type" "mve_move")
 513 ])
 514
 515 ;;
 516 ;; [vpnot])
 517 ;;
 518 (define_insn "mve_vpnotv16bi"
 519   [
 520    (set (match_operand:V16BI 0 "vpr_register_operand" "=Up")
 521         (unspec:V16BI [(match_operand:V16BI 1 "vpr_register_operand" "0")]
 522          VPNOT))
 523   ]
 524   "TARGET_HAVE_MVE"
 525   "vpnot"
 526   [(set_attr "type" "mve_move")
 527 ])
 528
 529 ;;
 530 ;; [vbrsrq_n_f])
 531 ;;
 532 (define_insn "mve_vbrsrq_n_f<mode>"
 533   [
 534    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 535         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 536                        (match_operand:SI 2 "s_register_operand" "r")]
 537          VBRSRQ_N_F))
 538   ]
 539   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 540   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 541   [(set_attr "type" "mve_move")
 542 ])
 543
 544 ;;
 545 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 546 ;;
 547 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 548   [
 549    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 550         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 551                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 552          VCVTQ_N_TO_F))
 553   ]
 554   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 555   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 556   [(set_attr "type" "mve_move")
 557 ])
 558
 559 ;; [vcreateq_f])
 560 ;;
 561 (define_insn "@mve_<mve_insn>q_f<mode>"
 562   [
 563    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 564         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 565                        (match_operand:DI 2 "s_register_operand" "r")]
 566          MVE_FP_CREATE_ONLY))
 567   ]
 568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 569   "vmov %q0[2], %q0[0], %Q1, %Q2\;vmov %q0[3], %q0[1], %R1, %R2"
 570   [(set_attr "type" "mve_move")
 571    (set_attr "length""8")])
 572
 573 ;;
 574 ;; [vcreateq_u, vcreateq_s])
 575 ;;
 576 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 577   [
 578    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 579         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 580                        (match_operand:DI 2 "s_register_operand" "r")]
 581          VCREATEQ))
 582   ]
 583   "TARGET_HAVE_MVE"
 584   "vmov %q0[2], %q0[0], %Q1, %Q2\;vmov %q0[3], %q0[1], %R1, %R2"
 585   [(set_attr "type" "mve_move")
 586    (set_attr "length""8")])
 587
 588 ;;
 589 ;; [vrshrq_n_s, vrshrq_n_u]
 590 ;; [vshrq_n_s, vshrq_n_u]
 591 ;;
 592 ;; Version that takes an immediate as operand 2.
 593 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
 594   [
 595    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 596         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 597                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 598          MVE_VSHRQ_N))
 599   ]
 600   "TARGET_HAVE_MVE"
 601   "<mve_insn>.<supf><V_sz_elem>\t%q0, %q1, %2"
 602   [(set_attr "type" "mve_move")
 603 ])
 604
 605 ;; Versions that take constant vectors as operand 2 (with all elements
 606 ;; equal).
 607 (define_insn "mve_vshrq_n_s<mode>_imm"
 608   [
 609    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 610         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 611                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 612   ]
 613   "TARGET_HAVE_MVE"
 614   {
 615     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 616                                         <MODE>mode,
 617                                         VALID_NEON_QREG_MODE (<MODE>mode),
 618                                         true);
 619   }
 620   [(set_attr "type" "mve_move")
 621 ])
 622 (define_insn "mve_vshrq_n_u<mode>_imm"
 623   [
 624    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 625         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 626                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 627   ]
 628   "TARGET_HAVE_MVE"
 629   {
 630     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 631                                         <MODE>mode,
 632                                         VALID_NEON_QREG_MODE (<MODE>mode),
 633                                         true);
 634   }
 635   [(set_attr "type" "mve_move")
 636 ])
 637
 638 ;;
 639 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 640 ;;
 641 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 642   [
 643    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 644         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 645                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 646          VCVTQ_N_FROM_F))
 647   ]
 648   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 649   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 650   [(set_attr "type" "mve_move")
 651 ])
 652
 653 ;;
 654 ;; [vaddlvq_p_s])
 655 ;;
 656 (define_insn "@mve_<mve_insn>q_p_<supf>v4si"
 657   [
 658    (set (match_operand:DI 0 "s_register_operand" "=r")
 659         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 660                     (match_operand:V4BI 2 "vpr_register_operand" "Up")]
 661          VADDLVQ_P))
 662   ]
 663   "TARGET_HAVE_MVE"
 664   "vpst\;<mve_insn>t.<supf>32\t%Q0, %R0, %q1"
 665   [(set_attr "type" "mve_move")
 666    (set_attr "length""8")])
 667
 668 ;;
 669 ;; [vcmpneq_, vcmpcsq_, vcmpeqq_, vcmpgeq_, vcmpgtq_, vcmphiq_, vcmpleq_, vcmpltq_])
 670 ;;
 671 (define_insn "@mve_vcmp<mve_cmp_op>q_<mode>"
 672   [
 673    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
 674         (MVE_COMPARISONS:<MVE_VPRED> (match_operand:MVE_2 1 "s_register_operand" "w")
 675                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 676   ]
 677   "TARGET_HAVE_MVE"
 678   "vcmp.<mve_cmp_type>%#<V_sz_elem>\t<mve_cmp_op>, %q1, %q2"
 679   [(set_attr "type" "mve_move")
 680 ])
 681
 682 ;;
 683 ;; [vcmpcsq_n_, vcmpeqq_n_, vcmpgeq_n_, vcmpgtq_n_, vcmphiq_n_, vcmpleq_n_, vcmpltq_n_, vcmpneq_n_])
 684 ;;
 685 (define_insn "@mve_vcmp<mve_cmp_op>q_n_<mode>"
 686   [
 687    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
 688         (MVE_COMPARISONS:<MVE_VPRED>
 689          (match_operand:MVE_2 1 "s_register_operand" "w")
 690          (vec_duplicate:MVE_2 (match_operand:<V_elem> 2 "s_register_operand" "r"))))
 691   ]
 692   "TARGET_HAVE_MVE"
 693   "vcmp.<mve_cmp_type>%#<V_sz_elem>     <mve_cmp_op>, %q1, %2"
 694   [(set_attr "type" "mve_move")
 695 ])
 696
 697 ;;
 698 ;; [vshlq_s, vshlq_u])
 699 ;; See vec-common.md
 700
 701 ;;
 702 ;; [vabdq_s, vabdq_u]
 703 ;; [vhaddq_s, vhaddq_u]
 704 ;; [vhsubq_s, vhsubq_u]
 705 ;; [vmulhq_s, vmulhq_u]
 706 ;; [vqaddq_u, vqaddq_s]
 707 ;; [vqdmulhq_s]
 708 ;; [vqrdmulhq_s]
 709 ;; [vqrshlq_s, vqrshlq_u]
 710 ;; [vqshlq_s, vqshlq_u]
 711 ;; [vqsubq_u, vqsubq_s]
 712 ;; [vrhaddq_s, vrhaddq_u]
 713 ;; [vrmulhq_s, vrmulhq_u]
 714 ;; [vrshlq_s, vrshlq_u]
 715 ;;
 716 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 717   [
 718    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 719         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 720                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 721          MVE_INT_SU_BINARY))
 722   ]
 723   "TARGET_HAVE_MVE"
 724   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
 725   [(set_attr "type" "mve_move")
 726 ])
 727
 728 ;;
 729 ;; [vaddq_n_s, vaddq_n_u]
 730 ;; [vsubq_n_s, vsubq_n_u]
 731 ;; [vmulq_n_s, vmulq_n_u]
 732 ;;
 733 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
 734   [
 735    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 736         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 737                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 738          MVE_INT_N_BINARY))
 739   ]
 740   "TARGET_HAVE_MVE"
 741   "<mve_insn>.i%#<V_sz_elem>\t%q0, %q1, %2"
 742   [(set_attr "type" "mve_move")
 743 ])
 744
 745 ;;
 746 ;; [vaddvaq_s, vaddvaq_u])
 747 ;;
 748 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 749   [
 750    (set (match_operand:SI 0 "s_register_operand" "=Te")
 751         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 752                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 753          VADDVAQ))
 754   ]
 755   "TARGET_HAVE_MVE"
 756   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q2"
 757   [(set_attr "type" "mve_move")
 758 ])
 759
 760 ;;
 761 ;; [vaddvq_p_u, vaddvq_p_s])
 762 ;;
 763 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
 764   [
 765    (set (match_operand:SI 0 "s_register_operand" "=Te")
 766         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 767                     (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")]
 768          VADDVQ_P))
 769   ]
 770   "TARGET_HAVE_MVE"
 771   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q1"
 772   [(set_attr "type" "mve_move")
 773    (set_attr "length""8")])
 774
 775 ;;
 776 ;; [vandq_u, vandq_s])
 777 ;;
 778 ;; signed and unsigned versions are the same: define the unsigned
 779 ;; insn, and use an expander for the signed one as we still reference
 780 ;; both names from arm_mve.h.
 781 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 782 (define_insn "mve_vandq_u<mode>"
 783   [
 784    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 785         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 786                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 787   ]
 788   "TARGET_HAVE_MVE"
 789   "@
 790    vand\t%q0, %q1, %q2
 791    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 792   [(set_attr "type" "mve_move")
 793 ])
 794 (define_expand "mve_vandq_s<mode>"
 795   [
 796    (set (match_operand:MVE_2 0 "s_register_operand")
 797         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 798                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 799   ]
 800   "TARGET_HAVE_MVE"
 801 )
 802
 803 ;;
 804 ;; [vbicq_s, vbicq_u])
 805 ;;
 806 (define_insn "mve_vbicq_u<mode>"
 807   [
 808    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 809         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 810                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 811   ]
 812   "TARGET_HAVE_MVE"
 813   "vbic\t%q0, %q1, %q2"
 814   [(set_attr "type" "mve_move")
 815 ])
 816
 817 (define_expand "mve_vbicq_s<mode>"
 818   [
 819    (set (match_operand:MVE_2 0 "s_register_operand")
 820         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 821                    (match_operand:MVE_2 1 "s_register_operand")))
 822   ]
 823   "TARGET_HAVE_MVE"
 824 )
 825
 826 ;;
 827 ;; [vbrsrq_n_u, vbrsrq_n_s])
 828 ;;
 829 (define_insn "mve_vbrsrq_n_<supf><mode>"
 830   [
 831    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 832         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 833                        (match_operand:SI 2 "s_register_operand" "r")]
 834          VBRSRQ_N))
 835   ]
 836   "TARGET_HAVE_MVE"
 837   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 838   [(set_attr "type" "mve_move")
 839 ])
 840
 841 ;;
 842 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 843 ;;
 844 (define_insn "mve_vcaddq<mve_rot><mode>"
 845   [
 846    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 847         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 848                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 849          VCADD))
 850   ]
 851   "TARGET_HAVE_MVE"
 852   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 853   [(set_attr "type" "mve_move")
 854 ])
 855
 856 ;; Auto vectorizer pattern for int vcadd
 857 (define_expand "cadd<rot><mode>3"
 858   [(set (match_operand:MVE_2 0 "register_operand")
 859         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
 860                        (match_operand:MVE_2 2 "register_operand")]
 861           VCADD))]
 862   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
 863 )
 864
 865 ;;
 866 ;; [veorq_u, veorq_s])
 867 ;;
 868 (define_insn "mve_veorq_u<mode>"
 869   [
 870    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 871         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 872                    (match_operand:MVE_2 2 "s_register_operand" "w")))
 873   ]
 874   "TARGET_HAVE_MVE"
 875   "veor\t%q0, %q1, %q2"
 876   [(set_attr "type" "mve_move")
 877 ])
 878 (define_expand "mve_veorq_s<mode>"
 879   [
 880    (set (match_operand:MVE_2 0 "s_register_operand")
 881         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 882                    (match_operand:MVE_2 2 "s_register_operand")))
 883   ]
 884   "TARGET_HAVE_MVE"
 885 )
 886
 887 ;;
 888 ;; [vhaddq_n_u, vhaddq_n_s]
 889 ;; [vhsubq_n_u, vhsubq_n_s]
 890 ;; [vqaddq_n_s, vqaddq_n_u]
 891 ;; [vqdmulhq_n_s]
 892 ;; [vqrdmulhq_n_s]
 893 ;; [vqsubq_n_s, vqsubq_n_u]
 894 ;;
 895 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
 896   [
 897    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 898         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 899                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 900          MVE_INT_SU_N_BINARY))
 901   ]
 902   "TARGET_HAVE_MVE"
 903   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
 904   [(set_attr "type" "mve_move")
 905 ])
 906
 907 ;;
 908 ;; [vhcaddq_rot270_s])
 909 ;;
 910 (define_insn "mve_vhcaddq_rot270_s<mode>"
 911   [
 912    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 913         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 914                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 915          VHCADDQ_ROT270_S))
 916   ]
 917   "TARGET_HAVE_MVE"
 918   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
 919   [(set_attr "type" "mve_move")
 920 ])
 921
 922 ;;
 923 ;; [vhcaddq_rot90_s])
 924 ;;
 925 (define_insn "mve_vhcaddq_rot90_s<mode>"
 926   [
 927    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 928         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 929                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 930          VHCADDQ_ROT90_S))
 931   ]
 932   "TARGET_HAVE_MVE"
 933   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
 934   [(set_attr "type" "mve_move")
 935 ])
 936
 937 ;;
 938 ;; [vmaxaq_s]
 939 ;; [vminaq_s]
 940 ;;
 941 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 942   [
 943    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 944         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
 945                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 946          MVE_VMAXAVMINAQ))
 947   ]
 948   "TARGET_HAVE_MVE"
 949   "<mve_insn>.s%#<V_sz_elem>\t%q0, %q2"
 950   [(set_attr "type" "mve_move")
 951 ])
 952
 953 ;;
 954 ;; [vmaxq_u, vmaxq_s]
 955 ;; [vminq_s, vminq_u]
 956 ;;
 957 (define_insn "mve_<max_min_su_str>q_<max_min_supf><mode>"
 958   [
 959    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 960         (MAX_MIN_SU:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 961                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 962   ]
 963   "TARGET_HAVE_MVE"
 964   "<max_min_su_str>.<max_min_supf>%#<V_sz_elem>\t%q0, %q1, %q2"
 965   [(set_attr "type" "mve_move")
 966 ])
 967
 968
 969 ;;
 970 ;; [vmaxavq_s]
 971 ;; [vmaxvq_u, vmaxvq_s]
 972 ;; [vminavq_s]
 973 ;; [vminvq_u, vminvq_s]
 974 ;;
 975 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 976   [
 977    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
 978         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
 979                           (match_operand:MVE_2 2 "s_register_operand" "w")]
 980          MVE_VMAXVQ_VMINVQ))
 981   ]
 982   "TARGET_HAVE_MVE"
 983   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q2"
 984   [(set_attr "type" "mve_move")
 985 ])
 986
 987 ;;
 988 ;; [vmladavq_u, vmladavq_s]
 989 ;; [vmladavxq_s]
 990 ;; [vmlsdavq_s]
 991 ;; [vmlsdavxq_s]
 992 ;;
 993 (define_insn "@mve_<mve_insn>q_<supf><mode>"
 994   [
 995    (set (match_operand:SI 0 "s_register_operand" "=Te")
 996         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 997                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 998          MVE_VMLxDAVQ))
 999   ]
1000   "TARGET_HAVE_MVE"
1001   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1002   [(set_attr "type" "mve_move")
1003 ])
1004
1005 ;;
1006 ;; [vmullbq_int_u, vmullbq_int_s])
1007 ;;
1008 (define_insn "mve_vmullbq_int_<supf><mode>"
1009   [
1010    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1011         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1012                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1013          VMULLBQ_INT))
1014   ]
1015   "TARGET_HAVE_MVE"
1016   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1017   [(set_attr "type" "mve_move")
1018 ])
1019
1020 ;;
1021 ;; [vmulltq_int_u, vmulltq_int_s])
1022 ;;
1023 (define_insn "mve_vmulltq_int_<supf><mode>"
1024   [
1025    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1026         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1027                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1028          VMULLTQ_INT))
1029   ]
1030   "TARGET_HAVE_MVE"
1031   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1032   [(set_attr "type" "mve_move")
1033 ])
1034
1035 ;;
1036 ;; [vaddq_s, vaddq_u]
1037 ;; [vmulq_u, vmulq_s]
1038 ;; [vsubq_s, vsubq_u]
1039 ;;
1040 (define_insn "mve_<mve_addsubmul>q<mode>"
1041   [
1042    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1043         (MVE_INT_BINARY_RTX:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1044                               (match_operand:MVE_2 2 "s_register_operand" "w")))
1045   ]
1046   "TARGET_HAVE_MVE"
1047   "<mve_addsubmul>.i%#<V_sz_elem>\t%q0, %q1, %q2"
1048   [(set_attr "type" "mve_move")
1049 ])
1050
1051 ;;
1052 ;; [vornq_u, vornq_s])
1053 ;;
1054 (define_insn "mve_vornq_s<mode>"
1055   [
1056    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1057         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1058                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1059   ]
1060   "TARGET_HAVE_MVE"
1061    "vorn\t%q0, %q1, %q2"
1062   [(set_attr "type" "mve_move")
1063 ])
1064
1065 (define_expand "mve_vornq_u<mode>"
1066   [
1067    (set (match_operand:MVE_2 0 "s_register_operand")
1068         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1069                    (match_operand:MVE_2 1 "s_register_operand")))
1070   ]
1071   "TARGET_HAVE_MVE"
1072 )
1073
1074 ;;
1075 ;; [vorrq_s, vorrq_u])
1076 ;;
1077 ;; signed and unsigned versions are the same: define the unsigned
1078 ;; insn, and use an expander for the signed one as we still reference
1079 ;; both names from arm_mve.h.
1080 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1081 (define_insn "mve_vorrq_s<mode>"
1082   [
1083    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1084         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1085                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1086   ]
1087   "TARGET_HAVE_MVE"
1088   "@
1089    vorr\t%q0, %q1, %q2
1090    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1091   [(set_attr "type" "mve_move")
1092 ])
1093 (define_expand "mve_vorrq_u<mode>"
1094   [
1095    (set (match_operand:MVE_2 0 "s_register_operand")
1096         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1097                    (match_operand:MVE_2 2 "neon_logic_op2")))
1098   ]
1099   "TARGET_HAVE_MVE"
1100 )
1101
1102 ;;
1103 ;; [vqrshlq_n_s, vqrshlq_n_u]
1104 ;; [vrshlq_n_u, vrshlq_n_s]
1105 ;;
1106 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1107   [
1108    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1109         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1110                        (match_operand:SI 2 "s_register_operand" "r")]
1111          MVE_RSHIFT_N))
1112   ]
1113   "TARGET_HAVE_MVE"
1114   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %2"
1115   [(set_attr "type" "mve_move")
1116 ])
1117
1118 ;;
1119 ;; [vqshlq_n_s, vqshlq_n_u]
1120 ;; [vshlq_n_u, vshlq_n_s]
1121 ;;
1122 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1123   [
1124    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1125         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1126                        (match_operand:SI 2 "immediate_operand" "i")]
1127          MVE_SHIFT_N))
1128   ]
1129   "TARGET_HAVE_MVE"
1130   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1131   [(set_attr "type" "mve_move")
1132 ])
1133
1134 ;;
1135 ;; [vqshlq_r_u, vqshlq_r_s]
1136 ;; [vshlq_r_s, vshlq_r_u]
1137 ;;
1138 (define_insn "@mve_<mve_insn>q_r_<supf><mode>"
1139   [
1140    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1141         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1142                        (match_operand:SI 2 "s_register_operand" "r")]
1143          MVE_SHIFT_R))
1144   ]
1145   "TARGET_HAVE_MVE"
1146   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %2"
1147   [(set_attr "type" "mve_move")
1148 ])
1149
1150 ;;
1151 ;; [vqshluq_n_s])
1152 ;;
1153 (define_insn "mve_vqshluq_n_s<mode>"
1154   [
1155    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1156         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1157                        (match_operand:SI 2 "<MVE_pred>" "<MVE_constraint>")]
1158          VQSHLUQ_N_S))
1159   ]
1160   "TARGET_HAVE_MVE"
1161   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1162   [(set_attr "type" "mve_move")
1163 ])
1164
1165 ;;
1166 ;; [vabdq_f]
1167 ;;
1168 (define_insn "@mve_<mve_insn>q_f<mode>"
1169   [
1170    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1171         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1172                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1173          MVE_FP_VABDQ_ONLY))
1174   ]
1175   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1176   "<mve_insn>.f%#<V_sz_elem>\t%q0, %q1, %q2"
1177   [(set_attr "type" "mve_move")
1178 ])
1179
1180 ;;
1181 ;; [vaddlvaq_s vaddlvaq_u])
1182 ;;
1183 (define_insn "@mve_<mve_insn>q_<supf>v4si"
1184   [
1185    (set (match_operand:DI 0 "s_register_operand" "=r")
1186         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1187                     (match_operand:V4SI 2 "s_register_operand" "w")]
1188          VADDLVAQ))
1189   ]
1190   "TARGET_HAVE_MVE"
1191   "<mve_insn>.<supf>32\t%Q0, %R0, %q2"
1192   [(set_attr "type" "mve_move")
1193 ])
1194
1195 ;;
1196 ;; [vaddq_n_f]
1197 ;; [vsubq_n_f]
1198 ;; [vmulq_n_f]
1199 ;;
1200 (define_insn "@mve_<mve_insn>q_n_f<mode>"
1201   [
1202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1204                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1205          MVE_FP_N_BINARY))
1206   ]
1207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1208   "<mve_insn>.f%#<V_sz_elem>\t%q0, %q1, %2"
1209   [(set_attr "type" "mve_move")
1210 ])
1211
1212 ;;
1213 ;; [vandq_f])
1214 ;;
1215 (define_insn "mve_vandq_f<mode>"
1216   [
1217    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1218         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1219                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1220   ]
1221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1222   "vand %q0, %q1, %q2"
1223   [(set_attr "type" "mve_move")
1224 ])
1225
1226 ;;
1227 ;; [vbicq_f])
1228 ;;
1229 (define_insn "mve_vbicq_f<mode>"
1230   [
1231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1232         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
1233                               (match_operand:MVE_0 2 "s_register_operand" "w")))
1234   ]
1235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1236   "vbic %q0, %q1, %q2"
1237   [(set_attr "type" "mve_move")
1238 ])
1239
1240 ;;
1241 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
1242 ;;
1243 (define_insn "mve_vcaddq<mve_rot><mode>"
1244   [
1245    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1246         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1247                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1248          VCADD))
1249   ]
1250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1251   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1252   [(set_attr "type" "mve_move")
1253 ])
1254
1255 ;;
1256 ;; [vcmpeqq_f, vcmpgeq_f, vcmpgtq_f, vcmpleq_f, vcmpltq_f, vcmpneq_f])
1257 ;;
1258 (define_insn "@mve_vcmp<mve_cmp_op>q_f<mode>"
1259   [
1260    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
1261         (MVE_FP_COMPARISONS:<MVE_VPRED> (match_operand:MVE_0 1 "s_register_operand" "w")
1262                                (match_operand:MVE_0 2 "s_register_operand" "w")))
1263   ]
1264   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1265   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
1266   [(set_attr "type" "mve_move")
1267 ])
1268
1269 ;;
1270 ;; [vcmpeqq_n_f, vcmpgeq_n_f, vcmpgtq_n_f, vcmpleq_n_f, vcmpltq_n_f, vcmpneq_n_f])
1271 ;;
1272 (define_insn "@mve_vcmp<mve_cmp_op>q_n_f<mode>"
1273   [
1274    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
1275         (MVE_FP_COMPARISONS:<MVE_VPRED>
1276          (match_operand:MVE_0 1 "s_register_operand" "w")
1277          (vec_duplicate:MVE_0 (match_operand:<V_elem> 2 "s_register_operand" "r"))))
1278   ]
1279   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1280   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %2"
1281   [(set_attr "type" "mve_move")
1282 ])
1283
1284 ;;
1285 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
1286 ;;
1287 (define_insn "mve_vcmulq<mve_rot><mode>"
1288   [
1289    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1290         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1291                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1292          VCMUL))
1293   ]
1294   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1295   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1296   [(set_attr "type" "mve_move")
1297 ])
1298
1299 ;;
1300 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
1301 ;;
1302 (define_insn "mve_vctp<MVE_vctp>q_m<MVE_vpred>"
1303   [
1304    (set (match_operand:MVE_7 0 "vpr_register_operand" "=Up")
1305         (unspec:MVE_7 [(match_operand:SI 1 "s_register_operand" "r")
1306                     (match_operand:MVE_7 2 "vpr_register_operand" "Up")]
1307          VCTP_M))
1308   ]
1309   "TARGET_HAVE_MVE"
1310   "vpst\;vctpt.<MVE_vctp> %1"
1311   [(set_attr "type" "mve_move")
1312    (set_attr "length""8")])
1313
1314 ;;
1315 ;; [vcvtbq_f16_f32])
1316 ;;
1317 (define_insn "mve_vcvtbq_f16_f32v8hf"
1318   [
1319    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1320         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1321                       (match_operand:V4SF 2 "s_register_operand" "w")]
1322          VCVTBQ_F16_F32))
1323   ]
1324   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1325   "vcvtb.f16.f32 %q0, %q2"
1326   [(set_attr "type" "mve_move")
1327 ])
1328
1329 ;;
1330 ;; [vcvttq_f16_f32])
1331 ;;
1332 (define_insn "mve_vcvttq_f16_f32v8hf"
1333   [
1334    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1335         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1336                       (match_operand:V4SF 2 "s_register_operand" "w")]
1337          VCVTTQ_F16_F32))
1338   ]
1339   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1340   "vcvtt.f16.f32 %q0, %q2"
1341   [(set_attr "type" "mve_move")
1342 ])
1343
1344 ;;
1345 ;; [veorq_f])
1346 ;;
1347 (define_insn "mve_veorq_f<mode>"
1348   [
1349    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1350         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1351                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1352   ]
1353   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1354   "veor %q0, %q1, %q2"
1355   [(set_attr "type" "mve_move")
1356 ])
1357
1358 ;;
1359 ;; [vmaxnmaq_f]
1360 ;; [vminnmaq_f]
1361 ;;
1362 (define_insn "@mve_<mve_insn>q_f<mode>"
1363   [
1364    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1365         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
1366                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1367          MVE_VMAXNMA_VMINNMAQ))
1368   ]
1369   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1370   "<mve_insn>.f%#<V_sz_elem>\t%q0, %q2"
1371   [(set_attr "type" "mve_move")
1372 ])
1373
1374 ;;
1375 ;; [vmaxnmavq_f]
1376 ;; [vmaxnmvq_f]
1377 ;; [vminnmavq_f]
1378 ;; [vminnmvq_f]
1379 ;;
1380 (define_insn "@mve_<mve_insn>q_f<mode>"
1381   [
1382    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1383         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1384                           (match_operand:MVE_0 2 "s_register_operand" "w")]
1385          MVE_VMAXNMxV_MINNMxVQ))
1386   ]
1387   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1388   "<mve_insn>.f%#<V_sz_elem>\t%0, %q2"
1389   [(set_attr "type" "mve_move")
1390 ])
1391
1392 ;;
1393 ;; [vmaxnmq_f]
1394 ;; [vminnmq_f]
1395 ;;
1396 (define_insn "@mve_<max_min_f_str>q_f<mode>"
1397   [
1398    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1399         (MAX_MIN_F:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1400                          (match_operand:MVE_0 2 "s_register_operand" "w")))
1401   ]
1402   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1403   "<max_min_f_str>.f%#<V_sz_elem>       %q0, %q1, %q2"
1404   [(set_attr "type" "mve_move")
1405 ])
1406
1407 ;;
1408 ;; [vmlaldavq_u, vmlaldavq_s]
1409 ;; [vmlaldavxq_s]
1410 ;; [vmlsldavq_s]
1411 ;; [vmlsldavxq_s]
1412 ;;
1413 (define_insn "@mve_<mve_insn>q_<supf><mode>"
1414   [
1415    (set (match_operand:DI 0 "s_register_operand" "=r")
1416         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
1417                     (match_operand:MVE_5 2 "s_register_operand" "w")]
1418          MVE_VMLxLDAVxQ))
1419   ]
1420   "TARGET_HAVE_MVE"
1421   "<mve_insn>.<supf>%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
1422   [(set_attr "type" "mve_move")
1423 ])
1424
1425 ;;
1426 ;; [vmovnbq_u, vmovnbq_s]
1427 ;; [vmovntq_s, vmovntq_u]
1428 ;; [vqmovnbq_u, vqmovnbq_s]
1429 ;; [vqmovntq_u, vqmovntq_s]
1430 ;; [vqmovunbq_s]
1431 ;; [vqmovuntq_s]
1432 ;;
1433 (define_insn "@mve_<mve_insn>q_<supf><mode>"
1434   [
1435    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
1436         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
1437                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
1438          MVE_MOVN))
1439   ]
1440   "TARGET_HAVE_MVE"
1441   "<mve_insn>.<isu>%#<V_sz_elem>\t%q0, %q2"
1442   [(set_attr "type" "mve_move")
1443 ])
1444
1445 ;;
1446 ;; [vaddq_f]
1447 ;; [vmulq_f]
1448 ;; [vsubq_f]
1449 ;;
1450 (define_insn "mve_<mve_addsubmul>q_f<mode>"
1451   [
1452    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1453         (MVE_INT_BINARY_RTX:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1454                     (match_operand:MVE_0 2 "s_register_operand" "w")))
1455   ]
1456   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1457   "<mve_addsubmul>.f%#<V_sz_elem>\t%q0, %q1, %q2"
1458   [(set_attr "type" "mve_move")
1459 ])
1460
1461 ;;
1462 ;; [vornq_f])
1463 ;;
1464 (define_insn "mve_vornq_f<mode>"
1465   [
1466    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1467         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
1468                    (match_operand:MVE_0 1 "s_register_operand" "w")))
1469   ]
1470   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1471   "vorn %q0, %q1, %q2"
1472   [(set_attr "type" "mve_move")
1473 ])
1474
1475 ;;
1476 ;; [vorrq_f])
1477 ;;
1478 (define_insn "mve_vorrq_f<mode>"
1479   [
1480    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1481         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1482                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1483   ]
1484   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1485   "vorr %q0, %q1, %q2"
1486   [(set_attr "type" "mve_move")
1487 ])
1488
1489 ;;
1490 ;; [vbicq_n_s, vbicq_n_u]
1491 ;; [vorrq_n_u, vorrq_n_s]
1492 ;;
1493 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1494   [
1495    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1496         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1497                        (match_operand:SI 2 "immediate_operand" "i")]
1498          MVE_INT_N_BINARY_LOGIC))
1499   ]
1500   "TARGET_HAVE_MVE"
1501   "<mve_insn>.i%#<V_sz_elem>    %q0, %2"
1502   [(set_attr "type" "mve_move")
1503 ])
1504
1505 ;;
1506 ;; [vqdmullbq_n_s])
1507 ;;
1508 (define_insn "mve_vqdmullbq_n_s<mode>"
1509   [
1510    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1511         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
1512                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
1513          VQDMULLBQ_N_S))
1514   ]
1515   "TARGET_HAVE_MVE"
1516   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
1517   [(set_attr "type" "mve_move")
1518 ])
1519
1520 ;;
1521 ;; [vqdmullbq_s])
1522 ;;
1523 (define_insn "mve_vqdmullbq_s<mode>"
1524   [
1525    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1526         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
1527                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
1528          VQDMULLBQ_S))
1529   ]
1530   "TARGET_HAVE_MVE"
1531   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
1532   [(set_attr "type" "mve_move")
1533 ])
1534
1535 ;;
1536 ;; [vqdmulltq_n_s])
1537 ;;
1538 (define_insn "mve_vqdmulltq_n_s<mode>"
1539   [
1540    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1541         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
1542                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
1543          VQDMULLTQ_N_S))
1544   ]
1545   "TARGET_HAVE_MVE"
1546   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
1547   [(set_attr "type" "mve_move")
1548 ])
1549
1550 ;;
1551 ;; [vqdmulltq_s])
1552 ;;
1553 (define_insn "mve_vqdmulltq_s<mode>"
1554   [
1555    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1556         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
1557                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
1558          VQDMULLTQ_S))
1559   ]
1560   "TARGET_HAVE_MVE"
1561   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
1562   [(set_attr "type" "mve_move")
1563 ])
1564
1565 ;;
1566 ;; [vrmlaldavhq_u vrmlaldavhq_s]
1567 ;; [vrmlaldavhxq_s]
1568 ;; [vrmlsldavhq_s]
1569 ;; [vrmlsldavhxq_s]
1570 ;;
1571 (define_insn "@mve_<mve_insn>q_<supf>v4si"
1572   [
1573    (set (match_operand:DI 0 "s_register_operand" "=r")
1574         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
1575                     (match_operand:V4SI 2 "s_register_operand" "w")]
1576          MVE_VRMLxLDAVxQ))
1577   ]
1578   "TARGET_HAVE_MVE"
1579   "<mve_insn>.<supf>32\t%Q0, %R0, %q1, %q2"
1580   [(set_attr "type" "mve_move")
1581 ])
1582
1583 ;;
1584 ;; [vshllbq_n_s, vshllbq_n_u]
1585 ;; [vshlltq_n_u, vshlltq_n_s]
1586 ;;
1587 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1588   [
1589    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1590         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
1591                                   (match_operand:SI 2 "immediate_operand" "i")]
1592          VSHLLxQ_N))
1593   ]
1594   "TARGET_HAVE_MVE"
1595   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1596   [(set_attr "type" "mve_move")
1597 ])
1598
1599 ;;
1600 ;; [vmulltq_poly_p])
1601 ;;
1602 (define_insn "mve_vmulltq_poly_p<mode>"
1603   [
1604    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1605         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
1606                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
1607          VMULLTQ_POLY_P))
1608   ]
1609   "TARGET_HAVE_MVE"
1610   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
1611   [(set_attr "type" "mve_move")
1612 ])
1613
1614 ;;
1615 ;; [vmullbq_poly_p])
1616 ;;
1617 (define_insn "mve_vmullbq_poly_p<mode>"
1618   [
1619    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1620         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
1621                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
1622          VMULLBQ_POLY_P))
1623   ]
1624   "TARGET_HAVE_MVE"
1625   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
1626   [(set_attr "type" "mve_move")
1627 ])
1628
1629 ;;
1630 ;; [vcmpeqq_m_f]
1631 ;; [vcmpgeq_m_f]
1632 ;; [vcmpgtq_m_f]
1633 ;; [vcmpleq_m_f]
1634 ;; [vcmpltq_m_f]
1635 ;; [vcmpneq_m_f]
1636 ;;
1637 (define_insn "@mve_vcmp<mve_cmp_op1>q_m_f<mode>"
1638   [
1639    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
1640         (unspec:<MVE_VPRED> [(match_operand:MVE_0 1 "s_register_operand" "w")
1641                     (match_operand:MVE_0 2 "s_register_operand" "w")
1642                     (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1643          MVE_CMP_M_F))
1644   ]
1645   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1646   "vpst\;vcmpt.f%#<V_sz_elem>\t<mve_cmp_op1>, %q1, %q2"
1647   [(set_attr "type" "mve_move")
1648    (set_attr "length""8")])
1649 ;;
1650 ;; [vcvtaq_m_u, vcvtaq_m_s])
1651 ;;
1652 (define_insn "mve_vcvtaq_m_<supf><mode>"
1653   [
1654    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1655         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1656                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
1657                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1658          VCVTAQ_M))
1659   ]
1660   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1661   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
1662   [(set_attr "type" "mve_move")
1663    (set_attr "length""8")])
1664 ;;
1665 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
1666 ;;
1667 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
1668   [
1669    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1670         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
1671                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
1672                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1673          VCVTQ_M_TO_F))
1674   ]
1675   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1676   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
1677   [(set_attr "type" "mve_move")
1678    (set_attr "length""8")])
1679
1680 ;;
1681 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s]
1682 ;; [vqrshrntq_n_u, vqrshrntq_n_s]
1683 ;; [vqrshrunbq_n_s]
1684 ;; [vqrshruntq_n_s]
1685 ;; [vqshrnbq_n_u, vqshrnbq_n_s]
1686 ;; [vqshrntq_n_u, vqshrntq_n_s]
1687 ;; [vqshrunbq_n_s]
1688 ;; [vqshruntq_n_s]
1689 ;; [vrshrnbq_n_s, vrshrnbq_n_u]
1690 ;; [vrshrntq_n_u, vrshrntq_n_s]
1691 ;; [vshrnbq_n_u, vshrnbq_n_s]
1692 ;; [vshrntq_n_s, vshrntq_n_u]
1693 ;;
1694 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1695   [
1696    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
1697         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
1698                                  (match_operand:MVE_5 2 "s_register_operand" "w")
1699                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
1700          MVE_SHRN_N))
1701   ]
1702   "TARGET_HAVE_MVE"
1703   "<mve_insn>.<isu>%#<V_sz_elem>\t%q0, %q2, %3"
1704   [(set_attr "type" "mve_move")
1705 ])
1706
1707 ;;
1708 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
1709 ;;
1710 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
1711   [
1712    (set (match_operand:DI 0 "s_register_operand" "=r")
1713         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1714                     (match_operand:V4SI 2 "s_register_operand" "w")
1715                     (match_operand:V4SI 3 "s_register_operand" "w")]
1716          VRMLALDAVHAQ))
1717   ]
1718   "TARGET_HAVE_MVE"
1719   "vrmlaldavha.<supf>32\t%Q0, %R0, %q2, %q3"
1720   [(set_attr "type" "mve_move")
1721 ])
1722
1723 ;;
1724 ;; [vabavq_s, vabavq_u])
1725 ;;
1726 (define_insn "@mve_<mve_insn>q_<supf><mode>"
1727   [
1728    (set (match_operand:SI 0 "s_register_operand" "=r")
1729         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1730                     (match_operand:MVE_2 2 "s_register_operand" "w")
1731                     (match_operand:MVE_2 3 "s_register_operand" "w")]
1732          VABAVQ))
1733   ]
1734   "TARGET_HAVE_MVE"
1735   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
1736   [(set_attr "type" "mve_move")
1737 ])
1738
1739 ;;
1740 ;; [vshlcq_u vshlcq_s]
1741 ;;
1742 (define_expand "mve_vshlcq_vec_<supf><mode>"
1743  [(match_operand:MVE_2 0 "s_register_operand")
1744   (match_operand:MVE_2 1 "s_register_operand")
1745   (match_operand:SI 2 "s_register_operand")
1746   (match_operand:SI 3 "mve_imm_32")
1747   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
1748  "TARGET_HAVE_MVE"
1749 {
1750   rtx ignore_wb = gen_reg_rtx (SImode);
1751   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
1752                                       operands[2], operands[3]));
1753   DONE;
1754 })
1755
1756 (define_expand "mve_vshlcq_carry_<supf><mode>"
1757  [(match_operand:SI 0 "s_register_operand")
1758   (match_operand:MVE_2 1 "s_register_operand")
1759   (match_operand:SI 2 "s_register_operand")
1760   (match_operand:SI 3 "mve_imm_32")
1761   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
1762  "TARGET_HAVE_MVE"
1763 {
1764   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
1765   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
1766                                       operands[2], operands[3]));
1767   DONE;
1768 })
1769
1770 (define_insn "mve_vshlcq_<supf><mode>"
1771  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
1772        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
1773                       (match_operand:SI 3 "s_register_operand" "1")
1774                       (match_operand:SI 4 "mve_imm_32" "Rf")]
1775         VSHLCQ))
1776   (set (match_operand:SI  1 "s_register_operand" "=r")
1777        (unspec:SI [(match_dup 2)
1778                    (match_dup 3)
1779                    (match_dup 4)]
1780         VSHLCQ))]
1781  "TARGET_HAVE_MVE"
1782  "vshlc %q0, %1, %4")
1783
1784 ;;
1785 ;; [vabsq_m_s]
1786 ;; [vclsq_m_s]
1787 ;; [vclzq_m_s, vclzq_m_u]
1788 ;; [vnegq_m_s]
1789 ;; [vqabsq_m_s]
1790 ;; [vqnegq_m_s]
1791 ;;
1792 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
1793   [
1794    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1795         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1796                        (match_operand:MVE_2 2 "s_register_operand" "w")
1797                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1798          MVE_INT_M_UNARY))
1799   ]
1800   "TARGET_HAVE_MVE"
1801   "vpst\;<mve_insn>t.<isu>%#<V_sz_elem>\t%q0, %q2"
1802   [(set_attr "type" "mve_move")
1803    (set_attr "length""8")])
1804
1805 ;;
1806 ;; [vaddvaq_p_u, vaddvaq_p_s])
1807 ;;
1808 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
1809   [
1810    (set (match_operand:SI 0 "s_register_operand" "=Te")
1811         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1812                        (match_operand:MVE_2 2 "s_register_operand" "w")
1813                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1814          VADDVAQ_P))
1815   ]
1816   "TARGET_HAVE_MVE"
1817   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q2"
1818   [(set_attr "type" "mve_move")
1819    (set_attr "length""8")])
1820
1821 ;;
1822 ;; [vcmpcsq_m_n_u])
1823 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
1824 ;; [vcmpgeq_m_n_s])
1825 ;; [vcmpgtq_m_n_s])
1826 ;; [vcmphiq_m_n_u])
1827 ;; [vcmpleq_m_n_s])
1828 ;; [vcmpltq_m_n_s])
1829 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
1830 ;;
1831 (define_insn "@mve_vcmp<mve_cmp_op1>q_m_n_<supf><mode>"
1832   [
1833    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
1834         (unspec:<MVE_VPRED> [(match_operand:MVE_2 1 "s_register_operand" "w")
1835                        (match_operand:<V_elem> 2 "s_register_operand" "r")
1836                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1837          MVE_CMP_M_N))
1838   ]
1839   "TARGET_HAVE_MVE"
1840   "vpst\;vcmpt.<isu>%#<V_sz_elem>\t<mve_cmp_op1>, %q1, %2"
1841   [(set_attr "type" "mve_move")
1842    (set_attr "length""8")])
1843
1844 ;;
1845 ;; [vcmpcsq_m_u]
1846 ;; [vcmpeqq_m_u, vcmpeqq_m_s]
1847 ;; [vcmpgeq_m_s]
1848 ;; [vcmpgtq_m_s]
1849 ;; [vcmphiq_m_u]
1850 ;; [vcmpleq_m_s]
1851 ;; [vcmpltq_m_s]
1852 ;; [vcmpneq_m_s, vcmpneq_m_u]
1853 ;;
1854 (define_insn "@mve_vcmp<mve_cmp_op1>q_m_<supf><mode>"
1855   [
1856    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
1857         (unspec:<MVE_VPRED> [(match_operand:MVE_2 1 "s_register_operand" "w")
1858                        (match_operand:MVE_2 2 "s_register_operand" "w")
1859                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1860          MVE_CMP_M))
1861   ]
1862   "TARGET_HAVE_MVE"
1863   "vpst\;vcmpt.<isu>%#<V_sz_elem>\t<mve_cmp_op1>, %q1, %q2"
1864   [(set_attr "type" "mve_move")
1865    (set_attr "length""8")])
1866
1867 ;;
1868 ;; [vdupq_m_n_s, vdupq_m_n_u])
1869 ;;
1870 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
1871   [
1872    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1873         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1874                        (match_operand:<V_elem> 2 "s_register_operand" "r")
1875                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1876          VDUPQ_M_N))
1877   ]
1878   "TARGET_HAVE_MVE"
1879   "vpst\;<mve_insn>t.%#<V_sz_elem>\t%q0, %2"
1880   [(set_attr "type" "mve_move")
1881    (set_attr "length""8")])
1882
1883 ;;
1884 ;; [vmaxaq_m_s]
1885 ;; [vminaq_m_s]
1886 ;;
1887 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
1888   [
1889    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1890         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1891                        (match_operand:MVE_2 2 "s_register_operand" "w")
1892                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1893          MVE_VMAXAVMINAQ_M))
1894   ]
1895   "TARGET_HAVE_MVE"
1896   "vpst\;<mve_insn>t.s%#<V_sz_elem>\t%q0, %q2"
1897   [(set_attr "type" "mve_move")
1898    (set_attr "length""8")])
1899
1900 ;;
1901 ;; [vmaxavq_p_s]
1902 ;; [vmaxvq_p_u, vmaxvq_p_s]
1903 ;; [vminavq_p_s]
1904 ;; [vminvq_p_s, vminvq_p_u]
1905 ;;
1906 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
1907   [
1908    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1909         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1910                        (match_operand:MVE_2 2 "s_register_operand" "w")
1911                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1912          MVE_VMAXVQ_VMINVQ_P))
1913   ]
1914   "TARGET_HAVE_MVE"
1915   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q2"
1916   [(set_attr "type" "mve_move")
1917    (set_attr "length""8")])
1918
1919 ;;
1920 ;; [vmladavaq_u, vmladavaq_s]
1921 ;; [vmladavaxq_s]
1922 ;; [vmlsdavaq_s]
1923 ;; [vmlsdavaxq_s]
1924 ;;
1925 (define_insn "@mve_<mve_insn>q_<supf><mode>"
1926   [
1927    (set (match_operand:SI 0 "s_register_operand" "=Te")
1928         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1929                        (match_operand:MVE_2 2 "s_register_operand" "w")
1930                        (match_operand:MVE_2 3 "s_register_operand" "w")]
1931          MVE_VMLxDAVAQ))
1932   ]
1933   "TARGET_HAVE_MVE"
1934   "<mve_insn>.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
1935   [(set_attr "type" "mve_move")
1936 ])
1937
1938 ;;
1939 ;; [vmladavq_p_u, vmladavq_p_s]
1940 ;; [vmladavxq_p_s]
1941 ;; [vmlsdavq_p_s]
1942 ;; [vmlsdavxq_p_s]
1943 ;;
1944 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
1945   [
1946    (set (match_operand:SI 0 "s_register_operand" "=Te")
1947         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1948                        (match_operand:MVE_2 2 "s_register_operand" "w")
1949                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1950          MVE_VMLxDAVQ_P))
1951   ]
1952   "TARGET_HAVE_MVE"
1953   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1954   [(set_attr "type" "mve_move")
1955    (set_attr "length""8")])
1956
1957 ;;
1958 ;; [vmlaq_n_u, vmlaq_n_s]
1959 ;; [vmlasq_n_u, vmlasq_n_s]
1960 ;; [vqdmlahq_n_s]
1961 ;; [vqdmlashq_n_s]
1962 ;; [vqrdmlahq_n_s]
1963 ;; [vqrdmlashq_n_s]
1964 ;;
1965 (define_insn "@mve_<mve_insn>q_n_<supf><mode>"
1966   [
1967    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1968         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1969                        (match_operand:MVE_2 2 "s_register_operand" "w")
1970                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
1971          MVE_VMLxQ_N))
1972   ]
1973   "TARGET_HAVE_MVE"
1974   "<mve_insn>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
1975   [(set_attr "type" "mve_move")
1976 ])
1977
1978 ;;
1979 ;; [vmvnq_m_s, vmvnq_m_u])
1980 ;;
1981 (define_insn "mve_vmvnq_m_<supf><mode>"
1982   [
1983    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1984         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1985                        (match_operand:MVE_2 2 "s_register_operand" "w")
1986                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
1987          VMVNQ_M))
1988   ]
1989   "TARGET_HAVE_MVE"
1990   "vpst\;vmvnt %q0, %q2"
1991   [(set_attr "type" "mve_move")
1992    (set_attr "length""8")])
1993
1994 ;;
1995 ;; [vpselq_u, vpselq_s])
1996 ;;
1997 (define_insn "@mve_vpselq_<supf><mode>"
1998   [
1999    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
2000         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
2001                        (match_operand:MVE_1 2 "s_register_operand" "w")
2002                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2003          VPSELQ))
2004   ]
2005   "TARGET_HAVE_MVE"
2006   "vpsel %q0, %q1, %q2"
2007   [(set_attr "type" "mve_move")
2008 ])
2009
2010 ;;
2011 ;; [vqdmladhq_s]
2012 ;; [vqdmladhxq_s]
2013 ;; [vqdmlsdhq_s]
2014 ;; [vqdmlsdhxq_s]
2015 ;; [vqrdmladhq_s]
2016 ;; [vqrdmladhxq_s]
2017 ;; [vqrdmlsdhq_s]
2018 ;; [vqrdmlsdhxq_s]
2019 ;;
2020 (define_insn "@mve_<mve_insn>q_<supf><mode>"
2021   [
2022    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2023         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2024                        (match_operand:MVE_2 2 "s_register_operand" "w")
2025                        (match_operand:MVE_2 3 "s_register_operand" "w")]
2026          MVE_VQxDMLxDHxQ_S))
2027   ]
2028   "TARGET_HAVE_MVE"
2029   "<mve_insn>.s%#<V_sz_elem>\t%q0, %q2, %q3"
2030   [(set_attr "type" "mve_move")
2031 ])
2032
2033 ;;
2034 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u]
2035 ;; [vrshlq_m_n_s, vrshlq_m_n_u]
2036 ;;
2037 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
2038   [
2039    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2040         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2041                        (match_operand:SI 2 "s_register_operand" "r")
2042                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2043          MVE_RSHIFT_M_N))
2044   ]
2045   "TARGET_HAVE_MVE"
2046   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %2"
2047   [(set_attr "type" "mve_move")
2048    (set_attr "length""8")])
2049
2050 ;;
2051 ;; [vqshlq_m_r_u, vqshlq_m_r_s]
2052 ;; [vshlq_m_r_u, vshlq_m_r_s]
2053 ;;
2054 (define_insn "@mve_<mve_insn>q_m_r_<supf><mode>"
2055   [
2056    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2057         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2058                        (match_operand:SI 2 "s_register_operand" "r")
2059                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2060          MVE_SHIFT_M_R))
2061   ]
2062   "TARGET_HAVE_MVE"
2063   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %2"
2064   [(set_attr "type" "mve_move")
2065    (set_attr "length""8")])
2066
2067 ;;
2068 ;; [vrev64q_m_u, vrev64q_m_s])
2069 ;;
2070 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2071   [
2072    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
2073         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2074                        (match_operand:MVE_2 2 "s_register_operand" "w")
2075                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2076          VREV64Q_M))
2077   ]
2078   "TARGET_HAVE_MVE"
2079   "vpst\;<mve_insn>t.%#<V_sz_elem>\t%q0, %q2"
2080   [(set_attr "type" "mve_move")
2081    (set_attr "length""8")])
2082
2083 ;;
2084 ;; [vsliq_n_u, vsliq_n_s])
2085 ;;
2086 (define_insn "mve_vsliq_n_<supf><mode>"
2087   [
2088    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2089         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2090                        (match_operand:MVE_2 2 "s_register_operand" "w")
2091                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
2092          VSLIQ_N))
2093   ]
2094   "TARGET_HAVE_MVE"
2095   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
2096   [(set_attr "type" "mve_move")
2097 ])
2098
2099 ;;
2100 ;; [vsriq_n_u, vsriq_n_s])
2101 ;;
2102 (define_insn "mve_vsriq_n_<supf><mode>"
2103   [
2104    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2105         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2106                        (match_operand:MVE_2 2 "s_register_operand" "w")
2107                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")]
2108          VSRIQ_N))
2109   ]
2110   "TARGET_HAVE_MVE"
2111   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
2112   [(set_attr "type" "mve_move")
2113 ])
2114
2115 ;;
2116 ;; [vabsq_m_f]
2117 ;; [vnegq_m_f]
2118 ;; [vrndaq_m_f]
2119 ;; [vrndmq_m_f]
2120 ;; [vrndnq_m_f]
2121 ;; [vrndpq_m_f]
2122 ;; [vrndq_m_f]
2123 ;; [vrndxq_m_f]
2124 ;;
2125 (define_insn "@mve_<mve_insn>q_m_f<mode>"
2126   [
2127    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2128         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2129                        (match_operand:MVE_0 2 "s_register_operand" "w")
2130                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2131          MVE_FP_M_UNARY))
2132   ]
2133   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2134   "vpst\;<mve_mnemo>t.f%#<V_sz_elem>\t%q0, %q2"
2135   [(set_attr "type" "mve_move")
2136    (set_attr "length""8")])
2137
2138 ;;
2139 ;; [vaddlvaq_p_s vaddlvaq_p_u])
2140 ;;
2141 (define_insn "@mve_<mve_insn>q_p_<supf>v4si"
2142   [
2143    (set (match_operand:DI 0 "s_register_operand" "=r")
2144         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2145                        (match_operand:V4SI 2 "s_register_operand" "w")
2146                        (match_operand:V4BI 3 "vpr_register_operand" "Up")]
2147          VADDLVAQ_P))
2148   ]
2149   "TARGET_HAVE_MVE"
2150   "vpst\;<mve_insn>t.<supf>32\t%Q0, %R0, %q2"
2151   [(set_attr "type" "mve_move")
2152    (set_attr "length""8")])
2153 ;;
2154 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
2155 ;;
2156 (define_insn "mve_vcmlaq<mve_rot><mode>"
2157   [
2158    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
2159         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
2160                     (unspec:MVE_0
2161                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
2162                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
2163                      VCMLA)))
2164   ]
2165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2166   "@
2167    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
2168    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
2169   [(set_attr "type" "mve_move")
2170 ])
2171
2172 ;;
2173 ;; [vcmpeqq_m_n_f])
2174 ;; [vcmpgeq_m_n_f])
2175 ;; [vcmpgtq_m_n_f])
2176 ;; [vcmpleq_m_n_f])
2177 ;; [vcmpltq_m_n_f])
2178 ;; [vcmpneq_m_n_f])
2179 ;;
2180 (define_insn "@mve_vcmp<mve_cmp_op1>q_m_n_f<mode>"
2181   [
2182    (set (match_operand:<MVE_VPRED> 0 "vpr_register_operand" "=Up")
2183         (unspec:<MVE_VPRED> [(match_operand:MVE_0 1 "s_register_operand" "w")
2184                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2185                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2186          MVE_CMP_M_N_F))
2187   ]
2188   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2189   "vpst\;vcmpt.f%#<V_sz_elem>\t<mve_cmp_op1>, %q1, %2"
2190   [(set_attr "type" "mve_move")
2191    (set_attr "length""8")])
2192
2193 ;;
2194 ;; [vcvtbq_m_f16_f32])
2195 ;;
2196 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
2197   [
2198    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2199         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2200                        (match_operand:V4SF 2 "s_register_operand" "w")
2201                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2202          VCVTBQ_M_F16_F32))
2203   ]
2204   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2205   "vpst\;vcvtbt.f16.f32 %q0, %q2"
2206   [(set_attr "type" "mve_move")
2207    (set_attr "length""8")])
2208
2209 ;;
2210 ;; [vcvtbq_m_f32_f16])
2211 ;;
2212 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
2213   [
2214    (set (match_operand:V4SF 0 "s_register_operand" "=w")
2215         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
2216                        (match_operand:V8HF 2 "s_register_operand" "w")
2217                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2218          VCVTBQ_M_F32_F16))
2219   ]
2220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2221   "vpst\;vcvtbt.f32.f16 %q0, %q2"
2222   [(set_attr "type" "mve_move")
2223    (set_attr "length""8")])
2224
2225 ;;
2226 ;; [vcvttq_m_f16_f32])
2227 ;;
2228 (define_insn "mve_vcvttq_m_f16_f32v8hf"
2229   [
2230    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2231         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2232                        (match_operand:V4SF 2 "s_register_operand" "w")
2233                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2234          VCVTTQ_M_F16_F32))
2235   ]
2236   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2237   "vpst\;vcvttt.f16.f32 %q0, %q2"
2238   [(set_attr "type" "mve_move")
2239    (set_attr "length""8")])
2240
2241 ;;
2242 ;; [vcvttq_m_f32_f16])
2243 ;;
2244 (define_insn "mve_vcvttq_m_f32_f16v4sf"
2245   [
2246    (set (match_operand:V4SF 0 "s_register_operand" "=w")
2247         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
2248                        (match_operand:V8HF 2 "s_register_operand" "w")
2249                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2250          VCVTTQ_M_F32_F16))
2251   ]
2252   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2253   "vpst\;vcvttt.f32.f16 %q0, %q2"
2254   [(set_attr "type" "mve_move")
2255    (set_attr "length""8")])
2256
2257 ;;
2258 ;; [vdupq_m_n_f])
2259 ;;
2260 (define_insn "@mve_<mve_insn>q_m_n_f<mode>"
2261   [
2262    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2263         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2264                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2265                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2266          MVE_FP_M_N_VDUPQ_ONLY))
2267   ]
2268   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2269   "vpst\;<mve_insn>t.%#<V_sz_elem>\t%q0, %2"
2270   [(set_attr "type" "mve_move")
2271    (set_attr "length""8")])
2272
2273 ;;
2274 ;; [vfmaq_f])
2275 ;;
2276 (define_insn "mve_vfmaq_f<mode>"
2277   [
2278    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2279         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2280                        (match_operand:MVE_0 2 "s_register_operand" "w")
2281                        (match_operand:MVE_0 3 "s_register_operand" "w")]
2282          VFMAQ_F))
2283   ]
2284   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2285   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
2286   [(set_attr "type" "mve_move")
2287 ])
2288
2289 ;;
2290 ;; [vfmaq_n_f])
2291 ;;
2292 (define_insn "mve_vfmaq_n_f<mode>"
2293   [
2294    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2295         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2296                        (match_operand:MVE_0 2 "s_register_operand" "w")
2297                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
2298          VFMAQ_N_F))
2299   ]
2300   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2301   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
2302   [(set_attr "type" "mve_move")
2303 ])
2304
2305 ;;
2306 ;; [vfmasq_n_f])
2307 ;;
2308 (define_insn "mve_vfmasq_n_f<mode>"
2309   [
2310    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2311         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2312                        (match_operand:MVE_0 2 "s_register_operand" "w")
2313                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
2314          VFMASQ_N_F))
2315   ]
2316   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2317   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
2318   [(set_attr "type" "mve_move")
2319 ])
2320 ;;
2321 ;; [vfmsq_f])
2322 ;;
2323 (define_insn "mve_vfmsq_f<mode>"
2324   [
2325    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2326         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2327                        (match_operand:MVE_0 2 "s_register_operand" "w")
2328                        (match_operand:MVE_0 3 "s_register_operand" "w")]
2329          VFMSQ_F))
2330   ]
2331   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2332   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
2333   [(set_attr "type" "mve_move")
2334 ])
2335
2336 ;;
2337 ;; [vmaxnmaq_m_f]
2338 ;; [vminnmaq_m_f]
2339 ;;
2340 (define_insn "@mve_<mve_insn>q_m_f<mode>"
2341   [
2342    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2343         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2344                        (match_operand:MVE_0 2 "s_register_operand" "w")
2345                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2346          MVE_VMAXNMA_VMINNMAQ_M))
2347   ]
2348   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2349   "vpst\;<mve_insn>t.f%#<V_sz_elem>\t%q0, %q2"
2350   [(set_attr "type" "mve_move")
2351    (set_attr "length""8")])
2352
2353 ;;
2354 ;; [vmaxnmavq_p_f]
2355 ;; [vmaxnmvq_p_f]
2356 ;; [vminnmavq_p_f]
2357 ;; [vminnmvq_p_f]
2358 ;;
2359 (define_insn "@mve_<mve_insn>q_p_f<mode>"
2360   [
2361    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2362         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2363                        (match_operand:MVE_0 2 "s_register_operand" "w")
2364                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2365          MVE_VMAXNMxV_MINNMxVQ_P))
2366   ]
2367   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2368   "vpst\;<mve_insn>t.f%#<V_sz_elem>\t%0, %q2"
2369   [(set_attr "type" "mve_move")
2370    (set_attr "length""8")])
2371
2372 ;;
2373 ;; [vmlaldavaq_s, vmlaldavaq_u]
2374 ;; [vmlaldavaxq_s]
2375 ;; [vmlsldavaq_s]
2376 ;; [vmlsldavaxq_s]
2377 ;;
2378 (define_insn "@mve_<mve_insn>q_<supf><mode>"
2379   [
2380    (set (match_operand:DI 0 "s_register_operand" "=r")
2381         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2382                        (match_operand:MVE_5 2 "s_register_operand" "w")
2383                        (match_operand:MVE_5 3 "s_register_operand" "w")]
2384          MVE_VMLxLDAVAxQ))
2385   ]
2386   "TARGET_HAVE_MVE"
2387   "<mve_insn>.<supf>%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
2388   [(set_attr "type" "mve_move")
2389 ])
2390
2391 ;;
2392 ;; [vmlaldavq_p_u, vmlaldavq_p_s]
2393 ;; [vmlaldavxq_p_s]
2394 ;; [vmlsldavq_p_s]
2395 ;; [vmlsldavxq_p_s]
2396 ;;
2397 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
2398   [
2399    (set (match_operand:DI 0 "s_register_operand" "=r")
2400         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2401                        (match_operand:MVE_5 2 "s_register_operand" "w")
2402                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2403          MVE_VMLxLDAVxQ_P))
2404   ]
2405   "TARGET_HAVE_MVE"
2406   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
2407   [(set_attr "type" "mve_move")
2408    (set_attr "length""8")])
2409
2410 ;;
2411 ;; [vmovlbq_m_u, vmovlbq_m_s])
2412 ;; [vmovltq_m_u, vmovltq_m_s])
2413 ;;
2414 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2415   [
2416    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2417         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2418                        (match_operand:MVE_3 2 "s_register_operand" "w")
2419                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2420          VMOVLxQ_M))
2421   ]
2422   "TARGET_HAVE_MVE"
2423   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2"
2424   [(set_attr "type" "mve_move")
2425    (set_attr "length""8")])
2426
2427 ;;
2428 ;; [vmovnbq_m_u, vmovnbq_m_s]
2429 ;; [vmovntq_m_u, vmovntq_m_s]
2430 ;; [vqmovnbq_m_s, vqmovnbq_m_u]
2431 ;; [vqmovntq_m_u, vqmovntq_m_s]
2432 ;; [vqmovunbq_m_s]
2433 ;; [vqmovuntq_m_s]
2434 ;;
2435 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2436   [
2437    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2438         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2439                        (match_operand:MVE_5 2 "s_register_operand" "w")
2440                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2441          MVE_MOVN_M))
2442   ]
2443   "TARGET_HAVE_MVE"
2444   "vpst\;<mve_insn>t.<isu>%#<V_sz_elem>\t%q0, %q2"
2445   [(set_attr "type" "mve_move")
2446    (set_attr "length""8")])
2447
2448 ;;
2449 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
2450 ;;
2451 (define_insn "mve_vmvnq_m_n_<supf><mode>"
2452   [
2453    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2454         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2455                        (match_operand:SI 2 "immediate_operand" "i")
2456                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2457          VMVNQ_M_N))
2458   ]
2459   "TARGET_HAVE_MVE"
2460   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
2461   [(set_attr "type" "mve_move")
2462    (set_attr "length""8")])
2463
2464 ;;
2465 ;; [vbicq_m_n_s, vbicq_m_n_u]
2466 ;; [vorrq_m_n_s, vorrq_m_n_u]
2467 ;;
2468 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
2469   [
2470    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2471         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2472                        (match_operand:SI 2 "immediate_operand" "i")
2473                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2474          MVE_INT_M_N_BINARY_LOGIC))
2475   ]
2476   "TARGET_HAVE_MVE"
2477   "vpst\;<mve_insn>t.i%#<V_sz_elem>     %q0, %2"
2478   [(set_attr "type" "mve_move")
2479    (set_attr "length""8")])
2480
2481 ;;
2482 ;; [vpselq_f])
2483 ;;
2484 (define_insn "@mve_vpselq_f<mode>"
2485   [
2486    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2487         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2488                        (match_operand:MVE_0 2 "s_register_operand" "w")
2489                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2490          VPSELQ_F))
2491   ]
2492   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2493   "vpsel %q0, %q1, %q2"
2494   [(set_attr "type" "mve_move")
2495 ])
2496
2497 ;;
2498 ;; [vrev32q_m_f])
2499 ;;
2500 (define_insn "@mve_<mve_insn>q_m_f<mode>"
2501   [
2502    (set (match_operand:MVE_V8HF 0 "s_register_operand" "=w")
2503         (unspec:MVE_V8HF [(match_operand:MVE_V8HF 1 "s_register_operand" "0")
2504                           (match_operand:MVE_V8HF 2 "s_register_operand" "w")
2505                           (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2506          MVE_FP_M_VREV32Q_ONLY))
2507   ]
2508   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2509   "vpst\;<mve_insn>t.<V_sz_elem>\t%q0, %q2"
2510   [(set_attr "type" "mve_move")
2511    (set_attr "length""8")])
2512
2513 ;;
2514 ;; [vrev32q_m_s, vrev32q_m_u])
2515 ;;
2516 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2517   [
2518    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
2519         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
2520                        (match_operand:MVE_3 2 "s_register_operand" "w")
2521                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2522          VREV32Q_M))
2523   ]
2524   "TARGET_HAVE_MVE"
2525   "vpst\;<mve_insn>t.%#<V_sz_elem>\t%q0, %q2"
2526   [(set_attr "type" "mve_move")
2527    (set_attr "length""8")])
2528
2529 ;;
2530 ;; [vrev64q_m_f])
2531 ;;
2532 (define_insn "@mve_<mve_insn>q_m_f<mode>"
2533   [
2534    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
2535         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2536                        (match_operand:MVE_0 2 "s_register_operand" "w")
2537                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2538          MVE_FP_M_VREV64Q_ONLY))
2539   ]
2540   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2541   "vpst\;<mve_insn>t.%#<V_sz_elem>\t%q0, %q2"
2542   [(set_attr "type" "mve_move")
2543    (set_attr "length""8")])
2544
2545 ;;
2546 ;; [vrmlaldavhaxq_s])
2547 ;;
2548 (define_insn "mve_vrmlaldavhaxq_sv4si"
2549   [
2550    (set (match_operand:DI 0 "s_register_operand" "=r")
2551         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2552                        (match_operand:V4SI 2 "s_register_operand" "w")
2553                        (match_operand:V4SI 3 "s_register_operand" "w")]
2554          VRMLALDAVHAXQ_S))
2555   ]
2556   "TARGET_HAVE_MVE"
2557   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
2558   [(set_attr "type" "mve_move")
2559 ])
2560
2561 ;;
2562 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s]
2563 ;; [vrmlaldavhxq_p_s]
2564 ;; [vrmlsldavhq_p_s]
2565 ;; [vrmlsldavhxq_p_s]
2566 ;;
2567 (define_insn "@mve_<mve_insn>q_p_<supf>v4si"
2568   [
2569    (set (match_operand:DI 0 "s_register_operand" "=r")
2570         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2571                        (match_operand:V4SI 2 "s_register_operand" "w")
2572                        (match_operand:V4BI 3 "vpr_register_operand" "Up")]
2573          MVE_VRMLxLDAVHxQ_P))
2574   ]
2575   "TARGET_HAVE_MVE"
2576   "vpst\;<mve_insn>t.<supf>32\t%Q0, %R0, %q1, %q2"
2577   [(set_attr "type" "mve_move")
2578    (set_attr "length""8")])
2579
2580 ;;
2581 ;; [vrmlsldavhaxq_s])
2582 ;;
2583 (define_insn "mve_vrmlsldavhaxq_sv4si"
2584   [
2585    (set (match_operand:DI 0 "s_register_operand" "=r")
2586         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2587                        (match_operand:V4SI 2 "s_register_operand" "w")
2588                        (match_operand:V4SI 3 "s_register_operand" "w")]
2589          VRMLSLDAVHAXQ_S))
2590   ]
2591   "TARGET_HAVE_MVE"
2592   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
2593   [(set_attr "type" "mve_move")
2594 ])
2595
2596 ;;
2597 ;; [vcvtmq_m_s, vcvtmq_m_u])
2598 ;;
2599 (define_insn "mve_vcvtmq_m_<supf><mode>"
2600   [
2601    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2602         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2603                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2604                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2605          VCVTMQ_M))
2606   ]
2607   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2608   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2609   [(set_attr "type" "mve_move")
2610    (set_attr "length""8")])
2611
2612 ;;
2613 ;; [vcvtpq_m_u, vcvtpq_m_s])
2614 ;;
2615 (define_insn "mve_vcvtpq_m_<supf><mode>"
2616   [
2617    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2618         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2619                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2620                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2621          VCVTPQ_M))
2622   ]
2623   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2624   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2625   [(set_attr "type" "mve_move")
2626    (set_attr "length""8")])
2627
2628 ;;
2629 ;; [vcvtnq_m_s, vcvtnq_m_u])
2630 ;;
2631 (define_insn "mve_vcvtnq_m_<supf><mode>"
2632   [
2633    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2634         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2635                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2636                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2637          VCVTNQ_M))
2638   ]
2639   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2640   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2641   [(set_attr "type" "mve_move")
2642    (set_attr "length""8")])
2643
2644 ;;
2645 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
2646 ;;
2647 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
2648   [
2649    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2650         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2651                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2652                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
2653                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2654          VCVTQ_M_N_FROM_F))
2655   ]
2656   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2657   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
2658   [(set_attr "type" "mve_move")
2659    (set_attr "length""8")])
2660
2661 ;;
2662 ;; [vrev16q_m_u, vrev16q_m_s])
2663 ;;
2664 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2665   [
2666    (set (match_operand:MVE_V16QI 0 "s_register_operand" "=w")
2667         (unspec:MVE_V16QI [(match_operand:MVE_V16QI 1 "s_register_operand" "0")
2668                            (match_operand:MVE_V16QI 2 "s_register_operand" "w")
2669                            (match_operand:V16BI 3 "vpr_register_operand" "Up")]
2670          VREV16Q_M))
2671   ]
2672   "TARGET_HAVE_MVE"
2673   "vpst\;<mve_insn>t.<V_sz_elem>\t%q0, %q2"
2674   [(set_attr "type" "mve_move")
2675    (set_attr "length""8")])
2676
2677 ;;
2678 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
2679 ;;
2680 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
2681   [
2682    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2683         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2684                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2685                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
2686          VCVTQ_M_FROM_F))
2687   ]
2688   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2689   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2690   [(set_attr "type" "mve_move")
2691    (set_attr "length""8")])
2692
2693 ;;
2694 ;; [vrmlsldavhaq_s])
2695 ;;
2696 (define_insn "mve_vrmlsldavhaq_sv4si"
2697   [
2698    (set (match_operand:DI 0 "s_register_operand" "=r")
2699         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2700                     (match_operand:V4SI 2 "s_register_operand" "w")
2701                     (match_operand:V4SI 3 "s_register_operand" "w")]
2702          VRMLSLDAVHAQ_S))
2703   ]
2704   "TARGET_HAVE_MVE"
2705   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
2706   [(set_attr "type" "mve_move")
2707 ])
2708
2709 ;;
2710 ;; [vabavq_p_s, vabavq_p_u])
2711 ;;
2712 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
2713   [
2714    (set (match_operand:SI 0 "s_register_operand" "=r")
2715         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2716                     (match_operand:MVE_2 2 "s_register_operand" "w")
2717                     (match_operand:MVE_2 3 "s_register_operand" "w")
2718                     (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2719          VABAVQ_P))
2720   ]
2721   "TARGET_HAVE_MVE"
2722   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2723   [(set_attr "type" "mve_move")
2724    (set_attr "length" "8")])
2725
2726 ;;
2727 ;; [vqshluq_m_n_s])
2728 ;;
2729 (define_insn "mve_vqshluq_m_n_s<mode>"
2730   [
2731    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2732         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2733                        (match_operand:MVE_2 2 "s_register_operand" "w")
2734                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
2735                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2736          VQSHLUQ_M_N_S))
2737   ]
2738   "TARGET_HAVE_MVE"
2739   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
2740   [(set_attr "type" "mve_move")
2741    (set_attr "length" "8")])
2742
2743 ;;
2744 ;; [vsriq_m_n_s, vsriq_m_n_u])
2745 ;;
2746 (define_insn "mve_vsriq_m_n_<supf><mode>"
2747   [
2748    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2749         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2750                        (match_operand:MVE_2 2 "s_register_operand" "w")
2751                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
2752                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2753          VSRIQ_M_N))
2754   ]
2755   "TARGET_HAVE_MVE"
2756   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
2757   [(set_attr "type" "mve_move")
2758    (set_attr "length" "8")])
2759
2760 ;;
2761 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
2762 ;;
2763 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
2764   [
2765    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2766         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2767                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2768                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
2769                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2770          VCVTQ_M_N_TO_F))
2771   ]
2772   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2773   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
2774   [(set_attr "type" "mve_move")
2775    (set_attr "length""8")])
2776
2777 ;;
2778 ;; [vabdq_m_s, vabdq_m_u]
2779 ;; [vhaddq_m_s, vhaddq_m_u]
2780 ;; [vhsubq_m_s, vhsubq_m_u]
2781 ;; [vmaxq_m_s, vmaxq_m_u]
2782 ;; [vminq_m_s, vminq_m_u]
2783 ;; [vmulhq_m_s, vmulhq_m_u]
2784 ;; [vqaddq_m_u, vqaddq_m_s]
2785 ;; [vqdmladhq_m_s]
2786 ;; [vqdmladhxq_m_s]
2787 ;; [vqdmlsdhq_m_s]
2788 ;; [vqdmlsdhxq_m_s]
2789 ;; [vqdmulhq_m_s]
2790 ;; [vqrdmladhq_m_s]
2791 ;; [vqrdmladhxq_m_s]
2792 ;; [vqrdmlsdhq_m_s]
2793 ;; [vqrdmlsdhxq_m_s]
2794 ;; [vqrdmulhq_m_s]
2795 ;; [vqrshlq_m_u, vqrshlq_m_s]
2796 ;; [vqshlq_m_u, vqshlq_m_s]
2797 ;; [vqsubq_m_u, vqsubq_m_s]
2798 ;; [vrhaddq_m_u, vrhaddq_m_s]
2799 ;; [vrmulhq_m_u, vrmulhq_m_s]
2800 ;; [vrshlq_m_s, vrshlq_m_u]
2801 ;; [vshlq_m_s, vshlq_m_u]
2802 ;;
2803 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2804   [
2805    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2806         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2807                        (match_operand:MVE_2 2 "s_register_operand" "w")
2808                        (match_operand:MVE_2 3 "s_register_operand" "w")
2809                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2810          MVE_INT_SU_M_BINARY))
2811   ]
2812   "TARGET_HAVE_MVE"
2813   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
2814   [(set_attr "type" "mve_move")
2815    (set_attr "length""8")])
2816
2817 ;;
2818 ;; [vaddq_m_n_s, vaddq_m_n_u]
2819 ;; [vsubq_m_n_s, vsubq_m_n_u]
2820 ;; [vmulq_m_n_s, vmulq_m_n_u]
2821 ;;
2822 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
2823   [
2824    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2825         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2826                        (match_operand:MVE_2 2 "s_register_operand" "w")
2827                        (match_operand:<V_elem> 3 "s_register_operand" "r")
2828                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2829          MVE_INT_M_N_BINARY))
2830   ]
2831   "TARGET_HAVE_MVE"
2832   "vpst\;<mve_insn>t.i%#<V_sz_elem>     %q0, %q2, %3"
2833   [(set_attr "type" "mve_move")
2834    (set_attr "length""8")])
2835
2836 ;;
2837 ;; [vaddq_m_u, vaddq_m_s]
2838 ;; [vsubq_m_u, vsubq_m_s]
2839 ;; [vmulq_m_u, vmulq_m_s]
2840 ;;
2841 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2842   [
2843    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2844         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2845                        (match_operand:MVE_2 2 "s_register_operand" "w")
2846                        (match_operand:MVE_2 3 "s_register_operand" "w")
2847                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2848          MVE_INT_M_BINARY))
2849   ]
2850   "TARGET_HAVE_MVE"
2851   "vpst\;<mve_insn>t.i%#<V_sz_elem>     %q0, %q2, %q3"
2852   [(set_attr "type" "mve_move")
2853    (set_attr "length""8")])
2854
2855 ;;
2856 ;; [vandq_m_u, vandq_m_s]
2857 ;; [vbicq_m_u, vbicq_m_s]
2858 ;; [veorq_m_u, veorq_m_s]
2859 ;; [vorrq_m_u, vorrq_m_s]
2860 ;;
2861 (define_insn "@mve_<mve_insn>q_m_<supf><mode>"
2862   [
2863    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2864         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2865                        (match_operand:MVE_2 2 "s_register_operand" "w")
2866                        (match_operand:MVE_2 3 "s_register_operand" "w")
2867                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2868          MVE_INT_M_BINARY_LOGIC))
2869   ]
2870   "TARGET_HAVE_MVE"
2871   "vpst\;<mve_insn>t %q0, %q2, %q3"
2872   [(set_attr "type" "mve_move")
2873    (set_attr "length""8")])
2874
2875 ;;
2876 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
2877 ;;
2878 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
2879   [
2880    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2881         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2882                        (match_operand:MVE_2 2 "s_register_operand" "w")
2883                        (match_operand:SI 3 "s_register_operand" "r")
2884                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2885          VBRSRQ_M_N))
2886   ]
2887   "TARGET_HAVE_MVE"
2888   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
2889   [(set_attr "type" "mve_move")
2890    (set_attr "length""8")])
2891
2892 ;;
2893 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
2894 ;;
2895 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
2896   [
2897    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
2898         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2899                        (match_operand:MVE_2 2 "s_register_operand" "w")
2900                        (match_operand:MVE_2 3 "s_register_operand" "w")
2901                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2902          VCADDQ_ROT270_M))
2903   ]
2904   "TARGET_HAVE_MVE"
2905   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
2906   [(set_attr "type" "mve_move")
2907    (set_attr "length""8")])
2908
2909 ;;
2910 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
2911 ;;
2912 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
2913   [
2914    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
2915         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2916                        (match_operand:MVE_2 2 "s_register_operand" "w")
2917                        (match_operand:MVE_2 3 "s_register_operand" "w")
2918                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2919          VCADDQ_ROT90_M))
2920   ]
2921   "TARGET_HAVE_MVE"
2922   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
2923   [(set_attr "type" "mve_move")
2924    (set_attr "length""8")])
2925
2926 ;;
2927 ;; [vhaddq_m_n_s, vhaddq_m_n_u]
2928 ;; [vhsubq_m_n_s, vhsubq_m_n_u]
2929 ;; [vmlaq_m_n_s, vmlaq_m_n_u]
2930 ;; [vmlasq_m_n_u, vmlasq_m_n_s]
2931 ;; [vqaddq_m_n_u, vqaddq_m_n_s]
2932 ;; [vqdmlahq_m_n_s]
2933 ;; [vqdmlashq_m_n_s]
2934 ;; [vqdmulhq_m_n_s]
2935 ;; [vqrdmlahq_m_n_s]
2936 ;; [vqrdmlashq_m_n_s]
2937 ;; [vqrdmulhq_m_n_s]
2938 ;; [vqsubq_m_n_u, vqsubq_m_n_s]
2939 ;;
2940 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
2941   [
2942    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2943         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2944                        (match_operand:MVE_2 2 "s_register_operand" "w")
2945                        (match_operand:<V_elem> 3 "s_register_operand" "r")
2946                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2947          MVE_INT_SU_M_N_BINARY))
2948   ]
2949   "TARGET_HAVE_MVE"
2950   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
2951   [(set_attr "type" "mve_move")
2952    (set_attr "length""8")])
2953
2954 ;;
2955 ;;
2956 ;; [vmladavaq_p_u, vmladavaq_p_s]
2957 ;; [vmladavaxq_p_s]
2958 ;; [vmlsdavaq_p_s]
2959 ;; [vmlsdavaxq_p_s]
2960 ;;
2961 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
2962   [
2963    (set (match_operand:SI 0 "s_register_operand" "=Te")
2964         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2965                     (match_operand:MVE_2 2 "s_register_operand" "w")
2966                     (match_operand:MVE_2 3 "s_register_operand" "w")
2967                     (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2968          MVE_VMLxDAVAQ_P))
2969   ]
2970   "TARGET_HAVE_MVE"
2971   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2972   [(set_attr "type" "mve_move")
2973    (set_attr "length""8")])
2974
2975 ;;
2976 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
2977 ;;
2978 (define_insn "mve_vmullbq_int_m_<supf><mode>"
2979   [
2980    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2981         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2982                                   (match_operand:MVE_2 2 "s_register_operand" "w")
2983                                   (match_operand:MVE_2 3 "s_register_operand" "w")
2984                                   (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
2985          VMULLBQ_INT_M))
2986   ]
2987   "TARGET_HAVE_MVE"
2988   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
2989   [(set_attr "type" "mve_move")
2990    (set_attr "length""8")])
2991
2992 ;;
2993 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
2994 ;;
2995 (define_insn "mve_vmulltq_int_m_<supf><mode>"
2996   [
2997    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2998         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2999                                   (match_operand:MVE_2 2 "s_register_operand" "w")
3000                                   (match_operand:MVE_2 3 "s_register_operand" "w")
3001                                   (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3002          VMULLTQ_INT_M))
3003   ]
3004   "TARGET_HAVE_MVE"
3005   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
3006   [(set_attr "type" "mve_move")
3007    (set_attr "length""8")])
3008
3009 ;;
3010 ;; [vornq_m_u, vornq_m_s])
3011 ;;
3012 (define_insn "mve_vornq_m_<supf><mode>"
3013   [
3014    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3015         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3016                        (match_operand:MVE_2 2 "s_register_operand" "w")
3017                        (match_operand:MVE_2 3 "s_register_operand" "w")
3018                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3019          VORNQ_M))
3020   ]
3021   "TARGET_HAVE_MVE"
3022   "vpst\;vornt %q0, %q2, %q3"
3023   [(set_attr "type" "mve_move")
3024    (set_attr "length""8")])
3025
3026 ;;
3027 ;; [vqshlq_m_n_s, vqshlq_m_n_u]
3028 ;; [vshlq_m_n_s, vshlq_m_n_u]
3029 ;;
3030 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
3031   [
3032    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3033         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3034                        (match_operand:MVE_2 2 "s_register_operand" "w")
3035                        (match_operand:SI 3 "immediate_operand" "i")
3036                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3037          MVE_SHIFT_M_N))
3038   ]
3039   "TARGET_HAVE_MVE"
3040   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3041   [(set_attr "type" "mve_move")
3042    (set_attr "length""8")])
3043
3044 ;;
3045 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
3046 ;; [vshrq_m_n_s, vshrq_m_n_u])
3047 ;;
3048 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
3049   [
3050    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3051         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3052                        (match_operand:MVE_2 2 "s_register_operand" "w")
3053                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
3054                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3055          MVE_VSHRQ_M_N))
3056   ]
3057   "TARGET_HAVE_MVE"
3058   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3059   [(set_attr "type" "mve_move")
3060    (set_attr "length""8")])
3061
3062 ;;
3063 ;; [vsliq_m_n_u, vsliq_m_n_s])
3064 ;;
3065 (define_insn "mve_vsliq_m_n_<supf><mode>"
3066    [
3067    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3068        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3069                        (match_operand:MVE_2 2 "s_register_operand" "w")
3070                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
3071                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3072          VSLIQ_M_N))
3073   ]
3074   "TARGET_HAVE_MVE"
3075   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
3076   [(set_attr "type" "mve_move")
3077    (set_attr "length""8")])
3078
3079 ;;
3080 ;; [vhcaddq_rot270_m_s])
3081 ;;
3082 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
3083   [
3084    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
3085         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3086                        (match_operand:MVE_2 2 "s_register_operand" "w")
3087                        (match_operand:MVE_2 3 "s_register_operand" "w")
3088                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3089          VHCADDQ_ROT270_M_S))
3090   ]
3091   "TARGET_HAVE_MVE"
3092   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
3093   [(set_attr "type" "mve_move")
3094    (set_attr "length""8")])
3095
3096 ;;
3097 ;; [vhcaddq_rot90_m_s])
3098 ;;
3099 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
3100   [
3101    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
3102         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3103                        (match_operand:MVE_2 2 "s_register_operand" "w")
3104                        (match_operand:MVE_2 3 "s_register_operand" "w")
3105                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3106          VHCADDQ_ROT90_M_S))
3107   ]
3108   "TARGET_HAVE_MVE"
3109   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
3110   [(set_attr "type" "mve_move")
3111    (set_attr "length""8")])
3112
3113 ;;
3114 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s]
3115 ;; [vmlaldavaxq_p_s]
3116 ;; [vmlsldavaq_p_s]
3117 ;; [vmlsldavaxq_p_s]
3118 ;;
3119 (define_insn "@mve_<mve_insn>q_p_<supf><mode>"
3120   [
3121    (set (match_operand:DI 0 "s_register_operand" "=r")
3122         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3123                        (match_operand:MVE_5 2 "s_register_operand" "w")
3124                        (match_operand:MVE_5 3 "s_register_operand" "w")
3125                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3126          MVE_VMLxLDAVAxQ_P))
3127   ]
3128   "TARGET_HAVE_MVE"
3129   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
3130   [(set_attr "type" "mve_move")
3131    (set_attr "length""8")])
3132
3133 ;;
3134 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s]
3135 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u]
3136 ;; [vqrshrunbq_m_n_s]
3137 ;; [vqrshruntq_m_n_s]
3138 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s]
3139 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u]
3140 ;; [vqshrunbq_m_n_s]
3141 ;; [vqshruntq_m_n_s]
3142 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s]
3143 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s]
3144 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u]
3145 ;; [vshrntq_m_n_s, vshrntq_m_n_u]
3146 ;;
3147 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
3148   [
3149    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3150         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3151                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3152                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
3153                                  (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3154          MVE_SHRN_M_N))
3155   ]
3156   "TARGET_HAVE_MVE"
3157   "vpst\;<mve_insn>t.<isu>%#<V_sz_elem>\t%q0, %q2, %3"
3158   [(set_attr "type" "mve_move")
3159    (set_attr "length""8")])
3160
3161 ;;
3162 ;; [vrmlaldavhaq_p_s])
3163 ;;
3164 (define_insn "mve_vrmlaldavhaq_p_sv4si"
3165   [
3166    (set (match_operand:DI 0 "s_register_operand" "=r")
3167         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3168                        (match_operand:V4SI 2 "s_register_operand" "w")
3169                        (match_operand:V4SI 3 "s_register_operand" "w")
3170                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3171          VRMLALDAVHAQ_P_S))
3172   ]
3173   "TARGET_HAVE_MVE"
3174   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
3175   [(set_attr "type" "mve_move")
3176    (set_attr "length""8")])
3177
3178 ;;
3179 ;; [vshllbq_m_n_u, vshllbq_m_n_s]
3180 ;; [vshlltq_m_n_u, vshlltq_m_n_s]
3181 ;;
3182 (define_insn "@mve_<mve_insn>q_m_n_<supf><mode>"
3183   [
3184    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3185         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3186                        (match_operand:MVE_3 2 "s_register_operand" "w")
3187                        (match_operand:SI 3 "immediate_operand" "i")
3188                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3189          VSHLLxQ_M_N))
3190   ]
3191   "TARGET_HAVE_MVE"
3192   "vpst\;<mve_insn>t.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3193   [(set_attr "type" "mve_move")
3194    (set_attr "length""8")])
3195
3196 ;;
3197 ;; [vmullbq_poly_m_p])
3198 ;;
3199 (define_insn "mve_vmullbq_poly_m_p<mode>"
3200   [
3201    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3202         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3203                        (match_operand:MVE_3 2 "s_register_operand" "w")
3204                        (match_operand:MVE_3 3 "s_register_operand" "w")
3205                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3206          VMULLBQ_POLY_M_P))
3207   ]
3208   "TARGET_HAVE_MVE"
3209   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
3210   [(set_attr "type" "mve_move")
3211    (set_attr "length""8")])
3212
3213 ;;
3214 ;; [vmulltq_poly_m_p])
3215 ;;
3216 (define_insn "mve_vmulltq_poly_m_p<mode>"
3217   [
3218    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3219         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3220                        (match_operand:MVE_3 2 "s_register_operand" "w")
3221                        (match_operand:MVE_3 3 "s_register_operand" "w")
3222                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3223          VMULLTQ_POLY_M_P))
3224   ]
3225   "TARGET_HAVE_MVE"
3226   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
3227   [(set_attr "type" "mve_move")
3228    (set_attr "length""8")])
3229
3230 ;;
3231 ;; [vqdmullbq_m_n_s])
3232 ;;
3233 (define_insn "mve_vqdmullbq_m_n_s<mode>"
3234   [
3235    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3236         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3237                        (match_operand:MVE_5 2 "s_register_operand" "w")
3238                        (match_operand:<V_elem> 3 "s_register_operand" "r")
3239                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3240          VQDMULLBQ_M_N_S))
3241   ]
3242   "TARGET_HAVE_MVE"
3243   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
3244   [(set_attr "type" "mve_move")
3245    (set_attr "length""8")])
3246
3247 ;;
3248 ;; [vqdmullbq_m_s])
3249 ;;
3250 (define_insn "mve_vqdmullbq_m_s<mode>"
3251   [
3252    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3253         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3254                        (match_operand:MVE_5 2 "s_register_operand" "w")
3255                        (match_operand:MVE_5 3 "s_register_operand" "w")
3256                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3257          VQDMULLBQ_M_S))
3258   ]
3259   "TARGET_HAVE_MVE"
3260   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
3261   [(set_attr "type" "mve_move")
3262    (set_attr "length""8")])
3263
3264 ;;
3265 ;; [vqdmulltq_m_n_s])
3266 ;;
3267 (define_insn "mve_vqdmulltq_m_n_s<mode>"
3268   [
3269    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3270         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3271                        (match_operand:MVE_5 2 "s_register_operand" "w")
3272                        (match_operand:<V_elem> 3 "s_register_operand" "r")
3273                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3274          VQDMULLTQ_M_N_S))
3275   ]
3276   "TARGET_HAVE_MVE"
3277   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
3278   [(set_attr "type" "mve_move")
3279    (set_attr "length""8")])
3280
3281 ;;
3282 ;; [vqdmulltq_m_s])
3283 ;;
3284 (define_insn "mve_vqdmulltq_m_s<mode>"
3285   [
3286    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3287         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
3288                        (match_operand:MVE_5 2 "s_register_operand" "w")
3289                        (match_operand:MVE_5 3 "s_register_operand" "w")
3290                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3291          VQDMULLTQ_M_S))
3292   ]
3293   "TARGET_HAVE_MVE"
3294   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
3295   [(set_attr "type" "mve_move")
3296    (set_attr "length""8")])
3297
3298 ;;
3299 ;; [vrmlaldavhaq_p_u])
3300 ;;
3301 (define_insn "mve_vrmlaldavhaq_p_uv4si"
3302   [
3303    (set (match_operand:DI 0 "s_register_operand" "=r")
3304         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3305                        (match_operand:V4SI 2 "s_register_operand" "w")
3306                        (match_operand:V4SI 3 "s_register_operand" "w")
3307                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3308          VRMLALDAVHAQ_P_U))
3309   ]
3310   "TARGET_HAVE_MVE"
3311   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
3312   [(set_attr "type" "mve_move")
3313    (set_attr "length""8")])
3314
3315 ;;
3316 ;; [vrmlaldavhaxq_p_s])
3317 ;;
3318 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
3319   [
3320    (set (match_operand:DI 0 "s_register_operand" "=r")
3321         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3322                        (match_operand:V4SI 2 "s_register_operand" "w")
3323                        (match_operand:V4SI 3 "s_register_operand" "w")
3324                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3325          VRMLALDAVHAXQ_P_S))
3326   ]
3327   "TARGET_HAVE_MVE"
3328   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
3329   [(set_attr "type" "mve_move")
3330    (set_attr "length""8")])
3331
3332 ;;
3333 ;; [vrmlsldavhaq_p_s])
3334 ;;
3335 (define_insn "mve_vrmlsldavhaq_p_sv4si"
3336   [
3337    (set (match_operand:DI 0 "s_register_operand" "=r")
3338         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3339                        (match_operand:V4SI 2 "s_register_operand" "w")
3340                        (match_operand:V4SI 3 "s_register_operand" "w")
3341                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3342          VRMLSLDAVHAQ_P_S))
3343   ]
3344   "TARGET_HAVE_MVE"
3345   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
3346   [(set_attr "type" "mve_move")
3347    (set_attr "length""8")])
3348
3349 ;;
3350 ;; [vrmlsldavhaxq_p_s])
3351 ;;
3352 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
3353   [
3354    (set (match_operand:DI 0 "s_register_operand" "=r")
3355         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3356                        (match_operand:V4SI 2 "s_register_operand" "w")
3357                        (match_operand:V4SI 3 "s_register_operand" "w")
3358                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3359          VRMLSLDAVHAXQ_P_S))
3360   ]
3361   "TARGET_HAVE_MVE"
3362   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
3363   [(set_attr "type" "mve_move")
3364    (set_attr "length""8")])
3365
3366 ;;
3367 ;; [vabdq_m_f]
3368 ;; [vaddq_m_f]
3369 ;; [vmaxnmq_m_f]
3370 ;; [vminnmq_m_f]
3371 ;; [vmulq_m_f]
3372 ;; [vsubq_m_f]
3373 ;;
3374 (define_insn "@mve_<mve_insn>q_m_f<mode>"
3375   [
3376    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3377         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3378                        (match_operand:MVE_0 2 "s_register_operand" "w")
3379                        (match_operand:MVE_0 3 "s_register_operand" "w")
3380                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3381          MVE_FP_M_BINARY))
3382   ]
3383   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3384   "vpst\;<mve_insn>t.f%#<V_sz_elem>     %q0, %q2, %q3"
3385   [(set_attr "type" "mve_move")
3386    (set_attr "length""8")])
3387
3388 ;;
3389 ;; [vaddq_m_n_f]
3390 ;; [vsubq_m_n_f]
3391 ;; [vmulq_m_n_f]
3392 ;;
3393 (define_insn "@mve_<mve_insn>q_m_n_f<mode>"
3394   [
3395    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3396         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3397                        (match_operand:MVE_0 2 "s_register_operand" "w")
3398                        (match_operand:<V_elem> 3 "s_register_operand" "r")
3399                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3400          MVE_FP_M_N_BINARY))
3401   ]
3402   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3403   "vpst\;<mve_insn>t.f%#<V_sz_elem>     %q0, %q2, %3"
3404   [(set_attr "type" "mve_move")
3405    (set_attr "length""8")])
3406
3407 ;;
3408 ;; [vandq_m_f]
3409 ;; [vbicq_m_f]
3410 ;; [veorq_m_f]
3411 ;; [vorrq_m_f]
3412 ;;
3413 (define_insn "@mve_<mve_insn>q_m_f<mode>"
3414   [
3415    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3416         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3417                        (match_operand:MVE_0 2 "s_register_operand" "w")
3418                        (match_operand:MVE_0 3 "s_register_operand" "w")
3419                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3420          MVE_FP_M_BINARY_LOGIC))
3421   ]
3422   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3423   "vpst\;<mve_insn>t %q0, %q2, %q3"
3424   [(set_attr "type" "mve_move")
3425    (set_attr "length""8")])
3426
3427 ;;
3428 ;; [vbrsrq_m_n_f])
3429 ;;
3430 (define_insn "mve_vbrsrq_m_n_f<mode>"
3431   [
3432    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3433         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3434                        (match_operand:MVE_0 2 "s_register_operand" "w")
3435                        (match_operand:SI 3 "s_register_operand" "r")
3436                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3437          VBRSRQ_M_N_F))
3438   ]
3439   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3440   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
3441   [(set_attr "type" "mve_move")
3442    (set_attr "length""8")])
3443
3444 ;;
3445 ;; [vcaddq_rot270_m_f])
3446 ;;
3447 (define_insn "mve_vcaddq_rot270_m_f<mode>"
3448   [
3449    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3450         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3451                        (match_operand:MVE_0 2 "s_register_operand" "w")
3452                        (match_operand:MVE_0 3 "s_register_operand" "w")
3453                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3454          VCADDQ_ROT270_M_F))
3455   ]
3456   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3457   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
3458   [(set_attr "type" "mve_move")
3459    (set_attr "length""8")])
3460
3461 ;;
3462 ;; [vcaddq_rot90_m_f])
3463 ;;
3464 (define_insn "mve_vcaddq_rot90_m_f<mode>"
3465   [
3466    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3467         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3468                        (match_operand:MVE_0 2 "s_register_operand" "w")
3469                        (match_operand:MVE_0 3 "s_register_operand" "w")
3470                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3471          VCADDQ_ROT90_M_F))
3472   ]
3473   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3474   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
3475   [(set_attr "type" "mve_move")
3476    (set_attr "length""8")])
3477
3478 ;;
3479 ;; [vcmlaq_m_f])
3480 ;;
3481 (define_insn "mve_vcmlaq_m_f<mode>"
3482   [
3483    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3484         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3485                        (match_operand:MVE_0 2 "s_register_operand" "w")
3486                        (match_operand:MVE_0 3 "s_register_operand" "w")
3487                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3488          VCMLAQ_M_F))
3489   ]
3490   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3491   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
3492   [(set_attr "type" "mve_move")
3493    (set_attr "length""8")])
3494
3495 ;;
3496 ;; [vcmlaq_rot180_m_f])
3497 ;;
3498 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
3499   [
3500    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3501         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3502                        (match_operand:MVE_0 2 "s_register_operand" "w")
3503                        (match_operand:MVE_0 3 "s_register_operand" "w")
3504                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3505          VCMLAQ_ROT180_M_F))
3506   ]
3507   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3508   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
3509   [(set_attr "type" "mve_move")
3510    (set_attr "length""8")])
3511
3512 ;;
3513 ;; [vcmlaq_rot270_m_f])
3514 ;;
3515 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
3516   [
3517    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3518         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3519                        (match_operand:MVE_0 2 "s_register_operand" "w")
3520                        (match_operand:MVE_0 3 "s_register_operand" "w")
3521                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3522          VCMLAQ_ROT270_M_F))
3523   ]
3524   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3525   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
3526   [(set_attr "type" "mve_move")
3527    (set_attr "length""8")])
3528
3529 ;;
3530 ;; [vcmlaq_rot90_m_f])
3531 ;;
3532 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
3533   [
3534    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3535         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3536                        (match_operand:MVE_0 2 "s_register_operand" "w")
3537                        (match_operand:MVE_0 3 "s_register_operand" "w")
3538                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3539          VCMLAQ_ROT90_M_F))
3540   ]
3541   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3542   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
3543   [(set_attr "type" "mve_move")
3544    (set_attr "length""8")])
3545
3546 ;;
3547 ;; [vcmulq_m_f])
3548 ;;
3549 (define_insn "mve_vcmulq_m_f<mode>"
3550   [
3551    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3552         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3553                        (match_operand:MVE_0 2 "s_register_operand" "w")
3554                        (match_operand:MVE_0 3 "s_register_operand" "w")
3555                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3556          VCMULQ_M_F))
3557   ]
3558   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3559   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
3560   [(set_attr "type" "mve_move")
3561    (set_attr "length""8")])
3562
3563 ;;
3564 ;; [vcmulq_rot180_m_f])
3565 ;;
3566 (define_insn "mve_vcmulq_rot180_m_f<mode>"
3567   [
3568    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3569         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3570                        (match_operand:MVE_0 2 "s_register_operand" "w")
3571                        (match_operand:MVE_0 3 "s_register_operand" "w")
3572                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3573          VCMULQ_ROT180_M_F))
3574   ]
3575   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3576   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
3577   [(set_attr "type" "mve_move")
3578    (set_attr "length""8")])
3579
3580 ;;
3581 ;; [vcmulq_rot270_m_f])
3582 ;;
3583 (define_insn "mve_vcmulq_rot270_m_f<mode>"
3584   [
3585    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3586         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3587                        (match_operand:MVE_0 2 "s_register_operand" "w")
3588                        (match_operand:MVE_0 3 "s_register_operand" "w")
3589                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3590          VCMULQ_ROT270_M_F))
3591   ]
3592   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3593   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
3594   [(set_attr "type" "mve_move")
3595    (set_attr "length""8")])
3596
3597 ;;
3598 ;; [vcmulq_rot90_m_f])
3599 ;;
3600 (define_insn "mve_vcmulq_rot90_m_f<mode>"
3601   [
3602    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
3603         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3604                        (match_operand:MVE_0 2 "s_register_operand" "w")
3605                        (match_operand:MVE_0 3 "s_register_operand" "w")
3606                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3607          VCMULQ_ROT90_M_F))
3608   ]
3609   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3610   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
3611   [(set_attr "type" "mve_move")
3612    (set_attr "length""8")])
3613
3614 ;;
3615 ;; [vfmaq_m_f])
3616 ;;
3617 (define_insn "mve_vfmaq_m_f<mode>"
3618   [
3619    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3620         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3621                        (match_operand:MVE_0 2 "s_register_operand" "w")
3622                        (match_operand:MVE_0 3 "s_register_operand" "w")
3623                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3624          VFMAQ_M_F))
3625   ]
3626   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3627   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
3628   [(set_attr "type" "mve_move")
3629    (set_attr "length""8")])
3630
3631 ;;
3632 ;; [vfmaq_m_n_f])
3633 ;;
3634 (define_insn "mve_vfmaq_m_n_f<mode>"
3635   [
3636    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3637         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3638                        (match_operand:MVE_0 2 "s_register_operand" "w")
3639                        (match_operand:<V_elem> 3 "s_register_operand" "r")
3640                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3641          VFMAQ_M_N_F))
3642   ]
3643   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3644   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
3645   [(set_attr "type" "mve_move")
3646    (set_attr "length""8")])
3647
3648 ;;
3649 ;; [vfmasq_m_n_f])
3650 ;;
3651 (define_insn "mve_vfmasq_m_n_f<mode>"
3652   [
3653    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3654         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3655                        (match_operand:MVE_0 2 "s_register_operand" "w")
3656                        (match_operand:<V_elem> 3 "s_register_operand" "r")
3657                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3658          VFMASQ_M_N_F))
3659   ]
3660   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3661   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
3662   [(set_attr "type" "mve_move")
3663    (set_attr "length""8")])
3664
3665 ;;
3666 ;; [vfmsq_m_f])
3667 ;;
3668 (define_insn "mve_vfmsq_m_f<mode>"
3669   [
3670    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3671         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3672                        (match_operand:MVE_0 2 "s_register_operand" "w")
3673                        (match_operand:MVE_0 3 "s_register_operand" "w")
3674                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3675          VFMSQ_M_F))
3676   ]
3677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3678   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
3679   [(set_attr "type" "mve_move")
3680    (set_attr "length""8")])
3681
3682 ;;
3683 ;; [vornq_m_f])
3684 ;;
3685 (define_insn "mve_vornq_m_f<mode>"
3686   [
3687    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3688         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3689                        (match_operand:MVE_0 2 "s_register_operand" "w")
3690                        (match_operand:MVE_0 3 "s_register_operand" "w")
3691                        (match_operand:<MVE_VPRED> 4 "vpr_register_operand" "Up")]
3692          VORNQ_M_F))
3693   ]
3694   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3695   "vpst\;vornt %q0, %q2, %q3"
3696   [(set_attr "type" "mve_move")
3697    (set_attr "length""8")])
3698
3699 ;;
3700 ;; [vstrbq_s vstrbq_u]
3701 ;;
3702 (define_insn "mve_vstrbq_<supf><mode>"
3703   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
3704         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
3705          VSTRBQ))
3706   ]
3707   "TARGET_HAVE_MVE"
3708 {
3709    rtx ops[2];
3710    int regno = REGNO (operands[1]);
3711    ops[1] = gen_rtx_REG (TImode, regno);
3712    ops[0]  = operands[0];
3713    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
3714    return "";
3715 }
3716   [(set_attr "length" "4")])
3717
3718 ;;
3719 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
3720 ;;
3721 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
3722   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
3723    (match_operand:MVE_2 1 "s_register_operand")
3724    (match_operand:MVE_2 2 "s_register_operand")
3725    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
3726   "TARGET_HAVE_MVE"
3727 {
3728   rtx ind = XEXP (operands[0], 0);
3729   gcc_assert (REG_P (ind));
3730   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
3731                                                               operands[2]));
3732   DONE;
3733 })
3734
3735 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
3736   [(set (mem:BLK (scratch))
3737         (unspec:BLK
3738           [(match_operand:SI 0 "register_operand" "r")
3739            (match_operand:MVE_2 1 "s_register_operand" "w")
3740            (match_operand:MVE_2 2 "s_register_operand" "w")]
3741           VSTRBSOQ))]
3742   "TARGET_HAVE_MVE"
3743   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
3744   [(set_attr "length" "4")])
3745
3746 ;;
3747 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
3748 ;;
3749 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
3750   [(set (mem:BLK (scratch))
3751         (unspec:BLK
3752                 [(match_operand:V4SI 0 "s_register_operand" "w")
3753                  (match_operand:SI 1 "immediate_operand" "i")
3754                  (match_operand:V4SI 2 "s_register_operand" "w")]
3755          VSTRWSBQ))
3756   ]
3757   "TARGET_HAVE_MVE"
3758 {
3759    rtx ops[3];
3760    ops[0] = operands[0];
3761    ops[1] = operands[1];
3762    ops[2] = operands[2];
3763    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
3764    return "";
3765 }
3766   [(set_attr "length" "4")])
3767
3768 ;;
3769 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
3770 ;;
3771 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
3772   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
3773         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
3774                        (match_operand:MVE_2 2 "s_register_operand" "w")]
3775          VLDRBGOQ))
3776   ]
3777   "TARGET_HAVE_MVE"
3778 {
3779    rtx ops[3];
3780    ops[0] = operands[0];
3781    ops[1] = operands[1];
3782    ops[2] = operands[2];
3783    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
3784      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
3785    else
3786      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
3787    return "";
3788 }
3789   [(set_attr "length" "4")])
3790
3791 ;;
3792 ;; [vldrbq_s vldrbq_u]
3793 ;;
3794 (define_insn "mve_vldrbq_<supf><mode>"
3795   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3796         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
3797          VLDRBQ))
3798   ]
3799   "TARGET_HAVE_MVE"
3800 {
3801    rtx ops[2];
3802    int regno = REGNO (operands[0]);
3803    ops[0] = gen_rtx_REG (TImode, regno);
3804    ops[1]  = operands[1];
3805    if (<V_sz_elem> == 8)
3806      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
3807    else
3808      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
3809    return "";
3810 }
3811   [(set_attr "length" "4")])
3812
3813 ;;
3814 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
3815 ;;
3816 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
3817   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
3818         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
3819                       (match_operand:SI 2 "immediate_operand" "i")]
3820          VLDRWGBQ))
3821   ]
3822   "TARGET_HAVE_MVE"
3823 {
3824    rtx ops[3];
3825    ops[0] = operands[0];
3826    ops[1] = operands[1];
3827    ops[2] = operands[2];
3828    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
3829    return "";
3830 }
3831   [(set_attr "length" "4")])
3832
3833 ;;
3834 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
3835 ;;
3836 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
3837   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
3838    (match_operand:MVE_2 1 "s_register_operand")
3839    (match_operand:MVE_2 2 "s_register_operand")
3840    (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")
3841    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
3842   "TARGET_HAVE_MVE"
3843 {
3844   rtx ind = XEXP (operands[0], 0);
3845   gcc_assert (REG_P (ind));
3846   emit_insn (
3847     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
3848                                                        operands[2],
3849                                                        operands[3]));
3850   DONE;
3851 })
3852
3853 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
3854   [(set (mem:BLK (scratch))
3855         (unspec:BLK
3856           [(match_operand:SI 0 "register_operand" "r")
3857            (match_operand:MVE_2 1 "s_register_operand" "w")
3858            (match_operand:MVE_2 2 "s_register_operand" "w")
3859            (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
3860           VSTRBSOQ))]
3861   "TARGET_HAVE_MVE"
3862   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
3863   [(set_attr "length" "8")])
3864
3865 ;;
3866 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
3867 ;;
3868 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
3869   [(set (mem:BLK (scratch))
3870         (unspec:BLK
3871                 [(match_operand:V4SI 0 "s_register_operand" "w")
3872                  (match_operand:SI 1 "immediate_operand" "i")
3873                  (match_operand:V4SI 2 "s_register_operand" "w")
3874                  (match_operand:V4BI 3 "vpr_register_operand" "Up")]
3875          VSTRWSBQ))
3876   ]
3877   "TARGET_HAVE_MVE"
3878 {
3879    rtx ops[3];
3880    ops[0] = operands[0];
3881    ops[1] = operands[1];
3882    ops[2] = operands[2];
3883    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
3884    return "";
3885 }
3886   [(set_attr "length" "8")])
3887
3888 (define_insn "mve_vstrbq_p_<supf><mode>"
3889   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
3890         (unspec:<MVE_B_ELEM>
3891          [(match_operand:MVE_2 1 "s_register_operand" "w")
3892           (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")
3893           (match_dup 0)]
3894          VSTRBQ))]
3895   "TARGET_HAVE_MVE"
3896 {
3897    rtx ops[2];
3898    int regno = REGNO (operands[1]);
3899    ops[1] = gen_rtx_REG (TImode, regno);
3900    ops[0]  = operands[0];
3901    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
3902    return "";
3903 }
3904   [(set_attr "length" "8")])
3905
3906 ;;
3907 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
3908 ;;
3909 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
3910   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
3911         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
3912                        (match_operand:MVE_2 2 "s_register_operand" "w")
3913                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
3914          VLDRBGOQ))
3915   ]
3916   "TARGET_HAVE_MVE"
3917 {
3918    rtx ops[4];
3919    ops[0] = operands[0];
3920    ops[1] = operands[1];
3921    ops[2] = operands[2];
3922    ops[3] = operands[3];
3923    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
3924      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
3925    else
3926      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
3927    return "";
3928 }
3929   [(set_attr "length" "8")])
3930
3931 ;;
3932 ;; [vldrbq_z_s vldrbq_z_u]
3933 ;;
3934 (define_insn "mve_vldrbq_z_<supf><mode>"
3935   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3936         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
3937                        (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")]
3938          VLDRBQ))
3939   ]
3940   "TARGET_HAVE_MVE"
3941 {
3942    rtx ops[2];
3943    int regno = REGNO (operands[0]);
3944    ops[0] = gen_rtx_REG (TImode, regno);
3945    ops[1]  = operands[1];
3946    if (<V_sz_elem> == 8)
3947      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
3948    else
3949      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
3950    return "";
3951 }
3952   [(set_attr "length" "8")])
3953
3954 ;;
3955 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
3956 ;;
3957 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
3958   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
3959         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
3960                       (match_operand:SI 2 "immediate_operand" "i")
3961                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
3962          VLDRWGBQ))
3963   ]
3964   "TARGET_HAVE_MVE"
3965 {
3966    rtx ops[3];
3967    ops[0] = operands[0];
3968    ops[1] = operands[1];
3969    ops[2] = operands[2];
3970    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
3971    return "";
3972 }
3973   [(set_attr "length" "8")])
3974
3975 ;;
3976 ;; [vldrhq_f]
3977 ;;
3978 (define_insn "mve_vldrhq_fv8hf"
3979   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
3980         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
3981          VLDRHQ_F))
3982   ]
3983   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3984 {
3985    rtx ops[2];
3986    int regno = REGNO (operands[0]);
3987    ops[0] = gen_rtx_REG (TImode, regno);
3988    ops[1]  = operands[1];
3989    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
3990    return "";
3991 }
3992   [(set_attr "length" "4")])
3993
3994 ;;
3995 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
3996 ;;
3997 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
3998   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
3999         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
4000                        (match_operand:MVE_6 2 "s_register_operand" "w")]
4001         VLDRHGOQ))
4002   ]
4003   "TARGET_HAVE_MVE"
4004 {
4005    rtx ops[3];
4006    ops[0] = operands[0];
4007    ops[1] = operands[1];
4008    ops[2] = operands[2];
4009    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
4010      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
4011    else
4012      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
4013    return "";
4014 }
4015   [(set_attr "length" "4")])
4016
4017 ;;
4018 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
4019 ;;
4020 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
4021   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
4022         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
4023                        (match_operand:MVE_6 2 "s_register_operand" "w")
4024                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")
4025         ]VLDRHGOQ))
4026   ]
4027   "TARGET_HAVE_MVE"
4028 {
4029    rtx ops[4];
4030    ops[0] = operands[0];
4031    ops[1] = operands[1];
4032    ops[2] = operands[2];
4033    ops[3] = operands[3];
4034    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
4035      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
4036    else
4037      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
4038    return "";
4039 }
4040  [(set_attr "length" "8")])
4041
4042 ;;
4043 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
4044 ;;
4045 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
4046   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
4047         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
4048                        (match_operand:MVE_6 2 "s_register_operand" "w")]
4049         VLDRHGSOQ))
4050   ]
4051   "TARGET_HAVE_MVE"
4052 {
4053    rtx ops[3];
4054    ops[0] = operands[0];
4055    ops[1] = operands[1];
4056    ops[2] = operands[2];
4057       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
4058      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
4059    else
4060      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
4061    return "";
4062 }
4063   [(set_attr "length" "4")])
4064
4065 ;;
4066 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
4067 ;;
4068 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
4069   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
4070         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
4071                        (match_operand:MVE_6 2 "s_register_operand" "w")
4072                        (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")
4073         ]VLDRHGSOQ))
4074   ]
4075   "TARGET_HAVE_MVE"
4076 {
4077    rtx ops[4];
4078    ops[0] = operands[0];
4079    ops[1] = operands[1];
4080    ops[2] = operands[2];
4081    ops[3] = operands[3];
4082    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
4083      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
4084    else
4085      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
4086    return "";
4087 }
4088   [(set_attr "length" "8")])
4089
4090 ;;
4091 ;; [vldrhq_s, vldrhq_u]
4092 ;;
4093 (define_insn "mve_vldrhq_<supf><mode>"
4094   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
4095         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
4096          VLDRHQ))
4097   ]
4098   "TARGET_HAVE_MVE"
4099 {
4100    rtx ops[2];
4101    int regno = REGNO (operands[0]);
4102    ops[0] = gen_rtx_REG (TImode, regno);
4103    ops[1]  = operands[1];
4104    if (<V_sz_elem> == 16)
4105      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
4106    else
4107      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
4108    return "";
4109 }
4110   [(set_attr "length" "4")])
4111
4112 ;;
4113 ;; [vldrhq_z_f]
4114 ;;
4115 (define_insn "mve_vldrhq_z_fv8hf"
4116   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
4117         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
4118         (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")]
4119          VLDRHQ_F))
4120   ]
4121   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4122 {
4123    rtx ops[2];
4124    int regno = REGNO (operands[0]);
4125    ops[0] = gen_rtx_REG (TImode, regno);
4126    ops[1]  = operands[1];
4127    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
4128    return "";
4129 }
4130   [(set_attr "length" "8")])
4131
4132 ;;
4133 ;; [vldrhq_z_s vldrhq_z_u]
4134 ;;
4135 (define_insn "mve_vldrhq_z_<supf><mode>"
4136   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
4137         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
4138         (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")]
4139          VLDRHQ))
4140   ]
4141   "TARGET_HAVE_MVE"
4142 {
4143    rtx ops[2];
4144    int regno = REGNO (operands[0]);
4145    ops[0] = gen_rtx_REG (TImode, regno);
4146    ops[1]  = operands[1];
4147    if (<V_sz_elem> == 16)
4148      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
4149    else
4150      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
4151    return "";
4152 }
4153   [(set_attr "length" "8")])
4154
4155 ;;
4156 ;; [vldrwq_f]
4157 ;;
4158 (define_insn "mve_vldrwq_fv4sf"
4159   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
4160         (unspec:V4SF [(match_operand:V4SI 1 "mve_memory_operand" "Ux")]
4161          VLDRWQ_F))
4162   ]
4163   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4164 {
4165    rtx ops[2];
4166    int regno = REGNO (operands[0]);
4167    ops[0] = gen_rtx_REG (TImode, regno);
4168    ops[1]  = operands[1];
4169    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
4170    return "";
4171 }
4172   [(set_attr "length" "4")])
4173
4174 ;;
4175 ;; [vldrwq_s vldrwq_u]
4176 ;;
4177 (define_insn "mve_vldrwq_<supf>v4si"
4178   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
4179         (unspec:V4SI [(match_operand:V4SI 1 "mve_memory_operand" "Ux")]
4180          VLDRWQ))
4181   ]
4182   "TARGET_HAVE_MVE"
4183 {
4184    rtx ops[2];
4185    int regno = REGNO (operands[0]);
4186    ops[0] = gen_rtx_REG (TImode, regno);
4187    ops[1]  = operands[1];
4188    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
4189    return "";
4190 }
4191   [(set_attr "length" "4")])
4192
4193 ;;
4194 ;; [vldrwq_z_f]
4195 ;;
4196 (define_insn "mve_vldrwq_z_fv4sf"
4197   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
4198         (unspec:V4SF [(match_operand:V4SI 1 "mve_memory_operand" "Ux")
4199         (match_operand:V4BI 2 "vpr_register_operand" "Up")]
4200          VLDRWQ_F))
4201   ]
4202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4203 {
4204    rtx ops[2];
4205    int regno = REGNO (operands[0]);
4206    ops[0] = gen_rtx_REG (TImode, regno);
4207    ops[1]  = operands[1];
4208    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
4209    return "";
4210 }
4211   [(set_attr "length" "8")])
4212
4213 ;;
4214 ;; [vldrwq_z_s vldrwq_z_u]
4215 ;;
4216 (define_insn "mve_vldrwq_z_<supf>v4si"
4217   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
4218         (unspec:V4SI [(match_operand:V4SI 1 "mve_memory_operand" "Ux")
4219         (match_operand:V4BI 2 "vpr_register_operand" "Up")]
4220          VLDRWQ))
4221   ]
4222   "TARGET_HAVE_MVE"
4223 {
4224    rtx ops[2];
4225    int regno = REGNO (operands[0]);
4226    ops[0] = gen_rtx_REG (TImode, regno);
4227    ops[1]  = operands[1];
4228    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
4229    return "";
4230 }
4231   [(set_attr "length" "8")])
4232
4233 (define_expand "mve_vld1q_f<mode>"
4234   [(match_operand:MVE_0 0 "s_register_operand")
4235    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
4236   ]
4237   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
4238 {
4239   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
4240   DONE;
4241 })
4242
4243 (define_expand "mve_vld1q_<supf><mode>"
4244   [(match_operand:MVE_2 0 "s_register_operand")
4245    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
4246   ]
4247   "TARGET_HAVE_MVE"
4248 {
4249   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
4250   DONE;
4251 })
4252
4253 ;;
4254 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
4255 ;;
4256 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
4257   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4258         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
4259                       (match_operand:SI 2 "immediate_operand" "i")]
4260          VLDRDGBQ))
4261   ]
4262   "TARGET_HAVE_MVE"
4263 {
4264    rtx ops[3];
4265    ops[0] = operands[0];
4266    ops[1] = operands[1];
4267    ops[2] = operands[2];
4268    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
4269    return "";
4270 }
4271   [(set_attr "length" "4")])
4272
4273 ;;
4274 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
4275 ;;
4276 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
4277   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4278         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
4279                       (match_operand:SI 2 "immediate_operand" "i")
4280                       (match_operand:V2QI 3 "vpr_register_operand" "Up")]
4281          VLDRDGBQ))
4282   ]
4283   "TARGET_HAVE_MVE"
4284 {
4285    rtx ops[3];
4286    ops[0] = operands[0];
4287    ops[1] = operands[1];
4288    ops[2] = operands[2];
4289    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
4290    return "";
4291 }
4292   [(set_attr "length" "8")])
4293
4294 ;;
4295 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
4296 ;;
4297 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
4298  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4299        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
4300                      (match_operand:V2DI 2 "s_register_operand" "w")]
4301         VLDRDGOQ))
4302  ]
4303  "TARGET_HAVE_MVE"
4304 {
4305   rtx ops[3];
4306   ops[0] = operands[0];
4307   ops[1] = operands[1];
4308   ops[2] = operands[2];
4309   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
4310   return "";
4311 }
4312  [(set_attr "length" "4")])
4313
4314 ;;
4315 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
4316 ;;
4317 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
4318  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4319        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
4320                      (match_operand:V2DI 2 "s_register_operand" "w")
4321                      (match_operand:V2QI 3 "vpr_register_operand" "Up")]
4322         VLDRDGOQ))
4323  ]
4324  "TARGET_HAVE_MVE"
4325 {
4326   rtx ops[3];
4327   ops[0] = operands[0];
4328   ops[1] = operands[1];
4329   ops[2] = operands[2];
4330   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
4331   return "";
4332 }
4333  [(set_attr "length" "8")])
4334
4335 ;;
4336 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
4337 ;;
4338 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
4339   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4340         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
4341                       (match_operand:V2DI 2 "s_register_operand" "w")]
4342          VLDRDGSOQ))
4343   ]
4344   "TARGET_HAVE_MVE"
4345 {
4346    rtx ops[3];
4347    ops[0] = operands[0];
4348    ops[1] = operands[1];
4349    ops[2] = operands[2];
4350    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
4351    return "";
4352 }
4353   [(set_attr "length" "4")])
4354
4355 ;;
4356 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
4357 ;;
4358 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
4359   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
4360         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
4361                       (match_operand:V2DI 2 "s_register_operand" "w")
4362                       (match_operand:V2QI 3 "vpr_register_operand" "Up")]
4363          VLDRDGSOQ))
4364   ]
4365   "TARGET_HAVE_MVE"
4366 {
4367    rtx ops[3];
4368    ops[0] = operands[0];
4369    ops[1] = operands[1];
4370    ops[2] = operands[2];
4371    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
4372    return "";
4373 }
4374   [(set_attr "length" "8")])
4375
4376 ;;
4377 ;; [vldrhq_gather_offset_f]
4378 ;;
4379 (define_insn "mve_vldrhq_gather_offset_fv8hf"
4380   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
4381         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
4382                       (match_operand:V8HI 2 "s_register_operand" "w")]
4383          VLDRHQGO_F))
4384   ]
4385   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4386 {
4387    rtx ops[3];
4388    ops[0] = operands[0];
4389    ops[1] = operands[1];
4390    ops[2] = operands[2];
4391    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
4392    return "";
4393 }
4394   [(set_attr "length" "4")])
4395
4396 ;;
4397 ;; [vldrhq_gather_offset_z_f]
4398 ;;
4399 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
4400   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
4401         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
4402                       (match_operand:V8HI 2 "s_register_operand" "w")
4403                       (match_operand:V8BI 3 "vpr_register_operand" "Up")]
4404          VLDRHQGO_F))
4405   ]
4406   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4407 {
4408    rtx ops[4];
4409    ops[0] = operands[0];
4410    ops[1] = operands[1];
4411    ops[2] = operands[2];
4412    ops[3] = operands[3];
4413    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
4414    return "";
4415 }
4416   [(set_attr "length" "8")])
4417
4418 ;;
4419 ;; [vldrhq_gather_shifted_offset_f]
4420 ;;
4421 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
4422   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
4423         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
4424                       (match_operand:V8HI 2 "s_register_operand" "w")]
4425          VLDRHQGSO_F))
4426   ]
4427   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4428 {
4429    rtx ops[3];
4430    ops[0] = operands[0];
4431    ops[1] = operands[1];
4432    ops[2] = operands[2];
4433    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
4434    return "";
4435 }
4436   [(set_attr "length" "4")])
4437
4438 ;;
4439 ;; [vldrhq_gather_shifted_offset_z_f]
4440 ;;
4441 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
4442   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
4443         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
4444                       (match_operand:V8HI 2 "s_register_operand" "w")
4445                       (match_operand:V8BI 3 "vpr_register_operand" "Up")]
4446          VLDRHQGSO_F))
4447   ]
4448   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4449 {
4450    rtx ops[4];
4451    ops[0] = operands[0];
4452    ops[1] = operands[1];
4453    ops[2] = operands[2];
4454    ops[3] = operands[3];
4455    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
4456    return "";
4457 }
4458   [(set_attr "length" "8")])
4459
4460 ;;
4461 ;; [vldrwq_gather_base_f]
4462 ;;
4463 (define_insn "mve_vldrwq_gather_base_fv4sf"
4464   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4465         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
4466                       (match_operand:SI 2 "immediate_operand" "i")]
4467          VLDRWQGB_F))
4468   ]
4469   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4470 {
4471    rtx ops[3];
4472    ops[0] = operands[0];
4473    ops[1] = operands[1];
4474    ops[2] = operands[2];
4475    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
4476    return "";
4477 }
4478   [(set_attr "length" "4")])
4479
4480 ;;
4481 ;; [vldrwq_gather_base_z_f]
4482 ;;
4483 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
4484   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4485         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
4486                       (match_operand:SI 2 "immediate_operand" "i")
4487                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
4488          VLDRWQGB_F))
4489   ]
4490   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4491 {
4492    rtx ops[3];
4493    ops[0] = operands[0];
4494    ops[1] = operands[1];
4495    ops[2] = operands[2];
4496    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
4497    return "";
4498 }
4499   [(set_attr "length" "8")])
4500
4501 ;;
4502 ;; [vldrwq_gather_offset_f]
4503 ;;
4504 (define_insn "mve_vldrwq_gather_offset_fv4sf"
4505   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4506         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
4507                        (match_operand:V4SI 2 "s_register_operand" "w")]
4508          VLDRWQGO_F))
4509   ]
4510   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4511 {
4512    rtx ops[3];
4513    ops[0] = operands[0];
4514    ops[1] = operands[1];
4515    ops[2] = operands[2];
4516    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
4517    return "";
4518 }
4519   [(set_attr "length" "4")])
4520
4521 ;;
4522 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
4523 ;;
4524 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
4525   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
4526         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
4527                        (match_operand:V4SI 2 "s_register_operand" "w")]
4528          VLDRWGOQ))
4529   ]
4530   "TARGET_HAVE_MVE"
4531 {
4532    rtx ops[3];
4533    ops[0] = operands[0];
4534    ops[1] = operands[1];
4535    ops[2] = operands[2];
4536    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
4537    return "";
4538 }
4539   [(set_attr "length" "4")])
4540
4541 ;;
4542 ;; [vldrwq_gather_offset_z_f]
4543 ;;
4544 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
4545   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4546         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
4547                       (match_operand:V4SI 2 "s_register_operand" "w")
4548                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
4549          VLDRWQGO_F))
4550   ]
4551   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4552 {
4553    rtx ops[4];
4554    ops[0] = operands[0];
4555    ops[1] = operands[1];
4556    ops[2] = operands[2];
4557    ops[3] = operands[3];
4558    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
4559    return "";
4560 }
4561   [(set_attr "length" "8")])
4562
4563 ;;
4564 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
4565 ;;
4566 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
4567   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
4568         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
4569                       (match_operand:V4SI 2 "s_register_operand" "w")
4570                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
4571          VLDRWGOQ))
4572   ]
4573   "TARGET_HAVE_MVE"
4574 {
4575    rtx ops[4];
4576    ops[0] = operands[0];
4577    ops[1] = operands[1];
4578    ops[2] = operands[2];
4579    ops[3] = operands[3];
4580    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
4581    return "";
4582 }
4583   [(set_attr "length" "8")])
4584
4585 ;;
4586 ;; [vldrwq_gather_shifted_offset_f]
4587 ;;
4588 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
4589   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4590         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
4591                       (match_operand:V4SI 2 "s_register_operand" "w")]
4592          VLDRWQGSO_F))
4593   ]
4594   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4595 {
4596    rtx ops[3];
4597    ops[0] = operands[0];
4598    ops[1] = operands[1];
4599    ops[2] = operands[2];
4600    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
4601    return "";
4602 }
4603   [(set_attr "length" "4")])
4604
4605 ;;
4606 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
4607 ;;
4608 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
4609   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
4610         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
4611                       (match_operand:V4SI 2 "s_register_operand" "w")]
4612          VLDRWGSOQ))
4613   ]
4614   "TARGET_HAVE_MVE"
4615 {
4616    rtx ops[3];
4617    ops[0] = operands[0];
4618    ops[1] = operands[1];
4619    ops[2] = operands[2];
4620    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
4621    return "";
4622 }
4623   [(set_attr "length" "4")])
4624
4625 ;;
4626 ;; [vldrwq_gather_shifted_offset_z_f]
4627 ;;
4628 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
4629   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
4630         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
4631                       (match_operand:V4SI 2 "s_register_operand" "w")
4632                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
4633          VLDRWQGSO_F))
4634   ]
4635   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4636 {
4637    rtx ops[4];
4638    ops[0] = operands[0];
4639    ops[1] = operands[1];
4640    ops[2] = operands[2];
4641    ops[3] = operands[3];
4642    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
4643    return "";
4644 }
4645   [(set_attr "length" "8")])
4646
4647 ;;
4648 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
4649 ;;
4650 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
4651   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
4652         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
4653                       (match_operand:V4SI 2 "s_register_operand" "w")
4654                       (match_operand:V4BI 3 "vpr_register_operand" "Up")]
4655          VLDRWGSOQ))
4656   ]
4657   "TARGET_HAVE_MVE"
4658 {
4659    rtx ops[4];
4660    ops[0] = operands[0];
4661    ops[1] = operands[1];
4662    ops[2] = operands[2];
4663    ops[3] = operands[3];
4664    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
4665    return "";
4666 }
4667   [(set_attr "length" "8")])
4668
4669 ;;
4670 ;; [vstrhq_f]
4671 ;;
4672 (define_insn "mve_vstrhq_fv8hf"
4673   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
4674         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
4675          VSTRHQ_F))
4676   ]
4677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4678 {
4679    rtx ops[2];
4680    int regno = REGNO (operands[1]);
4681    ops[1] = gen_rtx_REG (TImode, regno);
4682    ops[0]  = operands[0];
4683    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
4684    return "";
4685 }
4686   [(set_attr "length" "4")])
4687
4688 ;;
4689 ;; [vstrhq_p_f]
4690 ;;
4691 (define_insn "mve_vstrhq_p_fv8hf"
4692   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
4693         (unspec:V8HI
4694          [(match_operand:V8HF 1 "s_register_operand" "w")
4695           (match_operand:V8BI 2 "vpr_register_operand" "Up")
4696           (match_dup 0)]
4697          VSTRHQ_F))]
4698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4699 {
4700    rtx ops[2];
4701    int regno = REGNO (operands[1]);
4702    ops[1] = gen_rtx_REG (TImode, regno);
4703    ops[0]  = operands[0];
4704    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
4705    return "";
4706 }
4707   [(set_attr "length" "8")])
4708
4709 ;;
4710 ;; [vstrhq_p_s vstrhq_p_u]
4711 ;;
4712 (define_insn "mve_vstrhq_p_<supf><mode>"
4713   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
4714         (unspec:<MVE_H_ELEM>
4715          [(match_operand:MVE_6 1 "s_register_operand" "w")
4716           (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")
4717           (match_dup 0)]
4718          VSTRHQ))
4719   ]
4720   "TARGET_HAVE_MVE"
4721 {
4722    rtx ops[2];
4723    int regno = REGNO (operands[1]);
4724    ops[1] = gen_rtx_REG (TImode, regno);
4725    ops[0]  = operands[0];
4726    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
4727    return "";
4728 }
4729   [(set_attr "length" "8")])
4730
4731 ;;
4732 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
4733 ;;
4734 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
4735   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
4736    (match_operand:MVE_6 1 "s_register_operand")
4737    (match_operand:MVE_6 2 "s_register_operand")
4738    (match_operand:<MVE_VPRED> 3 "vpr_register_operand")
4739    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
4740   "TARGET_HAVE_MVE"
4741 {
4742   rtx ind = XEXP (operands[0], 0);
4743   gcc_assert (REG_P (ind));
4744   emit_insn (
4745     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
4746                                                        operands[2],
4747                                                        operands[3]));
4748   DONE;
4749 })
4750
4751 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
4752   [(set (mem:BLK (scratch))
4753         (unspec:BLK
4754           [(match_operand:SI 0 "register_operand" "r")
4755            (match_operand:MVE_6 1 "s_register_operand" "w")
4756            (match_operand:MVE_6 2 "s_register_operand" "w")
4757            (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
4758           VSTRHSOQ))]
4759   "TARGET_HAVE_MVE"
4760   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
4761   [(set_attr "length" "8")])
4762
4763 ;;
4764 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
4765 ;;
4766 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
4767   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
4768    (match_operand:MVE_6 1 "s_register_operand")
4769    (match_operand:MVE_6 2 "s_register_operand")
4770    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
4771   "TARGET_HAVE_MVE"
4772 {
4773   rtx ind = XEXP (operands[0], 0);
4774   gcc_assert (REG_P (ind));
4775   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
4776                                                               operands[2]));
4777   DONE;
4778 })
4779
4780 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
4781   [(set (mem:BLK (scratch))
4782         (unspec:BLK
4783           [(match_operand:SI 0 "register_operand" "r")
4784            (match_operand:MVE_6 1 "s_register_operand" "w")
4785            (match_operand:MVE_6 2 "s_register_operand" "w")]
4786           VSTRHSOQ))]
4787   "TARGET_HAVE_MVE"
4788   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
4789   [(set_attr "length" "4")])
4790
4791 ;;
4792 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
4793 ;;
4794 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
4795   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
4796    (match_operand:MVE_6 1 "s_register_operand")
4797    (match_operand:MVE_6 2 "s_register_operand")
4798    (match_operand:<MVE_VPRED> 3 "vpr_register_operand")
4799    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
4800   "TARGET_HAVE_MVE"
4801 {
4802   rtx ind = XEXP (operands[0], 0);
4803   gcc_assert (REG_P (ind));
4804   emit_insn (
4805     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
4806                                                                operands[2],
4807                                                                operands[3]));
4808   DONE;
4809 })
4810
4811 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
4812   [(set (mem:BLK (scratch))
4813         (unspec:BLK
4814           [(match_operand:SI 0 "register_operand" "r")
4815            (match_operand:MVE_6 1 "s_register_operand" "w")
4816            (match_operand:MVE_6 2 "s_register_operand" "w")
4817            (match_operand:<MVE_VPRED> 3 "vpr_register_operand" "Up")]
4818           VSTRHSSOQ))]
4819   "TARGET_HAVE_MVE"
4820   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
4821   [(set_attr "length" "8")])
4822
4823 ;;
4824 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
4825 ;;
4826 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
4827   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
4828    (match_operand:MVE_6 1 "s_register_operand")
4829    (match_operand:MVE_6 2 "s_register_operand")
4830    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
4831   "TARGET_HAVE_MVE"
4832 {
4833   rtx ind = XEXP (operands[0], 0);
4834   gcc_assert (REG_P (ind));
4835   emit_insn (
4836     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
4837                                                              operands[2]));
4838   DONE;
4839 })
4840
4841 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
4842   [(set (mem:BLK (scratch))
4843         (unspec:BLK
4844           [(match_operand:SI 0 "register_operand" "r")
4845            (match_operand:MVE_6 1 "s_register_operand" "w")
4846            (match_operand:MVE_6 2 "s_register_operand" "w")]
4847           VSTRHSSOQ))]
4848   "TARGET_HAVE_MVE"
4849   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
4850   [(set_attr "length" "4")])
4851
4852 ;;
4853 ;; [vstrhq_s, vstrhq_u]
4854 ;;
4855 (define_insn "mve_vstrhq_<supf><mode>"
4856   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
4857         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
4858          VSTRHQ))
4859   ]
4860   "TARGET_HAVE_MVE"
4861 {
4862    rtx ops[2];
4863    int regno = REGNO (operands[1]);
4864    ops[1] = gen_rtx_REG (TImode, regno);
4865    ops[0]  = operands[0];
4866    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
4867    return "";
4868 }
4869   [(set_attr "length" "4")])
4870
4871 ;;
4872 ;; [vstrwq_f]
4873 ;;
4874 (define_insn "mve_vstrwq_fv4sf"
4875   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
4876         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
4877          VSTRWQ_F))
4878   ]
4879   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4880 {
4881    rtx ops[2];
4882    int regno = REGNO (operands[1]);
4883    ops[1] = gen_rtx_REG (TImode, regno);
4884    ops[0]  = operands[0];
4885    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
4886    return "";
4887 }
4888   [(set_attr "length" "4")])
4889
4890 ;;
4891 ;; [vstrwq_p_f]
4892 ;;
4893 (define_insn "mve_vstrwq_p_fv4sf"
4894   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
4895         (unspec:V4SI
4896          [(match_operand:V4SF 1 "s_register_operand" "w")
4897           (match_operand:<MVE_VPRED> 2 "vpr_register_operand" "Up")
4898           (match_dup 0)]
4899          VSTRWQ_F))]
4900   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4901 {
4902    rtx ops[2];
4903    int regno = REGNO (operands[1]);
4904    ops[1] = gen_rtx_REG (TImode, regno);
4905    ops[0]  = operands[0];
4906    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
4907    return "";
4908 }
4909   [(set_attr "length" "8")])
4910
4911 ;;
4912 ;; [vstrwq_p_s vstrwq_p_u]
4913 ;;
4914 (define_insn "mve_vstrwq_p_<supf>v4si"
4915   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
4916         (unspec:V4SI
4917          [(match_operand:V4SI 1 "s_register_operand" "w")
4918           (match_operand:V4BI 2 "vpr_register_operand" "Up")
4919           (match_dup 0)]
4920          VSTRWQ))]
4921   "TARGET_HAVE_MVE"
4922 {
4923    rtx ops[2];
4924    int regno = REGNO (operands[1]);
4925    ops[1] = gen_rtx_REG (TImode, regno);
4926    ops[0]  = operands[0];
4927    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
4928    return "";
4929 }
4930   [(set_attr "length" "8")])
4931
4932 ;;
4933 ;; [vstrwq_s vstrwq_u]
4934 ;;
4935 (define_insn "mve_vstrwq_<supf>v4si"
4936   [(set (match_operand:V4SI 0 "mve_memory_operand" "=Ux")
4937         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
4938          VSTRWQ))
4939   ]
4940   "TARGET_HAVE_MVE"
4941 {
4942    rtx ops[2];
4943    int regno = REGNO (operands[1]);
4944    ops[1] = gen_rtx_REG (TImode, regno);
4945    ops[0]  = operands[0];
4946    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
4947    return "";
4948 }
4949   [(set_attr "length" "4")])
4950
4951 (define_expand "mve_vst1q_f<mode>"
4952   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
4953    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
4954   ]
4955   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
4956 {
4957   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
4958   DONE;
4959 })
4960
4961 (define_expand "mve_vst1q_<supf><mode>"
4962   [(match_operand:MVE_2 0 "mve_memory_operand")
4963    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
4964   ]
4965   "TARGET_HAVE_MVE"
4966 {
4967   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
4968   DONE;
4969 })
4970
4971 ;;
4972 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
4973 ;;
4974 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
4975   [(set (mem:BLK (scratch))
4976         (unspec:BLK
4977                 [(match_operand:V2DI 0 "s_register_operand" "w")
4978                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
4979                  (match_operand:V2DI 2 "s_register_operand" "w")
4980                  (match_operand:V2QI 3 "vpr_register_operand" "Up")]
4981          VSTRDSBQ))
4982   ]
4983   "TARGET_HAVE_MVE"
4984 {
4985    rtx ops[3];
4986    ops[0] = operands[0];
4987    ops[1] = operands[1];
4988    ops[2] = operands[2];
4989    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
4990    return "";
4991 }
4992   [(set_attr "length" "8")])
4993
4994 ;;
4995 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
4996 ;;
4997 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
4998   [(set (mem:BLK (scratch))
4999         (unspec:BLK
5000                 [(match_operand:V2DI 0 "s_register_operand" "=w")
5001                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
5002                  (match_operand:V2DI 2 "s_register_operand" "w")]
5003          VSTRDSBQ))
5004   ]
5005   "TARGET_HAVE_MVE"
5006 {
5007    rtx ops[3];
5008    ops[0] = operands[0];
5009    ops[1] = operands[1];
5010    ops[2] = operands[2];
5011    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
5012    return "";
5013 }
5014   [(set_attr "length" "4")])
5015
5016 ;;
5017 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
5018 ;;
5019 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
5020   [(match_operand:V2DI 0 "mve_scatter_memory")
5021    (match_operand:V2DI 1 "s_register_operand")
5022    (match_operand:V2DI 2 "s_register_operand")
5023    (match_operand:V2QI 3 "vpr_register_operand")
5024    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
5025   "TARGET_HAVE_MVE"
5026 {
5027   rtx ind = XEXP (operands[0], 0);
5028   gcc_assert (REG_P (ind));
5029   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
5030                                                               operands[2],
5031                                                               operands[3]));
5032   DONE;
5033 })
5034
5035 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
5036   [(set (mem:BLK (scratch))
5037         (unspec:BLK
5038           [(match_operand:SI 0 "register_operand" "r")
5039            (match_operand:V2DI 1 "s_register_operand" "w")
5040            (match_operand:V2DI 2 "s_register_operand" "w")
5041            (match_operand:V2QI 3 "vpr_register_operand" "Up")]
5042           VSTRDSOQ))]
5043   "TARGET_HAVE_MVE"
5044   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
5045   [(set_attr "length" "8")])
5046
5047 ;;
5048 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
5049 ;;
5050 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
5051   [(match_operand:V2DI 0 "mve_scatter_memory")
5052    (match_operand:V2DI 1 "s_register_operand")
5053    (match_operand:V2DI 2 "s_register_operand")
5054    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
5055   "TARGET_HAVE_MVE"
5056 {
5057   rtx ind = XEXP (operands[0], 0);
5058   gcc_assert (REG_P (ind));
5059   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
5060                                                             operands[2]));
5061   DONE;
5062 })
5063
5064 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
5065   [(set (mem:BLK (scratch))
5066         (unspec:BLK
5067           [(match_operand:SI 0 "register_operand" "r")
5068            (match_operand:V2DI 1 "s_register_operand" "w")
5069            (match_operand:V2DI 2 "s_register_operand" "w")]
5070           VSTRDSOQ))]
5071   "TARGET_HAVE_MVE"
5072   "vstrd.64\t%q2, [%0, %q1]"
5073   [(set_attr "length" "4")])
5074
5075 ;;
5076 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
5077 ;;
5078 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
5079   [(match_operand:V2DI 0 "mve_scatter_memory")
5080    (match_operand:V2DI 1 "s_register_operand")
5081    (match_operand:V2DI 2 "s_register_operand")
5082    (match_operand:V2QI 3 "vpr_register_operand")
5083    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
5084   "TARGET_HAVE_MVE"
5085 {
5086   rtx ind = XEXP (operands[0], 0);
5087   gcc_assert (REG_P (ind));
5088   emit_insn (
5089     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
5090                                                              operands[2],
5091                                                              operands[3]));
5092   DONE;
5093 })
5094
5095 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
5096   [(set (mem:BLK (scratch))
5097         (unspec:BLK
5098           [(match_operand:SI 0 "register_operand" "r")
5099            (match_operand:V2DI 1 "s_register_operand" "w")
5100            (match_operand:V2DI 2 "s_register_operand" "w")
5101            (match_operand:V2QI 3 "vpr_register_operand" "Up")]
5102           VSTRDSSOQ))]
5103   "TARGET_HAVE_MVE"
5104   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
5105   [(set_attr "length" "8")])
5106
5107 ;;
5108 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
5109 ;;
5110 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
5111   [(match_operand:V2DI 0 "mve_scatter_memory")
5112    (match_operand:V2DI 1 "s_register_operand")
5113    (match_operand:V2DI 2 "s_register_operand")
5114    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
5115   "TARGET_HAVE_MVE"
5116 {
5117   rtx ind = XEXP (operands[0], 0);
5118   gcc_assert (REG_P (ind));
5119   emit_insn (
5120     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
5121                                                            operands[2]));
5122   DONE;
5123 })
5124
5125 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
5126   [(set (mem:BLK (scratch))
5127         (unspec:BLK
5128           [(match_operand:SI 0 "register_operand" "r")
5129            (match_operand:V2DI 1 "s_register_operand" "w")
5130            (match_operand:V2DI 2 "s_register_operand" "w")]
5131           VSTRDSSOQ))]
5132   "TARGET_HAVE_MVE"
5133   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
5134   [(set_attr "length" "4")])
5135
5136 ;;
5137 ;; [vstrhq_scatter_offset_f]
5138 ;;
5139 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
5140   [(match_operand:V8HI 0 "mve_scatter_memory")
5141    (match_operand:V8HI 1 "s_register_operand")
5142    (match_operand:V8HF 2 "s_register_operand")
5143    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
5144   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5145 {
5146   rtx ind = XEXP (operands[0], 0);
5147   gcc_assert (REG_P (ind));
5148   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
5149                                                        operands[2]));
5150   DONE;
5151 })
5152
5153 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
5154   [(set (mem:BLK (scratch))
5155         (unspec:BLK
5156           [(match_operand:SI 0 "register_operand" "r")
5157            (match_operand:V8HI 1 "s_register_operand" "w")
5158            (match_operand:V8HF 2 "s_register_operand" "w")]
5159           VSTRHQSO_F))]
5160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5161   "vstrh.16\t%q2, [%0, %q1]"
5162   [(set_attr "length" "4")])
5163
5164 ;;
5165 ;; [vstrhq_scatter_offset_p_f]
5166 ;;
5167 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
5168   [(match_operand:V8HI 0 "mve_scatter_memory")
5169    (match_operand:V8HI 1 "s_register_operand")
5170    (match_operand:V8HF 2 "s_register_operand")
5171    (match_operand:V8BI 3 "vpr_register_operand")
5172    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
5173   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5174 {
5175   rtx ind = XEXP (operands[0], 0);
5176   gcc_assert (REG_P (ind));
5177   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
5178                                                          operands[2],
5179                                                          operands[3]));
5180   DONE;
5181 })
5182
5183 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
5184   [(set (mem:BLK (scratch))
5185         (unspec:BLK
5186           [(match_operand:SI 0 "register_operand" "r")
5187            (match_operand:V8HI 1 "s_register_operand" "w")
5188            (match_operand:V8HF 2 "s_register_operand" "w")
5189            (match_operand:V8BI 3 "vpr_register_operand" "Up")]
5190           VSTRHQSO_F))]
5191   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5192   "vpst\;vstrht.16\t%q2, [%0, %q1]"
5193   [(set_attr "length" "8")])
5194
5195 ;;
5196 ;; [vstrhq_scatter_shifted_offset_f]
5197 ;;
5198 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
5199   [(match_operand:V8HI 0 "memory_operand" "=Us")
5200    (match_operand:V8HI 1 "s_register_operand" "w")
5201    (match_operand:V8HF 2 "s_register_operand" "w")
5202    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
5203   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5204 {
5205   rtx ind = XEXP (operands[0], 0);
5206   gcc_assert (REG_P (ind));
5207   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
5208                                                                operands[2]));
5209   DONE;
5210 })
5211
5212 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
5213   [(set (mem:BLK (scratch))
5214         (unspec:BLK
5215           [(match_operand:SI 0 "register_operand" "r")
5216            (match_operand:V8HI 1 "s_register_operand" "w")
5217            (match_operand:V8HF 2 "s_register_operand" "w")]
5218           VSTRHQSSO_F))]
5219   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5220   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
5221   [(set_attr "length" "4")])
5222
5223 ;;
5224 ;; [vstrhq_scatter_shifted_offset_p_f]
5225 ;;
5226 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
5227   [(match_operand:V8HI 0 "memory_operand" "=Us")
5228    (match_operand:V8HI 1 "s_register_operand" "w")
5229    (match_operand:V8HF 2 "s_register_operand" "w")
5230    (match_operand:V8BI 3 "vpr_register_operand" "Up")
5231    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
5232   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5233 {
5234   rtx ind = XEXP (operands[0], 0);
5235   gcc_assert (REG_P (ind));
5236   emit_insn (
5237     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
5238                                                         operands[2],
5239                                                         operands[3]));
5240   DONE;
5241 })
5242
5243 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
5244   [(set (mem:BLK (scratch))
5245         (unspec:BLK
5246           [(match_operand:SI 0 "register_operand" "r")
5247            (match_operand:V8HI 1 "s_register_operand" "w")
5248            (match_operand:V8HF 2 "s_register_operand" "w")
5249            (match_operand:V8BI 3 "vpr_register_operand" "Up")]
5250           VSTRHQSSO_F))]
5251   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5252   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
5253   [(set_attr "length" "8")])
5254
5255 ;;
5256 ;; [vstrwq_scatter_base_f]
5257 ;;
5258 (define_insn "mve_vstrwq_scatter_base_fv4sf"
5259   [(set (mem:BLK (scratch))
5260         (unspec:BLK
5261                 [(match_operand:V4SI 0 "s_register_operand" "w")
5262                  (match_operand:SI 1 "immediate_operand" "i")
5263                  (match_operand:V4SF 2 "s_register_operand" "w")]
5264          VSTRWQSB_F))
5265   ]
5266   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5267 {
5268    rtx ops[3];
5269    ops[0] = operands[0];
5270    ops[1] = operands[1];
5271    ops[2] = operands[2];
5272    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
5273    return "";
5274 }
5275   [(set_attr "length" "4")])
5276
5277 ;;
5278 ;; [vstrwq_scatter_base_p_f]
5279 ;;
5280 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
5281   [(set (mem:BLK (scratch))
5282         (unspec:BLK
5283                 [(match_operand:V4SI 0 "s_register_operand" "w")
5284                  (match_operand:SI 1 "immediate_operand" "i")
5285                  (match_operand:V4SF 2 "s_register_operand" "w")
5286                  (match_operand:V4BI 3 "vpr_register_operand" "Up")]
5287          VSTRWQSB_F))
5288   ]
5289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5290 {
5291    rtx ops[3];
5292    ops[0] = operands[0];
5293    ops[1] = operands[1];
5294    ops[2] = operands[2];
5295    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
5296    return "";
5297 }
5298   [(set_attr "length" "8")])
5299
5300 ;;
5301 ;; [vstrwq_scatter_offset_f]
5302 ;;
5303 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
5304   [(match_operand:V4SI 0 "mve_scatter_memory")
5305    (match_operand:V4SI 1 "s_register_operand")
5306    (match_operand:V4SF 2 "s_register_operand")
5307    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
5308   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5309 {
5310   rtx ind = XEXP (operands[0], 0);
5311   gcc_assert (REG_P (ind));
5312   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
5313                                                        operands[2]));
5314   DONE;
5315 })
5316
5317 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
5318   [(set (mem:BLK (scratch))
5319         (unspec:BLK
5320           [(match_operand:SI 0 "register_operand" "r")
5321            (match_operand:V4SI 1 "s_register_operand" "w")
5322            (match_operand:V4SF 2 "s_register_operand" "w")]
5323           VSTRWQSO_F))]
5324   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5325   "vstrw.32\t%q2, [%0, %q1]"
5326   [(set_attr "length" "4")])
5327
5328 ;;
5329 ;; [vstrwq_scatter_offset_p_f]
5330 ;;
5331 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
5332   [(match_operand:V4SI 0 "mve_scatter_memory")
5333    (match_operand:V4SI 1 "s_register_operand")
5334    (match_operand:V4SF 2 "s_register_operand")
5335    (match_operand:V4BI 3 "vpr_register_operand")
5336    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
5337   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5338 {
5339   rtx ind = XEXP (operands[0], 0);
5340   gcc_assert (REG_P (ind));
5341   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
5342                                                          operands[2],
5343                                                          operands[3]));
5344   DONE;
5345 })
5346
5347 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
5348   [(set (mem:BLK (scratch))
5349         (unspec:BLK
5350           [(match_operand:SI 0 "register_operand" "r")
5351            (match_operand:V4SI 1 "s_register_operand" "w")
5352            (match_operand:V4SF 2 "s_register_operand" "w")
5353            (match_operand:V4BI 3 "vpr_register_operand" "Up")]
5354           VSTRWQSO_F))]
5355   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5356   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
5357   [(set_attr "length" "8")])
5358
5359 ;;
5360 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
5361 ;;
5362 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
5363   [(match_operand:V4SI 0 "mve_scatter_memory")
5364    (match_operand:V4SI 1 "s_register_operand")
5365    (match_operand:V4SI 2 "s_register_operand")
5366    (match_operand:V4BI 3 "vpr_register_operand")
5367    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
5368   "TARGET_HAVE_MVE"
5369 {
5370   rtx ind = XEXP (operands[0], 0);
5371   gcc_assert (REG_P (ind));
5372   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
5373                                                               operands[2],
5374                                                               operands[3]));
5375   DONE;
5376 })
5377
5378 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
5379   [(set (mem:BLK (scratch))
5380         (unspec:BLK
5381           [(match_operand:SI 0 "register_operand" "r")
5382            (match_operand:V4SI 1 "s_register_operand" "w")
5383            (match_operand:V4SI 2 "s_register_operand" "w")
5384            (match_operand:V4BI 3 "vpr_register_operand" "Up")]
5385           VSTRWSOQ))]
5386   "TARGET_HAVE_MVE"
5387   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
5388   [(set_attr "length" "8")])
5389
5390 ;;
5391 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
5392 ;;
5393 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
5394   [(match_operand:V4SI 0 "mve_scatter_memory")
5395    (match_operand:V4SI 1 "s_register_operand")
5396    (match_operand:V4SI 2 "s_register_operand")
5397    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
5398   "TARGET_HAVE_MVE"
5399 {
5400   rtx ind = XEXP (operands[0], 0);
5401   gcc_assert (REG_P (ind));
5402   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
5403                                                             operands[2]));
5404   DONE;
5405 })
5406
5407 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
5408   [(set (mem:BLK (scratch))
5409         (unspec:BLK
5410           [(match_operand:SI 0 "register_operand" "r")
5411            (match_operand:V4SI 1 "s_register_operand" "w")
5412            (match_operand:V4SI 2 "s_register_operand" "w")]
5413           VSTRWSOQ))]
5414   "TARGET_HAVE_MVE"
5415   "vstrw.32\t%q2, [%0, %q1]"
5416   [(set_attr "length" "4")])
5417
5418 ;;
5419 ;; [vstrwq_scatter_shifted_offset_f]
5420 ;;
5421 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
5422   [(match_operand:V4SI 0 "mve_scatter_memory")
5423    (match_operand:V4SI 1 "s_register_operand")
5424    (match_operand:V4SF 2 "s_register_operand")
5425    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
5426   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5427 {
5428   rtx ind = XEXP (operands[0], 0);
5429   gcc_assert (REG_P (ind));
5430   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
5431                                                                operands[2]));
5432   DONE;
5433 })
5434
5435 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
5436   [(set (mem:BLK (scratch))
5437         (unspec:BLK
5438           [(match_operand:SI 0 "register_operand" "r")
5439            (match_operand:V4SI 1 "s_register_operand" "w")
5440            (match_operand:V4SF 2 "s_register_operand" "w")]
5441          VSTRWQSSO_F))]
5442   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5443   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
5444   [(set_attr "length" "8")])
5445
5446 ;;
5447 ;; [vstrwq_scatter_shifted_offset_p_f]
5448 ;;
5449 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
5450   [(match_operand:V4SI 0 "mve_scatter_memory")
5451    (match_operand:V4SI 1 "s_register_operand")
5452    (match_operand:V4SF 2 "s_register_operand")
5453    (match_operand:V4BI 3 "vpr_register_operand")
5454    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
5455   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5456 {
5457   rtx ind = XEXP (operands[0], 0);
5458   gcc_assert (REG_P (ind));
5459   emit_insn (
5460     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
5461                                                         operands[2],
5462                                                         operands[3]));
5463   DONE;
5464 })
5465
5466 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
5467   [(set (mem:BLK (scratch))
5468         (unspec:BLK
5469           [(match_operand:SI 0 "register_operand" "r")
5470            (match_operand:V4SI 1 "s_register_operand" "w")
5471            (match_operand:V4SF 2 "s_register_operand" "w")
5472            (match_operand:V4BI 3 "vpr_register_operand" "Up")]
5473           VSTRWQSSO_F))]
5474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5475   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
5476   [(set_attr "length" "8")])
5477
5478 ;;
5479 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
5480 ;;
5481 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
5482   [(match_operand:V4SI 0 "mve_scatter_memory")
5483    (match_operand:V4SI 1 "s_register_operand")
5484    (match_operand:V4SI 2 "s_register_operand")
5485    (match_operand:V4BI 3 "vpr_register_operand")
5486    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
5487   "TARGET_HAVE_MVE"
5488 {
5489   rtx ind = XEXP (operands[0], 0);
5490   gcc_assert (REG_P (ind));
5491   emit_insn (
5492     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
5493                                                              operands[2],
5494                                                              operands[3]));
5495   DONE;
5496 })
5497
5498 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
5499   [(set (mem:BLK (scratch))
5500         (unspec:BLK
5501           [(match_operand:SI 0 "register_operand" "r")
5502            (match_operand:V4SI 1 "s_register_operand" "w")
5503            (match_operand:V4SI 2 "s_register_operand" "w")
5504            (match_operand:V4BI 3 "vpr_register_operand" "Up")]
5505           VSTRWSSOQ))]
5506   "TARGET_HAVE_MVE"
5507   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
5508   [(set_attr "length" "8")])
5509
5510 ;;
5511 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
5512 ;;
5513 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
5514   [(match_operand:V4SI 0 "mve_scatter_memory")
5515    (match_operand:V4SI 1 "s_register_operand")
5516    (match_operand:V4SI 2 "s_register_operand")
5517    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
5518   "TARGET_HAVE_MVE"
5519 {
5520   rtx ind = XEXP (operands[0], 0);
5521   gcc_assert (REG_P (ind));
5522   emit_insn (
5523     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
5524                                                            operands[2]));
5525   DONE;
5526 })
5527
5528 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
5529   [(set (mem:BLK (scratch))
5530         (unspec:BLK
5531           [(match_operand:SI 0 "register_operand" "r")
5532            (match_operand:V4SI 1 "s_register_operand" "w")
5533            (match_operand:V4SI 2 "s_register_operand" "w")]
5534           VSTRWSSOQ))]
5535   "TARGET_HAVE_MVE"
5536   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
5537   [(set_attr "length" "4")])
5538
5539 ;;
5540 ;; [vidupq_n_u])
5541 ;;
5542 (define_expand "mve_vidupq_n_u<mode>"
5543  [(match_operand:MVE_2 0 "s_register_operand")
5544   (match_operand:SI 1 "s_register_operand")
5545   (match_operand:SI 2 "mve_imm_selective_upto_8")]
5546  "TARGET_HAVE_MVE"
5547 {
5548   rtx temp = gen_reg_rtx (SImode);
5549   emit_move_insn (temp, operands[1]);
5550   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
5551   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
5552                                           operands[2], inc));
5553   DONE;
5554 })
5555
5556 ;;
5557 ;; [vidupq_u_insn])
5558 ;;
5559 (define_insn "mve_vidupq_u<mode>_insn"
5560  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5561        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
5562                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
5563          VIDUPQ))
5564   (set (match_operand:SI 1 "s_register_operand" "=Te")
5565        (plus:SI (match_dup 2)
5566                 (match_operand:SI 4 "immediate_operand" "i")))]
5567  "TARGET_HAVE_MVE"
5568  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
5569
5570 ;;
5571 ;; [vidupq_m_n_u])
5572 ;;
5573 (define_expand "mve_vidupq_m_n_u<mode>"
5574   [(match_operand:MVE_2 0 "s_register_operand")
5575    (match_operand:MVE_2 1 "s_register_operand")
5576    (match_operand:SI 2 "s_register_operand")
5577    (match_operand:SI 3 "mve_imm_selective_upto_8")
5578    (match_operand:<MVE_VPRED> 4 "vpr_register_operand")]
5579   "TARGET_HAVE_MVE"
5580 {
5581   rtx temp = gen_reg_rtx (SImode);
5582   emit_move_insn (temp, operands[2]);
5583   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
5584   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
5585                                              operands[2], operands[3],
5586                                              operands[4], inc));
5587   DONE;
5588 })
5589
5590 ;;
5591 ;; [vidupq_m_wb_u_insn])
5592 ;;
5593 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
5594  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5595        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5596                       (match_operand:SI 3 "s_register_operand" "2")
5597                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
5598                       (match_operand:<MVE_VPRED> 5 "vpr_register_operand" "Up")]
5599         VIDUPQ_M))
5600   (set (match_operand:SI 2 "s_register_operand" "=Te")
5601        (plus:SI (match_dup 3)
5602                 (match_operand:SI 6 "immediate_operand" "i")))]
5603  "TARGET_HAVE_MVE"
5604  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
5605  [(set_attr "length""8")])
5606
5607 ;;
5608 ;; [vddupq_n_u])
5609 ;;
5610 (define_expand "mve_vddupq_n_u<mode>"
5611  [(match_operand:MVE_2 0 "s_register_operand")
5612   (match_operand:SI 1 "s_register_operand")
5613   (match_operand:SI 2 "mve_imm_selective_upto_8")]
5614  "TARGET_HAVE_MVE"
5615 {
5616   rtx temp = gen_reg_rtx (SImode);
5617   emit_move_insn (temp, operands[1]);
5618   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
5619   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
5620                                           operands[2], inc));
5621   DONE;
5622 })
5623
5624 ;;
5625 ;; [vddupq_u_insn])
5626 ;;
5627 (define_insn "mve_vddupq_u<mode>_insn"
5628  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5629        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
5630                       (match_operand:SI 3 "immediate_operand" "i")]
5631         VDDUPQ))
5632   (set (match_operand:SI 1 "s_register_operand" "=Te")
5633        (minus:SI (match_dup 2)
5634                  (match_operand:SI 4 "immediate_operand" "i")))]
5635  "TARGET_HAVE_MVE"
5636  "vddup.u%#<V_sz_elem>\t%q0, %1, %3")
5637
5638 ;;
5639 ;; [vddupq_m_n_u])
5640 ;;
5641 (define_expand "mve_vddupq_m_n_u<mode>"
5642   [(match_operand:MVE_2 0 "s_register_operand")
5643    (match_operand:MVE_2 1 "s_register_operand")
5644    (match_operand:SI 2 "s_register_operand")
5645    (match_operand:SI 3 "mve_imm_selective_upto_8")
5646    (match_operand:<MVE_VPRED> 4 "vpr_register_operand")]
5647   "TARGET_HAVE_MVE"
5648 {
5649   rtx temp = gen_reg_rtx (SImode);
5650   emit_move_insn (temp, operands[2]);
5651   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
5652   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
5653                                              operands[2], operands[3],
5654                                              operands[4], inc));
5655   DONE;
5656 })
5657
5658 ;;
5659 ;; [vddupq_m_wb_u_insn])
5660 ;;
5661 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
5662  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5663        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5664                       (match_operand:SI 3 "s_register_operand" "2")
5665                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
5666                       (match_operand:<MVE_VPRED> 5 "vpr_register_operand" "Up")]
5667         VDDUPQ_M))
5668   (set (match_operand:SI 2 "s_register_operand" "=Te")
5669        (minus:SI (match_dup 3)
5670                  (match_operand:SI 6 "immediate_operand" "i")))]
5671  "TARGET_HAVE_MVE"
5672  "vpst\;vddupt.u%#<V_sz_elem>\t%q0, %2, %4"
5673  [(set_attr "length""8")])
5674
5675 ;;
5676 ;; [vdwdupq_n_u])
5677 ;;
5678 (define_expand "mve_vdwdupq_n_u<mode>"
5679  [(match_operand:MVE_2 0 "s_register_operand")
5680   (match_operand:SI 1 "s_register_operand")
5681   (match_operand:DI 2 "s_register_operand")
5682   (match_operand:SI 3 "mve_imm_selective_upto_8")]
5683  "TARGET_HAVE_MVE"
5684 {
5685   rtx ignore_wb = gen_reg_rtx (SImode);
5686   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
5687                                               operands[1], operands[2],
5688                                               operands[3]));
5689   DONE;
5690 })
5691
5692 ;;
5693 ;; [vdwdupq_wb_u])
5694 ;;
5695 (define_expand "mve_vdwdupq_wb_u<mode>"
5696  [(match_operand:SI 0 "s_register_operand")
5697   (match_operand:SI 1 "s_register_operand")
5698   (match_operand:DI 2 "s_register_operand")
5699   (match_operand:SI 3 "mve_imm_selective_upto_8")
5700   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5701  "TARGET_HAVE_MVE"
5702 {
5703   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
5704   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
5705                                               operands[1], operands[2],
5706                                               operands[3]));
5707   DONE;
5708 })
5709
5710 ;;
5711 ;; [vdwdupq_wb_u_insn])
5712 ;;
5713 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
5714   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5715         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
5716                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
5717                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
5718          VDWDUPQ))
5719    (set (match_operand:SI 1 "s_register_operand" "=Te")
5720         (unspec:SI [(match_dup 2)
5721                     (subreg:SI (match_dup 3) 4)
5722                     (match_dup 4)]
5723          VDWDUPQ))]
5724   "TARGET_HAVE_MVE"
5725   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
5726 )
5727
5728 ;;
5729 ;; [vdwdupq_m_n_u])
5730 ;;
5731 (define_expand "mve_vdwdupq_m_n_u<mode>"
5732  [(match_operand:MVE_2 0 "s_register_operand")
5733   (match_operand:MVE_2 1 "s_register_operand")
5734   (match_operand:SI 2 "s_register_operand")
5735   (match_operand:DI 3 "s_register_operand")
5736   (match_operand:SI 4 "mve_imm_selective_upto_8")
5737   (match_operand:<MVE_VPRED> 5 "vpr_register_operand")]
5738  "TARGET_HAVE_MVE"
5739 {
5740   rtx ignore_wb = gen_reg_rtx (SImode);
5741   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
5742                                                 operands[1], operands[2],
5743                                                 operands[3], operands[4],
5744                                                 operands[5]));
5745   DONE;
5746 })
5747
5748 ;;
5749 ;; [vdwdupq_m_wb_u])
5750 ;;
5751 (define_expand "mve_vdwdupq_m_wb_u<mode>"
5752  [(match_operand:SI 0 "s_register_operand")
5753   (match_operand:MVE_2 1 "s_register_operand")
5754   (match_operand:SI 2 "s_register_operand")
5755   (match_operand:DI 3 "s_register_operand")
5756   (match_operand:SI 4 "mve_imm_selective_upto_8")
5757   (match_operand:<MVE_VPRED> 5 "vpr_register_operand")]
5758  "TARGET_HAVE_MVE"
5759 {
5760   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
5761   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
5762                                                 operands[1], operands[2],
5763                                                 operands[3], operands[4],
5764                                                 operands[5]));
5765   DONE;
5766 })
5767
5768 ;;
5769 ;; [vdwdupq_m_wb_u_insn])
5770 ;;
5771 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
5772   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5773         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
5774                        (match_operand:SI 3 "s_register_operand" "1")
5775                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
5776                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
5777                        (match_operand:<MVE_VPRED> 6 "vpr_register_operand" "Up")]
5778          VDWDUPQ_M))
5779    (set (match_operand:SI 1 "s_register_operand" "=Te")
5780         (unspec:SI [(match_dup 2)
5781                     (match_dup 3)
5782                     (subreg:SI (match_dup 4) 4)
5783                     (match_dup 5)
5784                     (match_dup 6)]
5785          VDWDUPQ_M))
5786   ]
5787   "TARGET_HAVE_MVE"
5788   "vpst\;vdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
5789   [(set_attr "type" "mve_move")
5790    (set_attr "length""8")])
5791
5792 ;;
5793 ;; [viwdupq_n_u])
5794 ;;
5795 (define_expand "mve_viwdupq_n_u<mode>"
5796  [(match_operand:MVE_2 0 "s_register_operand")
5797   (match_operand:SI 1 "s_register_operand")
5798   (match_operand:DI 2 "s_register_operand")
5799   (match_operand:SI 3 "mve_imm_selective_upto_8")]
5800  "TARGET_HAVE_MVE"
5801 {
5802   rtx ignore_wb = gen_reg_rtx (SImode);
5803   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
5804                                               operands[1], operands[2],
5805                                               operands[3]));
5806   DONE;
5807 })
5808
5809 ;;
5810 ;; [viwdupq_wb_u])
5811 ;;
5812 (define_expand "mve_viwdupq_wb_u<mode>"
5813  [(match_operand:SI 0 "s_register_operand")
5814   (match_operand:SI 1 "s_register_operand")
5815   (match_operand:DI 2 "s_register_operand")
5816   (match_operand:SI 3 "mve_imm_selective_upto_8")
5817   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5818  "TARGET_HAVE_MVE"
5819 {
5820   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
5821   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
5822                                               operands[1], operands[2],
5823                                               operands[3]));
5824   DONE;
5825 })
5826
5827 ;;
5828 ;; [viwdupq_wb_u_insn])
5829 ;;
5830 (define_insn "mve_viwdupq_wb_u<mode>_insn"
5831   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5832         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
5833                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
5834                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
5835          VIWDUPQ))
5836    (set (match_operand:SI 1 "s_register_operand" "=Te")
5837         (unspec:SI [(match_dup 2)
5838                     (subreg:SI (match_dup 3) 4)
5839                     (match_dup 4)]
5840          VIWDUPQ))]
5841   "TARGET_HAVE_MVE"
5842   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
5843 )
5844
5845 ;;
5846 ;; [viwdupq_m_n_u])
5847 ;;
5848 (define_expand "mve_viwdupq_m_n_u<mode>"
5849  [(match_operand:MVE_2 0 "s_register_operand")
5850   (match_operand:MVE_2 1 "s_register_operand")
5851   (match_operand:SI 2 "s_register_operand")
5852   (match_operand:DI 3 "s_register_operand")
5853   (match_operand:SI 4 "mve_imm_selective_upto_8")
5854   (match_operand:<MVE_VPRED> 5 "vpr_register_operand")]
5855  "TARGET_HAVE_MVE"
5856 {
5857   rtx ignore_wb = gen_reg_rtx (SImode);
5858   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
5859                                                 operands[1], operands[2],
5860                                                 operands[3], operands[4],
5861                                                 operands[5]));
5862   DONE;
5863 })
5864
5865 ;;
5866 ;; [viwdupq_m_wb_u])
5867 ;;
5868 (define_expand "mve_viwdupq_m_wb_u<mode>"
5869  [(match_operand:SI 0 "s_register_operand")
5870   (match_operand:MVE_2 1 "s_register_operand")
5871   (match_operand:SI 2 "s_register_operand")
5872   (match_operand:DI 3 "s_register_operand")
5873   (match_operand:SI 4 "mve_imm_selective_upto_8")
5874   (match_operand:<MVE_VPRED> 5 "vpr_register_operand")]
5875  "TARGET_HAVE_MVE"
5876 {
5877   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
5878   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
5879                                                 operands[1], operands[2],
5880                                                 operands[3], operands[4],
5881                                                 operands[5]));
5882   DONE;
5883 })
5884
5885 ;;
5886 ;; [viwdupq_m_wb_u_insn])
5887 ;;
5888 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
5889   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
5890         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
5891                        (match_operand:SI 3 "s_register_operand" "1")
5892                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
5893                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
5894                        (match_operand:<MVE_VPRED> 6 "vpr_register_operand" "Up")]
5895          VIWDUPQ_M))
5896    (set (match_operand:SI 1 "s_register_operand" "=Te")
5897         (unspec:SI [(match_dup 2)
5898                     (match_dup 3)
5899                     (subreg:SI (match_dup 4) 4)
5900                     (match_dup 5)
5901                     (match_dup 6)]
5902          VIWDUPQ_M))
5903   ]
5904   "TARGET_HAVE_MVE"
5905   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
5906   [(set_attr "type" "mve_move")
5907    (set_attr "length""8")])
5908
5909 ;;
5910 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
5911 ;;
5912 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
5913   [(set (mem:BLK (scratch))
5914         (unspec:BLK
5915                 [(match_operand:V4SI 1 "s_register_operand" "0")
5916                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
5917                  (match_operand:V4SI 3 "s_register_operand" "w")]
5918          VSTRWSBWBQ))
5919    (set (match_operand:V4SI 0 "s_register_operand" "=w")
5920         (unspec:V4SI [(match_dup 1) (match_dup 2)]
5921          VSTRWSBWBQ))
5922   ]
5923   "TARGET_HAVE_MVE"
5924 {
5925    rtx ops[3];
5926    ops[0] = operands[1];
5927    ops[1] = operands[2];
5928    ops[2] = operands[3];
5929    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
5930    return "";
5931 }
5932   [(set_attr "length" "4")])
5933
5934 ;;
5935 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
5936 ;;
5937 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
5938  [(set (mem:BLK (scratch))
5939        (unspec:BLK
5940                 [(match_operand:V4SI 1 "s_register_operand" "0")
5941                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
5942                  (match_operand:V4SI 3 "s_register_operand" "w")
5943                  (match_operand:V4BI 4 "vpr_register_operand")]
5944         VSTRWSBWBQ))
5945    (set (match_operand:V4SI 0 "s_register_operand" "=w")
5946         (unspec:V4SI [(match_dup 1) (match_dup 2)]
5947          VSTRWSBWBQ))
5948   ]
5949   "TARGET_HAVE_MVE"
5950 {
5951    rtx ops[3];
5952    ops[0] = operands[1];
5953    ops[1] = operands[2];
5954    ops[2] = operands[3];
5955    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
5956    return "";
5957 }
5958   [(set_attr "length" "8")])
5959
5960 ;;
5961 ;; [vstrwq_scatter_base_wb_f]
5962 ;;
5963 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
5964  [(set (mem:BLK (scratch))
5965        (unspec:BLK
5966                 [(match_operand:V4SI 1 "s_register_operand" "0")
5967                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
5968                  (match_operand:V4SF 3 "s_register_operand" "w")]
5969          VSTRWQSBWB_F))
5970    (set (match_operand:V4SI 0 "s_register_operand" "=w")
5971         (unspec:V4SI [(match_dup 1) (match_dup 2)]
5972          VSTRWQSBWB_F))
5973   ]
5974   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5975 {
5976    rtx ops[3];
5977    ops[0] = operands[1];
5978    ops[1] = operands[2];
5979    ops[2] = operands[3];
5980    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
5981    return "";
5982 }
5983   [(set_attr "length" "4")])
5984
5985 ;;
5986 ;; [vstrwq_scatter_base_wb_p_f]
5987 ;;
5988 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
5989  [(set (mem:BLK (scratch))
5990        (unspec:BLK
5991                 [(match_operand:V4SI 1 "s_register_operand" "0")
5992                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
5993                  (match_operand:V4SF 3 "s_register_operand" "w")
5994                  (match_operand:V4BI 4 "vpr_register_operand")]
5995         VSTRWQSBWB_F))
5996    (set (match_operand:V4SI 0 "s_register_operand" "=w")
5997         (unspec:V4SI [(match_dup 1) (match_dup 2)]
5998          VSTRWQSBWB_F))
5999   ]
6000   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6001 {
6002    rtx ops[3];
6003    ops[0] = operands[1];
6004    ops[1] = operands[2];
6005    ops[2] = operands[3];
6006    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
6007    return "";
6008 }
6009   [(set_attr "length" "8")])
6010
6011 ;;
6012 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
6013 ;;
6014 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
6015   [(set (mem:BLK (scratch))
6016         (unspec:BLK
6017                 [(match_operand:V2DI 1 "s_register_operand" "0")
6018                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
6019                  (match_operand:V2DI 3 "s_register_operand" "w")]
6020          VSTRDSBWBQ))
6021    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
6022         (unspec:V2DI [(match_dup 1) (match_dup 2)]
6023          VSTRDSBWBQ))
6024   ]
6025   "TARGET_HAVE_MVE"
6026 {
6027    rtx ops[3];
6028    ops[0] = operands[1];
6029    ops[1] = operands[2];
6030    ops[2] = operands[3];
6031    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
6032    return "";
6033 }
6034   [(set_attr "length" "4")])
6035
6036 ;;
6037 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
6038 ;;
6039 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
6040   [(set (mem:BLK (scratch))
6041         (unspec:BLK
6042                 [(match_operand:V2DI 1 "s_register_operand" "0")
6043                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
6044                  (match_operand:V2DI 3 "s_register_operand" "w")
6045                  (match_operand:V2QI 4 "vpr_register_operand")]
6046          VSTRDSBWBQ))
6047    (set (match_operand:V2DI 0 "s_register_operand" "=w")
6048         (unspec:V2DI [(match_dup 1) (match_dup 2)]
6049          VSTRDSBWBQ))
6050   ]
6051   "TARGET_HAVE_MVE"
6052 {
6053    rtx ops[3];
6054    ops[0] = operands[1];
6055    ops[1] = operands[2];
6056    ops[2] = operands[3];
6057    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
6058    return "";
6059 }
6060   [(set_attr "length" "8")])
6061
6062 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
6063   [(match_operand:V4SI 0 "s_register_operand")
6064    (match_operand:V4SI 1 "s_register_operand")
6065    (match_operand:SI 2 "mve_vldrd_immediate")
6066    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
6067   "TARGET_HAVE_MVE"
6068 {
6069   rtx ignore_result = gen_reg_rtx (V4SImode);
6070   emit_insn (
6071   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
6072                                                  operands[1], operands[2]));
6073   DONE;
6074 })
6075
6076 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
6077   [(match_operand:V4SI 0 "s_register_operand")
6078    (match_operand:V4SI 1 "s_register_operand")
6079    (match_operand:SI 2 "mve_vldrd_immediate")
6080    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
6081   "TARGET_HAVE_MVE"
6082 {
6083   rtx ignore_wb = gen_reg_rtx (V4SImode);
6084   emit_insn (
6085   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
6086                                                  operands[1], operands[2]));
6087   DONE;
6088 })
6089
6090 ;;
6091 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
6092 ;;
6093 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
6094   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
6095         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
6096                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6097                       (mem:BLK (scratch))]
6098          VLDRWGBWBQ))
6099    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
6100         (unspec:V4SI [(match_dup 2) (match_dup 3)]
6101          VLDRWGBWBQ))
6102   ]
6103   "TARGET_HAVE_MVE"
6104 {
6105    rtx ops[3];
6106    ops[0] = operands[0];
6107    ops[1] = operands[2];
6108    ops[2] = operands[3];
6109    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
6110    return "";
6111 }
6112   [(set_attr "length" "4")])
6113
6114 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
6115   [(match_operand:V4SI 0 "s_register_operand")
6116    (match_operand:V4SI 1 "s_register_operand")
6117    (match_operand:SI 2 "mve_vldrd_immediate")
6118    (match_operand:V4BI 3 "vpr_register_operand")
6119    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
6120   "TARGET_HAVE_MVE"
6121 {
6122   rtx ignore_result = gen_reg_rtx (V4SImode);
6123   emit_insn (
6124   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
6125                                                    operands[1], operands[2],
6126                                                    operands[3]));
6127   DONE;
6128 })
6129 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
6130   [(match_operand:V4SI 0 "s_register_operand")
6131    (match_operand:V4SI 1 "s_register_operand")
6132    (match_operand:SI 2 "mve_vldrd_immediate")
6133    (match_operand:V4BI 3 "vpr_register_operand")
6134    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
6135   "TARGET_HAVE_MVE"
6136 {
6137   rtx ignore_wb = gen_reg_rtx (V4SImode);
6138   emit_insn (
6139   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
6140                                                    operands[1], operands[2],
6141                                                    operands[3]));
6142   DONE;
6143 })
6144
6145 ;;
6146 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
6147 ;;
6148 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
6149   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
6150         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
6151                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6152                       (match_operand:V4BI 4 "vpr_register_operand" "Up")
6153                       (mem:BLK (scratch))]
6154          VLDRWGBWBQ))
6155    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
6156         (unspec:V4SI [(match_dup 2) (match_dup 3)]
6157          VLDRWGBWBQ))
6158   ]
6159   "TARGET_HAVE_MVE"
6160 {
6161    rtx ops[3];
6162    ops[0] = operands[0];
6163    ops[1] = operands[2];
6164    ops[2] = operands[3];
6165    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
6166    return "";
6167 }
6168   [(set_attr "length" "8")])
6169
6170 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
6171   [(match_operand:V4SI 0 "s_register_operand")
6172    (match_operand:V4SI 1 "s_register_operand")
6173    (match_operand:SI 2 "mve_vldrd_immediate")
6174    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
6175   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6176 {
6177   rtx ignore_result = gen_reg_rtx (V4SFmode);
6178   emit_insn (
6179   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
6180                                             operands[1], operands[2]));
6181   DONE;
6182 })
6183
6184 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
6185   [(match_operand:V4SF 0 "s_register_operand")
6186    (match_operand:V4SI 1 "s_register_operand")
6187    (match_operand:SI 2 "mve_vldrd_immediate")
6188    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
6189   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6190 {
6191   rtx ignore_wb = gen_reg_rtx (V4SImode);
6192   emit_insn (
6193   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
6194                                             operands[1], operands[2]));
6195   DONE;
6196 })
6197
6198 ;;
6199 ;; [vldrwq_gather_base_wb_f]
6200 ;;
6201 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
6202   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
6203         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
6204                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6205                       (mem:BLK (scratch))]
6206          VLDRWQGBWB_F))
6207    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
6208         (unspec:V4SI [(match_dup 2) (match_dup 3)]
6209          VLDRWQGBWB_F))
6210   ]
6211   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6212 {
6213    rtx ops[3];
6214    ops[0] = operands[0];
6215    ops[1] = operands[2];
6216    ops[2] = operands[3];
6217    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
6218    return "";
6219 }
6220   [(set_attr "length" "4")])
6221
6222 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
6223   [(match_operand:V4SI 0 "s_register_operand")
6224    (match_operand:V4SI 1 "s_register_operand")
6225    (match_operand:SI 2 "mve_vldrd_immediate")
6226    (match_operand:V4BI 3 "vpr_register_operand")
6227    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
6228   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6229 {
6230   rtx ignore_result = gen_reg_rtx (V4SFmode);
6231   emit_insn (
6232   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
6233                                               operands[1], operands[2],
6234                                               operands[3]));
6235   DONE;
6236 })
6237
6238 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
6239   [(match_operand:V4SF 0 "s_register_operand")
6240    (match_operand:V4SI 1 "s_register_operand")
6241    (match_operand:SI 2 "mve_vldrd_immediate")
6242    (match_operand:V4BI 3 "vpr_register_operand")
6243    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
6244   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6245 {
6246   rtx ignore_wb = gen_reg_rtx (V4SImode);
6247   emit_insn (
6248   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
6249                                               operands[1], operands[2],
6250                                               operands[3]));
6251   DONE;
6252 })
6253
6254 ;;
6255 ;; [vldrwq_gather_base_wb_z_f]
6256 ;;
6257 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
6258   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
6259         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
6260                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6261                       (match_operand:V4BI 4 "vpr_register_operand" "Up")
6262                       (mem:BLK (scratch))]
6263          VLDRWQGBWB_F))
6264    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
6265         (unspec:V4SI [(match_dup 2) (match_dup 3)]
6266          VLDRWQGBWB_F))
6267   ]
6268   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6269 {
6270    rtx ops[3];
6271    ops[0] = operands[0];
6272    ops[1] = operands[2];
6273    ops[2] = operands[3];
6274    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
6275    return "";
6276 }
6277   [(set_attr "length" "8")])
6278
6279 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
6280   [(match_operand:V2DI 0 "s_register_operand")
6281    (match_operand:V2DI 1 "s_register_operand")
6282    (match_operand:SI 2 "mve_vldrd_immediate")
6283    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
6284   "TARGET_HAVE_MVE"
6285 {
6286   rtx ignore_result = gen_reg_rtx (V2DImode);
6287   emit_insn (
6288   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
6289                                                  operands[1], operands[2]));
6290   DONE;
6291 })
6292
6293 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
6294   [(match_operand:V2DI 0 "s_register_operand")
6295    (match_operand:V2DI 1 "s_register_operand")
6296    (match_operand:SI 2 "mve_vldrd_immediate")
6297    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
6298   "TARGET_HAVE_MVE"
6299 {
6300   rtx ignore_wb = gen_reg_rtx (V2DImode);
6301   emit_insn (
6302   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
6303                                                  operands[1], operands[2]));
6304   DONE;
6305 })
6306
6307
6308 ;;
6309 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
6310 ;;
6311 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
6312   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
6313         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
6314                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6315                       (mem:BLK (scratch))]
6316          VLDRDGBWBQ))
6317    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
6318         (unspec:V2DI [(match_dup 2) (match_dup 3)]
6319          VLDRDGBWBQ))
6320   ]
6321   "TARGET_HAVE_MVE"
6322 {
6323    rtx ops[3];
6324    ops[0] = operands[0];
6325    ops[1] = operands[2];
6326    ops[2] = operands[3];
6327    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
6328    return "";
6329 }
6330   [(set_attr "length" "4")])
6331
6332 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
6333   [(match_operand:V2DI 0 "s_register_operand")
6334    (match_operand:V2DI 1 "s_register_operand")
6335    (match_operand:SI 2 "mve_vldrd_immediate")
6336    (match_operand:V2QI 3 "vpr_register_operand")
6337    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
6338   "TARGET_HAVE_MVE"
6339 {
6340   rtx ignore_result = gen_reg_rtx (V2DImode);
6341   emit_insn (
6342   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
6343                                                    operands[1], operands[2],
6344                                                    operands[3]));
6345   DONE;
6346 })
6347
6348 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
6349   [(match_operand:V2DI 0 "s_register_operand")
6350    (match_operand:V2DI 1 "s_register_operand")
6351    (match_operand:SI 2 "mve_vldrd_immediate")
6352    (match_operand:V2QI 3 "vpr_register_operand")
6353    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
6354   "TARGET_HAVE_MVE"
6355 {
6356   rtx ignore_wb = gen_reg_rtx (V2DImode);
6357   emit_insn (
6358   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
6359                                                    operands[1], operands[2],
6360                                                    operands[3]));
6361   DONE;
6362 })
6363
6364 (define_insn "get_fpscr_nzcvqc"
6365  [(set (match_operand:SI 0 "register_operand" "=r")
6366    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
6367  "TARGET_HAVE_MVE"
6368  "vmrs\\t%0, FPSCR_nzcvqc"
6369   [(set_attr "type" "mve_move")])
6370
6371 (define_insn "set_fpscr_nzcvqc"
6372  [(set (reg:SI VFPCC_REGNUM)
6373    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
6374     VUNSPEC_SET_FPSCR_NZCVQC))]
6375  "TARGET_HAVE_MVE"
6376  "vmsr\\tFPSCR_nzcvqc, %0"
6377   [(set_attr "type" "mve_move")])
6378
6379 ;;
6380 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
6381 ;;
6382 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
6383   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
6384         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
6385                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
6386                       (match_operand:V2QI 4 "vpr_register_operand" "Up")
6387                       (mem:BLK (scratch))]
6388          VLDRDGBWBQ))
6389    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
6390         (unspec:V2DI [(match_dup 2) (match_dup 3)]
6391          VLDRDGBWBQ))
6392   ]
6393   "TARGET_HAVE_MVE"
6394 {
6395    rtx ops[3];
6396    ops[0] = operands[0];
6397    ops[1] = operands[2];
6398    ops[2] = operands[3];
6399    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
6400    return "";
6401 }
6402   [(set_attr "length" "8")])
6403 ;;
6404 ;; [vadciq_m_s, vadciq_m_u])
6405 ;;
6406 (define_insn "mve_vadciq_m_<supf>v4si"
6407   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6408         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
6409                       (match_operand:V4SI 2 "s_register_operand" "w")
6410                       (match_operand:V4SI 3 "s_register_operand" "w")
6411                       (match_operand:V4BI 4 "vpr_register_operand" "Up")]
6412          VADCIQ_M))
6413    (set (reg:SI VFPCC_REGNUM)
6414         (unspec:SI [(const_int 0)]
6415          VADCIQ_M))
6416   ]
6417   "TARGET_HAVE_MVE"
6418   "vpst\;vadcit.i32\t%q0, %q2, %q3"
6419   [(set_attr "type" "mve_move")
6420    (set_attr "length" "8")])
6421
6422 ;;
6423 ;; [vadciq_u, vadciq_s])
6424 ;;
6425 (define_insn "mve_vadciq_<supf>v4si"
6426   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6427         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6428                       (match_operand:V4SI 2 "s_register_operand" "w")]
6429          VADCIQ))
6430    (set (reg:SI VFPCC_REGNUM)
6431         (unspec:SI [(const_int 0)]
6432          VADCIQ))
6433   ]
6434   "TARGET_HAVE_MVE"
6435   "vadci.i32\t%q0, %q1, %q2"
6436   [(set_attr "type" "mve_move")
6437    (set_attr "length" "4")])
6438
6439 ;;
6440 ;; [vadcq_m_s, vadcq_m_u])
6441 ;;
6442 (define_insn "mve_vadcq_m_<supf>v4si"
6443   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6444         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
6445                       (match_operand:V4SI 2 "s_register_operand" "w")
6446                       (match_operand:V4SI 3 "s_register_operand" "w")
6447                       (match_operand:V4BI 4 "vpr_register_operand" "Up")]
6448          VADCQ_M))
6449    (set (reg:SI VFPCC_REGNUM)
6450         (unspec:SI [(reg:SI VFPCC_REGNUM)]
6451          VADCQ_M))
6452   ]
6453   "TARGET_HAVE_MVE"
6454   "vpst\;vadct.i32\t%q0, %q2, %q3"
6455   [(set_attr "type" "mve_move")
6456    (set_attr "length" "8")])
6457
6458 ;;
6459 ;; [vadcq_u, vadcq_s])
6460 ;;
6461 (define_insn "mve_vadcq_<supf>v4si"
6462   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6463         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6464                        (match_operand:V4SI 2 "s_register_operand" "w")]
6465          VADCQ))
6466    (set (reg:SI VFPCC_REGNUM)
6467         (unspec:SI [(reg:SI VFPCC_REGNUM)]
6468          VADCQ))
6469   ]
6470   "TARGET_HAVE_MVE"
6471   "vadc.i32\t%q0, %q1, %q2"
6472   [(set_attr "type" "mve_move")
6473    (set_attr "length" "4")
6474    (set_attr "conds" "set")])
6475
6476 ;;
6477 ;; [vsbciq_m_u, vsbciq_m_s])
6478 ;;
6479 (define_insn "mve_vsbciq_m_<supf>v4si"
6480   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6481         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6482                       (match_operand:V4SI 2 "s_register_operand" "w")
6483                       (match_operand:V4SI 3 "s_register_operand" "w")
6484                       (match_operand:V4BI 4 "vpr_register_operand" "Up")]
6485          VSBCIQ_M))
6486    (set (reg:SI VFPCC_REGNUM)
6487         (unspec:SI [(const_int 0)]
6488          VSBCIQ_M))
6489   ]
6490   "TARGET_HAVE_MVE"
6491   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
6492   [(set_attr "type" "mve_move")
6493    (set_attr "length" "8")])
6494
6495 ;;
6496 ;; [vsbciq_s, vsbciq_u])
6497 ;;
6498 (define_insn "mve_vsbciq_<supf>v4si"
6499   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6500         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6501                       (match_operand:V4SI 2 "s_register_operand" "w")]
6502          VSBCIQ))
6503    (set (reg:SI VFPCC_REGNUM)
6504         (unspec:SI [(const_int 0)]
6505          VSBCIQ))
6506   ]
6507   "TARGET_HAVE_MVE"
6508   "vsbci.i32\t%q0, %q1, %q2"
6509   [(set_attr "type" "mve_move")
6510    (set_attr "length" "4")])
6511
6512 ;;
6513 ;; [vsbcq_m_u, vsbcq_m_s])
6514 ;;
6515 (define_insn "mve_vsbcq_m_<supf>v4si"
6516   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6517         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6518                       (match_operand:V4SI 2 "s_register_operand" "w")
6519                       (match_operand:V4SI 3 "s_register_operand" "w")
6520                       (match_operand:V4BI 4 "vpr_register_operand" "Up")]
6521          VSBCQ_M))
6522    (set (reg:SI VFPCC_REGNUM)
6523         (unspec:SI [(reg:SI VFPCC_REGNUM)]
6524          VSBCQ_M))
6525   ]
6526   "TARGET_HAVE_MVE"
6527   "vpst\;vsbct.i32\t%q0, %q2, %q3"
6528   [(set_attr "type" "mve_move")
6529    (set_attr "length" "8")])
6530
6531 ;;
6532 ;; [vsbcq_s, vsbcq_u])
6533 ;;
6534 (define_insn "mve_vsbcq_<supf>v4si"
6535   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
6536         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
6537                       (match_operand:V4SI 2 "s_register_operand" "w")]
6538          VSBCQ))
6539    (set (reg:SI VFPCC_REGNUM)
6540         (unspec:SI [(reg:SI VFPCC_REGNUM)]
6541          VSBCQ))
6542   ]
6543   "TARGET_HAVE_MVE"
6544   "vsbc.i32\t%q0, %q1, %q2"
6545   [(set_attr "type" "mve_move")
6546    (set_attr "length" "4")])
6547
6548 ;;
6549 ;; [vst2q])
6550 ;;
6551 (define_insn "mve_vst2q<mode>"
6552   [(set (match_operand:OI 0 "mve_struct_operand" "=Ug")
6553         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
6554                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
6555          VST2Q))
6556   ]
6557   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6558    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6559 {
6560    rtx ops[4];
6561    int regno = REGNO (operands[1]);
6562    ops[0] = gen_rtx_REG (TImode, regno);
6563    ops[1] = gen_rtx_REG (TImode, regno + 4);
6564    rtx reg  = operands[0];
6565    while (reg && !REG_P (reg))
6566     reg = XEXP (reg, 0);
6567    gcc_assert (REG_P (reg));
6568    ops[2] = reg;
6569    ops[3] = operands[0];
6570    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
6571                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
6572    return "";
6573 }
6574   [(set_attr "length" "8")])
6575
6576 ;;
6577 ;; [vld2q])
6578 ;;
6579 (define_insn "mve_vld2q<mode>"
6580   [(set (match_operand:OI 0 "s_register_operand" "=w")
6581         (unspec:OI [(match_operand:OI 1 "mve_struct_operand" "Ug")
6582                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
6583          VLD2Q))
6584   ]
6585   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6586    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6587 {
6588    rtx ops[4];
6589    int regno = REGNO (operands[0]);
6590    ops[0] = gen_rtx_REG (TImode, regno);
6591    ops[1] = gen_rtx_REG (TImode, regno + 4);
6592    rtx reg  = operands[1];
6593    while (reg && !REG_P (reg))
6594     reg = XEXP (reg, 0);
6595    gcc_assert (REG_P (reg));
6596    ops[2] = reg;
6597    ops[3] = operands[1];
6598    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
6599                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
6600    return "";
6601 }
6602   [(set_attr "length" "8")])
6603
6604 ;;
6605 ;; [vld4q])
6606 ;;
6607 (define_insn "mve_vld4q<mode>"
6608   [(set (match_operand:XI 0 "s_register_operand" "=w")
6609         (unspec:XI [(match_operand:XI 1 "mve_struct_operand" "Ug")
6610                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
6611          VLD4Q))
6612   ]
6613   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6614    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6615 {
6616    rtx ops[6];
6617    int regno = REGNO (operands[0]);
6618    ops[0] = gen_rtx_REG (TImode, regno);
6619    ops[1] = gen_rtx_REG (TImode, regno+4);
6620    ops[2] = gen_rtx_REG (TImode, regno+8);
6621    ops[3] = gen_rtx_REG (TImode, regno + 12);
6622    rtx reg  = operands[1];
6623    while (reg && !REG_P (reg))
6624     reg = XEXP (reg, 0);
6625    gcc_assert (REG_P (reg));
6626    ops[4] = reg;
6627    ops[5] = operands[1];
6628    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
6629                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
6630                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
6631                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
6632    return "";
6633 }
6634   [(set_attr "length" "16")])
6635 ;;
6636 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
6637 ;;
6638 (define_insn "mve_vec_extract<mode><V_elem_l>"
6639  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
6640    (vec_select:<V_elem>
6641     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
6642     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
6643   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6644    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6645 {
6646   if (BYTES_BIG_ENDIAN)
6647     {
6648       int elt = INTVAL (operands[2]);
6649       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
6650       operands[2] = GEN_INT (elt);
6651     }
6652   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
6653 }
6654  [(set_attr "type" "mve_move")])
6655
6656 (define_insn "mve_vec_extractv2didi"
6657  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
6658    (vec_select:DI
6659     (match_operand:V2DI 1 "s_register_operand" "w")
6660     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
6661   "TARGET_HAVE_MVE"
6662 {
6663   int elt = INTVAL (operands[2]);
6664   if (BYTES_BIG_ENDIAN)
6665     elt = 1 - elt;
6666
6667   if (elt == 0)
6668    return "vmov\t%Q0, %R0, %e1";
6669   else
6670    return "vmov\t%Q0, %R0, %f1";
6671 }
6672  [(set_attr "type" "mve_move")])
6673
6674 (define_insn "*mve_vec_extract_sext_internal<mode>"
6675  [(set (match_operand:SI 0 "s_register_operand" "=r")
6676    (sign_extend:SI
6677     (vec_select:<V_elem>
6678      (match_operand:MVE_2 1 "s_register_operand" "w")
6679      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
6680   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6681    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6682 {
6683   if (BYTES_BIG_ENDIAN)
6684     {
6685       int elt = INTVAL (operands[2]);
6686       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
6687       operands[2] = GEN_INT (elt);
6688     }
6689   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
6690 }
6691  [(set_attr "type" "mve_move")])
6692
6693 (define_insn "*mve_vec_extract_zext_internal<mode>"
6694  [(set (match_operand:SI 0 "s_register_operand" "=r")
6695    (zero_extend:SI
6696     (vec_select:<V_elem>
6697      (match_operand:MVE_2 1 "s_register_operand" "w")
6698      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
6699   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6700    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6701 {
6702   if (BYTES_BIG_ENDIAN)
6703     {
6704       int elt = INTVAL (operands[2]);
6705       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
6706       operands[2] = GEN_INT (elt);
6707     }
6708   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
6709 }
6710  [(set_attr "type" "mve_move")])
6711
6712 ;;
6713 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
6714 ;;
6715 (define_insn "mve_vec_set<mode>_internal"
6716  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
6717        (vec_merge:VQ2
6718         (vec_duplicate:VQ2
6719           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
6720         (match_operand:VQ2 3 "s_register_operand" "0")
6721         (match_operand:SI 2 "immediate_operand" "i")))]
6722   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
6723    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
6724 {
6725   int elt = ffs ((int) INTVAL (operands[2])) - 1;
6726   if (BYTES_BIG_ENDIAN)
6727     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
6728   operands[2] = GEN_INT (elt);
6729
6730   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
6731 }
6732  [(set_attr "type" "mve_move")])
6733
6734 (define_insn "mve_vec_setv2di_internal"
6735  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
6736        (vec_merge:V2DI
6737         (vec_duplicate:V2DI
6738           (match_operand:DI 1 "nonimmediate_operand" "r"))
6739         (match_operand:V2DI 3 "s_register_operand" "0")
6740         (match_operand:SI 2 "immediate_operand" "i")))]
6741  "TARGET_HAVE_MVE"
6742 {
6743   int elt = ffs ((int) INTVAL (operands[2])) - 1;
6744   if (BYTES_BIG_ENDIAN)
6745     elt = 1 - elt;
6746
6747   if (elt == 0)
6748    return "vmov\t%e0, %Q1, %R1";
6749   else
6750    return "vmov\t%f0, %J1, %K1";
6751 }
6752  [(set_attr "type" "mve_move")])
6753
6754 ;;
6755 ;; [uqrshll_di]
6756 ;;
6757 (define_insn "mve_uqrshll_sat<supf>_di"
6758   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6759         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
6760                     (match_operand:SI 2 "register_operand" "r")]
6761          UQRSHLLQ))]
6762   "TARGET_HAVE_MVE"
6763   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
6764   [(set_attr "predicable" "yes")])
6765
6766 ;;
6767 ;; [sqrshrl_di]
6768 ;;
6769 (define_insn "mve_sqrshrl_sat<supf>_di"
6770   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6771         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
6772                     (match_operand:SI 2 "register_operand" "r")]
6773          SQRSHRLQ))]
6774   "TARGET_HAVE_MVE"
6775   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
6776   [(set_attr "predicable" "yes")])
6777
6778 ;;
6779 ;; [uqrshl_si]
6780 ;;
6781 (define_insn "mve_uqrshl_si"
6782   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6783         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
6784                     (match_operand:SI 2 "register_operand" "r")]
6785          UQRSHL))]
6786   "TARGET_HAVE_MVE"
6787   "uqrshl%?\\t%1, %2"
6788   [(set_attr "predicable" "yes")])
6789
6790 ;;
6791 ;; [sqrshr_si]
6792 ;;
6793 (define_insn "mve_sqrshr_si"
6794   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6795         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
6796                     (match_operand:SI 2 "register_operand" "r")]
6797          SQRSHR))]
6798   "TARGET_HAVE_MVE"
6799   "sqrshr%?\\t%1, %2"
6800   [(set_attr "predicable" "yes")])
6801
6802 ;;
6803 ;; [uqshll_di]
6804 ;;
6805 (define_insn "mve_uqshll_di"
6806   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6807         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
6808                       (match_operand:SI 2 "immediate_operand" "Pg")))]
6809   "TARGET_HAVE_MVE"
6810   "uqshll%?\\t%Q1, %R1, %2"
6811   [(set_attr "predicable" "yes")])
6812
6813 ;;
6814 ;; [urshrl_di]
6815 ;;
6816 (define_insn "mve_urshrl_di"
6817   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6818         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
6819                     (match_operand:SI 2 "immediate_operand" "Pg")]
6820          URSHRL))]
6821   "TARGET_HAVE_MVE"
6822   "urshrl%?\\t%Q1, %R1, %2"
6823   [(set_attr "predicable" "yes")])
6824
6825 ;;
6826 ;; [uqshl_si]
6827 ;;
6828 (define_insn "mve_uqshl_si"
6829   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6830         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
6831                       (match_operand:SI 2 "immediate_operand" "Pg")))]
6832   "TARGET_HAVE_MVE"
6833   "uqshl%?\\t%1, %2"
6834   [(set_attr "predicable" "yes")])
6835
6836 ;;
6837 ;; [urshr_si]
6838 ;;
6839 (define_insn "mve_urshr_si"
6840   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6841         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
6842                     (match_operand:SI 2 "immediate_operand" "Pg")]
6843          URSHR))]
6844   "TARGET_HAVE_MVE"
6845   "urshr%?\\t%1, %2"
6846   [(set_attr "predicable" "yes")])
6847
6848 ;;
6849 ;; [sqshl_si]
6850 ;;
6851 (define_insn "mve_sqshl_si"
6852   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6853         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
6854                       (match_operand:SI 2 "immediate_operand" "Pg")))]
6855   "TARGET_HAVE_MVE"
6856   "sqshl%?\\t%1, %2"
6857   [(set_attr "predicable" "yes")])
6858
6859 ;;
6860 ;; [srshr_si]
6861 ;;
6862 (define_insn "mve_srshr_si"
6863   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
6864         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
6865                     (match_operand:SI 2 "immediate_operand" "Pg")]
6866          SRSHR))]
6867   "TARGET_HAVE_MVE"
6868   "srshr%?\\t%1, %2"
6869   [(set_attr "predicable" "yes")])
6870
6871 ;;
6872 ;; [srshrl_di]
6873 ;;
6874 (define_insn "mve_srshrl_di"
6875   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6876         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
6877                     (match_operand:SI 2 "immediate_operand" "Pg")]
6878          SRSHRL))]
6879   "TARGET_HAVE_MVE"
6880   "srshrl%?\\t%Q1, %R1, %2"
6881   [(set_attr "predicable" "yes")])
6882
6883 ;;
6884 ;; [sqshll_di]
6885 ;;
6886 (define_insn "mve_sqshll_di"
6887   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
6888         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
6889                       (match_operand:SI 2 "immediate_operand" "Pg")))]
6890   "TARGET_HAVE_MVE"
6891   "sqshll%?\\t%Q1, %R1, %2"
6892   [(set_attr "predicable" "yes")])
6893
6894 ;;
6895 ;; [vshlcq_m_u vshlcq_m_s]
6896 ;;
6897 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
6898  [(match_operand:MVE_2 0 "s_register_operand")
6899   (match_operand:MVE_2 1 "s_register_operand")
6900   (match_operand:SI 2 "s_register_operand")
6901   (match_operand:SI 3 "mve_imm_32")
6902   (match_operand:<MVE_VPRED> 4 "vpr_register_operand")
6903   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
6904  "TARGET_HAVE_MVE"
6905 {
6906   rtx ignore_wb = gen_reg_rtx (SImode);
6907   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
6908                                             operands[2], operands[3],
6909                                             operands[4]));
6910   DONE;
6911 })
6912
6913 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
6914  [(match_operand:SI 0 "s_register_operand")
6915   (match_operand:MVE_2 1 "s_register_operand")
6916   (match_operand:SI 2 "s_register_operand")
6917   (match_operand:SI 3 "mve_imm_32")
6918   (match_operand:<MVE_VPRED> 4 "vpr_register_operand")
6919   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
6920  "TARGET_HAVE_MVE"
6921 {
6922   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
6923   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
6924                                             operands[1], operands[2],
6925                                             operands[3], operands[4]));
6926   DONE;
6927 })
6928
6929 (define_insn "mve_vshlcq_m_<supf><mode>"
6930  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
6931        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
6932                       (match_operand:SI 3 "s_register_operand" "1")
6933                       (match_operand:SI 4 "mve_imm_32" "Rf")
6934                       (match_operand:<MVE_VPRED> 5 "vpr_register_operand" "Up")]
6935         VSHLCQ_M))
6936   (set (match_operand:SI  1 "s_register_operand" "=r")
6937        (unspec:SI [(match_dup 2)
6938                    (match_dup 3)
6939                    (match_dup 4)
6940                    (match_dup 5)]
6941         VSHLCQ_M))
6942  ]
6943  "TARGET_HAVE_MVE"
6944  "vpst\;vshlct\t%q0, %1, %4"
6945  [(set_attr "type" "mve_move")
6946   (set_attr "length" "8")])
6947
6948 ;; CDE instructions on MVE registers.
6949
6950 (define_insn "arm_vcx1qv16qi"
6951   [(set (match_operand:V16QI 0 "register_operand" "=t")
6952         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
6953                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
6954          UNSPEC_VCDE))]
6955   "TARGET_CDE && TARGET_HAVE_MVE"
6956   "vcx1\\tp%c1, %q0, #%c2"
6957   [(set_attr "type" "coproc")]
6958 )
6959
6960 (define_insn "arm_vcx1qav16qi"
6961   [(set (match_operand:V16QI 0 "register_operand" "=t")
6962         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
6963                             (match_operand:V16QI 2 "register_operand" "0")
6964                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
6965          UNSPEC_VCDEA))]
6966   "TARGET_CDE && TARGET_HAVE_MVE"
6967   "vcx1a\\tp%c1, %q0, #%c3"
6968   [(set_attr "type" "coproc")]
6969 )
6970
6971 (define_insn "arm_vcx2qv16qi"
6972   [(set (match_operand:V16QI 0 "register_operand" "=t")
6973         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
6974                           (match_operand:V16QI 2 "register_operand" "t")
6975                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
6976          UNSPEC_VCDE))]
6977   "TARGET_CDE && TARGET_HAVE_MVE"
6978   "vcx2\\tp%c1, %q0, %q2, #%c3"
6979   [(set_attr "type" "coproc")]
6980 )
6981
6982 (define_insn "arm_vcx2qav16qi"
6983   [(set (match_operand:V16QI 0 "register_operand" "=t")
6984         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
6985                           (match_operand:V16QI 2 "register_operand" "0")
6986                           (match_operand:V16QI 3 "register_operand" "t")
6987                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
6988          UNSPEC_VCDEA))]
6989   "TARGET_CDE && TARGET_HAVE_MVE"
6990   "vcx2a\\tp%c1, %q0, %q3, #%c4"
6991   [(set_attr "type" "coproc")]
6992 )
6993
6994 (define_insn "arm_vcx3qv16qi"
6995   [(set (match_operand:V16QI 0 "register_operand" "=t")
6996         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
6997                           (match_operand:V16QI 2 "register_operand" "t")
6998                           (match_operand:V16QI 3 "register_operand" "t")
6999                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
7000          UNSPEC_VCDE))]
7001   "TARGET_CDE && TARGET_HAVE_MVE"
7002   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
7003   [(set_attr "type" "coproc")]
7004 )
7005
7006 (define_insn "arm_vcx3qav16qi"
7007   [(set (match_operand:V16QI 0 "register_operand" "=t")
7008         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
7009                           (match_operand:V16QI 2 "register_operand" "0")
7010                           (match_operand:V16QI 3 "register_operand" "t")
7011                           (match_operand:V16QI 4 "register_operand" "t")
7012                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
7013          UNSPEC_VCDEA))]
7014   "TARGET_CDE && TARGET_HAVE_MVE"
7015   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
7016   [(set_attr "type" "coproc")]
7017 )
7018
7019 (define_insn "arm_vcx1q<a>_p_v16qi"
7020   [(set (match_operand:V16QI 0 "register_operand" "=t")
7021         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
7022                            (match_operand:V16QI 2 "register_operand" "0")
7023                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
7024                            (match_operand:V16BI 4 "vpr_register_operand" "Up")]
7025          CDE_VCX))]
7026   "TARGET_CDE && TARGET_HAVE_MVE"
7027   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
7028   [(set_attr "type" "coproc")
7029    (set_attr "length" "8")]
7030 )
7031
7032 (define_insn "arm_vcx2q<a>_p_v16qi"
7033   [(set (match_operand:V16QI 0 "register_operand" "=t")
7034         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
7035                           (match_operand:V16QI 2 "register_operand" "0")
7036                           (match_operand:V16QI 3 "register_operand" "t")
7037                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
7038                           (match_operand:V16BI 5 "vpr_register_operand" "Up")]
7039          CDE_VCX))]
7040   "TARGET_CDE && TARGET_HAVE_MVE"
7041   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
7042   [(set_attr "type" "coproc")
7043    (set_attr "length" "8")]
7044 )
7045
7046 (define_insn "arm_vcx3q<a>_p_v16qi"
7047   [(set (match_operand:V16QI 0 "register_operand" "=t")
7048         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
7049                           (match_operand:V16QI 2 "register_operand" "0")
7050                           (match_operand:V16QI 3 "register_operand" "t")
7051                           (match_operand:V16QI 4 "register_operand" "t")
7052                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
7053                           (match_operand:V16BI 6 "vpr_register_operand" "Up")]
7054          CDE_VCX))]
7055   "TARGET_CDE && TARGET_HAVE_MVE"
7056   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
7057   [(set_attr "type" "coproc")
7058    (set_attr "length" "8")]
7059 )
7060
7061 (define_insn "*movmisalign<mode>_mve_store"
7062   [(set (match_operand:MVE_VLD_ST 0 "mve_memory_operand"             "=Ux")
7063         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
7064          UNSPEC_MISALIGNED_ACCESS))]
7065   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
7066     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
7067    && !BYTES_BIG_ENDIAN && unaligned_access"
7068   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
7069   [(set_attr "type" "mve_store")]
7070 )
7071
7072
7073 (define_insn "*movmisalign<mode>_mve_load"
7074   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
7075         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "mve_memory_operand" " Ux")]
7076          UNSPEC_MISALIGNED_ACCESS))]
7077   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
7078     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
7079    && !BYTES_BIG_ENDIAN && unaligned_access"
7080   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
7081   [(set_attr "type" "mve_load")]
7082 )
7083
7084 ;; Expander for VxBI moves
7085 (define_expand "mov<mode>"
7086   [(set (match_operand:MVE_7 0 "nonimmediate_operand")
7087         (match_operand:MVE_7 1 "general_operand"))]
7088   "TARGET_HAVE_MVE"
7089   {
7090     if (!register_operand (operands[0], <MODE>mode))
7091       operands[1] = force_reg (<MODE>mode, operands[1]);
7092   }
7093 )
7094
7095 ;; Expanders for vec_cmp and vcond
7096
7097 (define_expand "vec_cmp<mode><MVE_vpred>"
7098   [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
7099         (match_operator:<MVE_VPRED> 1 "comparison_operator"
7100           [(match_operand:MVE_VLD_ST 2 "s_register_operand")
7101            (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
7102   "TARGET_HAVE_MVE
7103    && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
7104 {
7105   arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
7106                              operands[2], operands[3], false);
7107   DONE;
7108 })
7109
7110 (define_expand "vec_cmpu<mode><MVE_vpred>"
7111   [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
7112         (match_operator:<MVE_VPRED> 1 "comparison_operator"
7113           [(match_operand:MVE_2 2 "s_register_operand")
7114            (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
7115   "TARGET_HAVE_MVE"
7116 {
7117   arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
7118                              operands[2], operands[3], false);
7119   DONE;
7120 })
7121
7122 (define_expand "vcond_mask_<mode><MVE_vpred>"
7123   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
7124         (if_then_else:MVE_VLD_ST
7125           (match_operand:<MVE_VPRED> 3 "s_register_operand")
7126           (match_operand:MVE_VLD_ST 1 "s_register_operand")
7127           (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
7128   "TARGET_HAVE_MVE"
7129 {
7130   switch (GET_MODE_CLASS (<MODE>mode))
7131     {
7132       case MODE_VECTOR_INT:
7133         emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
7134                                    operands[1], operands[2], operands[3]));
7135         break;
7136       case MODE_VECTOR_FLOAT:
7137         emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
7138                                      operands[1], operands[2], operands[3]));
7139         break;
7140       default:
7141         gcc_unreachable ();
7142     }
7143   DONE;
7144 })
7145
7146 ;; Reinterpret operand 1 in operand 0's mode, without changing its contents.
7147 (define_expand "@arm_mve_reinterpret<mode>"
7148   [(set (match_operand:MVE_vecs 0 "register_operand")
7149         (unspec:MVE_vecs
7150           [(match_operand 1 "arm_any_register_operand")]
7151           REINTERPRET))]
7152   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
7153     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
7154   {
7155     machine_mode src_mode = GET_MODE (operands[1]);
7156     if (targetm.can_change_mode_class (<MODE>mode, src_mode, VFP_REGS))
7157       {
7158         emit_move_insn (operands[0], gen_lowpart (<MODE>mode, operands[1]));
7159         DONE;
7160       }
7161   }
7162 )