gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2021 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,UxUi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode || <MODE>mode == TImode)
  45         {
  46           if (which_alternative == 7)
  47             output_asm_insn ("vstrw.32\t%q1, %E0", operands);
  48           else
  49             output_asm_insn ("vldrw.u32\t%q0, %E1",operands);
  50         }
  51       else
  52         {
  53           if (which_alternative == 7)
  54             output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0", operands);
  55           else
  56             output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1", operands);
  57         }
  58       return "";
  59     }
  60   switch (which_alternative)
  61     {
  62     case 0:
  63       return "vmov\t%q0, %q1";
  64     case 1:
  65       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  66     case 2:
  67       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  68     case 5:
  69       return output_move_quad (operands);
  70     case 8:
  71         return output_move_neon (operands);
  72     default:
  73       gcc_unreachable ();
  74       return "";
  75     }
  76 }
  77   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
  78    (set_attr "length" "4,8,8,4,8,8,4,4,4")
  79    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
  80    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
  81
  82 (define_insn "*mve_vdup<mode>"
  83   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  84         (vec_duplicate:MVE_vecs
  85           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  86   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  87   "vdup.<V_sz_elem>\t%q0, %1"
  88   [(set_attr "length" "4")
  89    (set_attr "type" "mve_move")])
  90
  91 ;;
  92 ;; [vst4q])
  93 ;;
  94 (define_insn "mve_vst4q<mode>"
  95   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
  96         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
  97                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
  98          VST4Q))
  99   ]
 100   "TARGET_HAVE_MVE"
 101 {
 102    rtx ops[6];
 103    int regno = REGNO (operands[1]);
 104    ops[0] = gen_rtx_REG (TImode, regno);
 105    ops[1] = gen_rtx_REG (TImode, regno+4);
 106    ops[2] = gen_rtx_REG (TImode, regno+8);
 107    ops[3] = gen_rtx_REG (TImode, regno+12);
 108    rtx reg  = operands[0];
 109    while (reg && !REG_P (reg))
 110     reg = XEXP (reg, 0);
 111    gcc_assert (REG_P (reg));
 112    ops[4] = reg;
 113    ops[5] = operands[0];
 114    /* Here in first three instructions data is stored to ops[4]'s location but
 115       in the fourth instruction data is stored to operands[0], this is to
 116       support the writeback.  */
 117    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 118                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 119                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 120                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 121    return "";
 122 }
 123   [(set_attr "length" "16")])
 124
 125 ;;
 126 ;; [vrndq_m_f])
 127 ;;
 128 (define_insn "mve_vrndq_m_f<mode>"
 129   [
 130    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 131         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 132                        (match_operand:MVE_0 2 "s_register_operand" "w")
 133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 134          VRNDQ_M_F))
 135   ]
 136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 137   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 138   [(set_attr "type" "mve_move")
 139    (set_attr "length""8")])
 140
 141 ;;
 142 ;; [vrndxq_f])
 143 ;;
 144 (define_insn "mve_vrndxq_f<mode>"
 145   [
 146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 148          VRNDXQ_F))
 149   ]
 150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 151   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 152   [(set_attr "type" "mve_move")
 153 ])
 154
 155 ;;
 156 ;; [vrndq_f])
 157 ;;
 158 (define_insn "mve_vrndq_f<mode>"
 159   [
 160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 162          VRNDQ_F))
 163   ]
 164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 165   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 166   [(set_attr "type" "mve_move")
 167 ])
 168
 169 ;;
 170 ;; [vrndpq_f])
 171 ;;
 172 (define_insn "mve_vrndpq_f<mode>"
 173   [
 174    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 175         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 176          VRNDPQ_F))
 177   ]
 178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 179   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 180   [(set_attr "type" "mve_move")
 181 ])
 182
 183 ;;
 184 ;; [vrndnq_f])
 185 ;;
 186 (define_insn "mve_vrndnq_f<mode>"
 187   [
 188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 190          VRNDNQ_F))
 191   ]
 192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 193   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 194   [(set_attr "type" "mve_move")
 195 ])
 196
 197 ;;
 198 ;; [vrndmq_f])
 199 ;;
 200 (define_insn "mve_vrndmq_f<mode>"
 201   [
 202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 204          VRNDMQ_F))
 205   ]
 206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 207   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 208   [(set_attr "type" "mve_move")
 209 ])
 210
 211 ;;
 212 ;; [vrndaq_f])
 213 ;;
 214 (define_insn "mve_vrndaq_f<mode>"
 215   [
 216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 217         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 218          VRNDAQ_F))
 219   ]
 220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 221   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 222   [(set_attr "type" "mve_move")
 223 ])
 224
 225 ;;
 226 ;; [vrev64q_f])
 227 ;;
 228 (define_insn "mve_vrev64q_f<mode>"
 229   [
 230    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 232          VREV64Q_F))
 233   ]
 234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 235   "vrev64.%#<V_sz_elem> %q0, %q1"
 236   [(set_attr "type" "mve_move")
 237 ])
 238
 239 ;;
 240 ;; [vnegq_f])
 241 ;;
 242 (define_insn "mve_vnegq_f<mode>"
 243   [
 244    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 245         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 246   ]
 247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 248   "vneg.f%#<V_sz_elem>  %q0, %q1"
 249   [(set_attr "type" "mve_move")
 250 ])
 251
 252 ;;
 253 ;; [vdupq_n_f])
 254 ;;
 255 (define_insn "mve_vdupq_n_f<mode>"
 256   [
 257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 258         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 259          VDUPQ_N_F))
 260   ]
 261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 262   "vdup.%#<V_sz_elem>   %q0, %1"
 263   [(set_attr "type" "mve_move")
 264 ])
 265
 266 ;;
 267 ;; [vabsq_f])
 268 ;;
 269 (define_insn "mve_vabsq_f<mode>"
 270   [
 271    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 272         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 273          VABSQ_F))
 274   ]
 275   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 276   "vabs.f%#<V_sz_elem>  %q0, %q1"
 277   [(set_attr "type" "mve_move")
 278 ])
 279
 280 ;;
 281 ;; [vrev32q_f])
 282 ;;
 283 (define_insn "mve_vrev32q_fv8hf"
 284   [
 285    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 286         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 287          VREV32Q_F))
 288   ]
 289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 290   "vrev32.16 %q0, %q1"
 291   [(set_attr "type" "mve_move")
 292 ])
 293 ;;
 294 ;; [vcvttq_f32_f16])
 295 ;;
 296 (define_insn "mve_vcvttq_f32_f16v4sf"
 297   [
 298    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 299         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 300          VCVTTQ_F32_F16))
 301   ]
 302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 303   "vcvtt.f32.f16 %q0, %q1"
 304   [(set_attr "type" "mve_move")
 305 ])
 306
 307 ;;
 308 ;; [vcvtbq_f32_f16])
 309 ;;
 310 (define_insn "mve_vcvtbq_f32_f16v4sf"
 311   [
 312    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 313         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 314          VCVTBQ_F32_F16))
 315   ]
 316   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 317   "vcvtb.f32.f16 %q0, %q1"
 318   [(set_attr "type" "mve_move")
 319 ])
 320
 321 ;;
 322 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 323 ;;
 324 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 325   [
 326    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 327         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 328          VCVTQ_TO_F))
 329   ]
 330   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 331   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 332   [(set_attr "type" "mve_move")
 333 ])
 334
 335 ;;
 336 ;; [vrev64q_u, vrev64q_s])
 337 ;;
 338 (define_insn "mve_vrev64q_<supf><mode>"
 339   [
 340    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 341         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 342          VREV64Q))
 343   ]
 344   "TARGET_HAVE_MVE"
 345   "vrev64.%#<V_sz_elem> %q0, %q1"
 346   [(set_attr "type" "mve_move")
 347 ])
 348
 349 ;;
 350 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 351 ;;
 352 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 353   [
 354    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 355         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 356          VCVTQ_FROM_F))
 357   ]
 358   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 359   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 360   [(set_attr "type" "mve_move")
 361 ])
 362 ;; [vqnegq_s])
 363 ;;
 364 (define_insn "mve_vqnegq_s<mode>"
 365   [
 366    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 367         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 368          VQNEGQ_S))
 369   ]
 370   "TARGET_HAVE_MVE"
 371   "vqneg.s%#<V_sz_elem> %q0, %q1"
 372   [(set_attr "type" "mve_move")
 373 ])
 374
 375 ;;
 376 ;; [vqabsq_s])
 377 ;;
 378 (define_insn "mve_vqabsq_s<mode>"
 379   [
 380    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 381         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 382          VQABSQ_S))
 383   ]
 384   "TARGET_HAVE_MVE"
 385   "vqabs.s%#<V_sz_elem> %q0, %q1"
 386   [(set_attr "type" "mve_move")
 387 ])
 388
 389 ;;
 390 ;; [vnegq_s])
 391 ;;
 392 (define_insn "mve_vnegq_s<mode>"
 393   [
 394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 395         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 396   ]
 397   "TARGET_HAVE_MVE"
 398   "vneg.s%#<V_sz_elem>  %q0, %q1"
 399   [(set_attr "type" "mve_move")
 400 ])
 401
 402 ;;
 403 ;; [vmvnq_u, vmvnq_s])
 404 ;;
 405 (define_insn "mve_vmvnq_u<mode>"
 406   [
 407    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 408         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 409   ]
 410   "TARGET_HAVE_MVE"
 411   "vmvn\t%q0, %q1"
 412   [(set_attr "type" "mve_move")
 413 ])
 414 (define_expand "mve_vmvnq_s<mode>"
 415   [
 416    (set (match_operand:MVE_2 0 "s_register_operand")
 417         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 418   ]
 419   "TARGET_HAVE_MVE"
 420 )
 421
 422 ;;
 423 ;; [vdupq_n_u, vdupq_n_s])
 424 ;;
 425 (define_insn "mve_vdupq_n_<supf><mode>"
 426   [
 427    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 428         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 429          VDUPQ_N))
 430   ]
 431   "TARGET_HAVE_MVE"
 432   "vdup.%#<V_sz_elem>   %q0, %1"
 433   [(set_attr "type" "mve_move")
 434 ])
 435
 436 ;;
 437 ;; [vclzq_u, vclzq_s])
 438 ;;
 439 (define_insn "mve_vclzq_<supf><mode>"
 440   [
 441    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 442         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 443          VCLZQ))
 444   ]
 445   "TARGET_HAVE_MVE"
 446   "vclz.i%#<V_sz_elem>  %q0, %q1"
 447   [(set_attr "type" "mve_move")
 448 ])
 449
 450 ;;
 451 ;; [vclsq_s])
 452 ;;
 453 (define_insn "mve_vclsq_s<mode>"
 454   [
 455    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 456         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 457          VCLSQ_S))
 458   ]
 459   "TARGET_HAVE_MVE"
 460   "vcls.s%#<V_sz_elem>  %q0, %q1"
 461   [(set_attr "type" "mve_move")
 462 ])
 463
 464 ;;
 465 ;; [vaddvq_s, vaddvq_u])
 466 ;;
 467 (define_insn "mve_vaddvq_<supf><mode>"
 468   [
 469    (set (match_operand:SI 0 "s_register_operand" "=Te")
 470         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 471          VADDVQ))
 472   ]
 473   "TARGET_HAVE_MVE"
 474   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 475   [(set_attr "type" "mve_move")
 476 ])
 477
 478 ;;
 479 ;; [vabsq_s])
 480 ;;
 481 (define_insn "mve_vabsq_s<mode>"
 482   [
 483    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 484         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 485          VABSQ_S))
 486   ]
 487   "TARGET_HAVE_MVE"
 488   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 489   [(set_attr "type" "mve_move")
 490 ])
 491
 492 ;;
 493 ;; [vrev32q_u, vrev32q_s])
 494 ;;
 495 (define_insn "mve_vrev32q_<supf><mode>"
 496   [
 497    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 498         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 499          VREV32Q))
 500   ]
 501   "TARGET_HAVE_MVE"
 502   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 503   [(set_attr "type" "mve_move")
 504 ])
 505
 506 ;;
 507 ;; [vmovltq_u, vmovltq_s])
 508 ;;
 509 (define_insn "mve_vmovltq_<supf><mode>"
 510   [
 511    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 512         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 513          VMOVLTQ))
 514   ]
 515   "TARGET_HAVE_MVE"
 516   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 517   [(set_attr "type" "mve_move")
 518 ])
 519
 520 ;;
 521 ;; [vmovlbq_s, vmovlbq_u])
 522 ;;
 523 (define_insn "mve_vmovlbq_<supf><mode>"
 524   [
 525    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 526         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 527          VMOVLBQ))
 528   ]
 529   "TARGET_HAVE_MVE"
 530   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 531   [(set_attr "type" "mve_move")
 532 ])
 533
 534 ;;
 535 ;; [vcvtpq_s, vcvtpq_u])
 536 ;;
 537 (define_insn "mve_vcvtpq_<supf><mode>"
 538   [
 539    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 540         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 541          VCVTPQ))
 542   ]
 543   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 544   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 545   [(set_attr "type" "mve_move")
 546 ])
 547
 548 ;;
 549 ;; [vcvtnq_s, vcvtnq_u])
 550 ;;
 551 (define_insn "mve_vcvtnq_<supf><mode>"
 552   [
 553    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 554         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 555          VCVTNQ))
 556   ]
 557   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 558   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 559   [(set_attr "type" "mve_move")
 560 ])
 561
 562 ;;
 563 ;; [vcvtmq_s, vcvtmq_u])
 564 ;;
 565 (define_insn "mve_vcvtmq_<supf><mode>"
 566   [
 567    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 568         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 569          VCVTMQ))
 570   ]
 571   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 572   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 573   [(set_attr "type" "mve_move")
 574 ])
 575
 576 ;;
 577 ;; [vcvtaq_u, vcvtaq_s])
 578 ;;
 579 (define_insn "mve_vcvtaq_<supf><mode>"
 580   [
 581    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 582         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 583          VCVTAQ))
 584   ]
 585   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 586   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 587   [(set_attr "type" "mve_move")
 588 ])
 589
 590 ;;
 591 ;; [vmvnq_n_u, vmvnq_n_s])
 592 ;;
 593 (define_insn "mve_vmvnq_n_<supf><mode>"
 594   [
 595    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 596         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 597          VMVNQ_N))
 598   ]
 599   "TARGET_HAVE_MVE"
 600   "vmvn.i%#<V_sz_elem>  %q0, %1"
 601   [(set_attr "type" "mve_move")
 602 ])
 603
 604 ;;
 605 ;; [vrev16q_u, vrev16q_s])
 606 ;;
 607 (define_insn "mve_vrev16q_<supf>v16qi"
 608   [
 609    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 610         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 611          VREV16Q))
 612   ]
 613   "TARGET_HAVE_MVE"
 614   "vrev16.8 %q0, %q1"
 615   [(set_attr "type" "mve_move")
 616 ])
 617
 618 ;;
 619 ;; [vaddlvq_s vaddlvq_u])
 620 ;;
 621 (define_insn "mve_vaddlvq_<supf>v4si"
 622   [
 623    (set (match_operand:DI 0 "s_register_operand" "=r")
 624         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 625          VADDLVQ))
 626   ]
 627   "TARGET_HAVE_MVE"
 628   "vaddlv.<supf>32 %Q0, %R0, %q1"
 629   [(set_attr "type" "mve_move")
 630 ])
 631
 632 ;;
 633 ;; [vctp8q vctp16q vctp32q vctp64q])
 634 ;;
 635 (define_insn "mve_vctp<mode1>qhi"
 636   [
 637    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 638         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 639         VCTPQ))
 640   ]
 641   "TARGET_HAVE_MVE"
 642   "vctp.<mode1> %1"
 643   [(set_attr "type" "mve_move")
 644 ])
 645
 646 ;;
 647 ;; [vpnot])
 648 ;;
 649 (define_insn "mve_vpnothi"
 650   [
 651    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 652         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 653          VPNOT))
 654   ]
 655   "TARGET_HAVE_MVE"
 656   "vpnot"
 657   [(set_attr "type" "mve_move")
 658 ])
 659
 660 ;;
 661 ;; [vsubq_n_f])
 662 ;;
 663 (define_insn "mve_vsubq_n_f<mode>"
 664   [
 665    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 666         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 667                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 668          VSUBQ_N_F))
 669   ]
 670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 671   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 672   [(set_attr "type" "mve_move")
 673 ])
 674
 675 ;;
 676 ;; [vbrsrq_n_f])
 677 ;;
 678 (define_insn "mve_vbrsrq_n_f<mode>"
 679   [
 680    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 681         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 682                        (match_operand:SI 2 "s_register_operand" "r")]
 683          VBRSRQ_N_F))
 684   ]
 685   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 686   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 687   [(set_attr "type" "mve_move")
 688 ])
 689
 690 ;;
 691 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 692 ;;
 693 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 694   [
 695    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 696         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 697                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 698          VCVTQ_N_TO_F))
 699   ]
 700   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 701   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 702   [(set_attr "type" "mve_move")
 703 ])
 704
 705 ;; [vcreateq_f])
 706 ;;
 707 (define_insn "mve_vcreateq_f<mode>"
 708   [
 709    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 710         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 711                        (match_operand:DI 2 "s_register_operand" "r")]
 712          VCREATEQ_F))
 713   ]
 714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 715   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 716   [(set_attr "type" "mve_move")
 717    (set_attr "length""8")])
 718
 719 ;;
 720 ;; [vcreateq_u, vcreateq_s])
 721 ;;
 722 (define_insn "mve_vcreateq_<supf><mode>"
 723   [
 724    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 725         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 726                        (match_operand:DI 2 "s_register_operand" "r")]
 727          VCREATEQ))
 728   ]
 729   "TARGET_HAVE_MVE"
 730   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 731   [(set_attr "type" "mve_move")
 732    (set_attr "length""8")])
 733
 734 ;;
 735 ;; [vshrq_n_s, vshrq_n_u])
 736 ;;
 737 ;; Version that takes an immediate as operand 2.
 738 (define_insn "mve_vshrq_n_<supf><mode>"
 739   [
 740    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 741         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 742                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 743          VSHRQ_N))
 744   ]
 745   "TARGET_HAVE_MVE"
 746   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 747   [(set_attr "type" "mve_move")
 748 ])
 749
 750 ;; Versions that take constant vectors as operand 2 (with all elements
 751 ;; equal).
 752 (define_insn "mve_vshrq_n_s<mode>_imm"
 753   [
 754    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 755         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 756                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 757   ]
 758   "TARGET_HAVE_MVE"
 759   {
 760     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 761                                         <MODE>mode,
 762                                         VALID_NEON_QREG_MODE (<MODE>mode),
 763                                         true);
 764   }
 765   [(set_attr "type" "mve_move")
 766 ])
 767 (define_insn "mve_vshrq_n_u<mode>_imm"
 768   [
 769    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 770         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 771                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 772   ]
 773   "TARGET_HAVE_MVE"
 774   {
 775     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 776                                         <MODE>mode,
 777                                         VALID_NEON_QREG_MODE (<MODE>mode),
 778                                         true);
 779   }
 780   [(set_attr "type" "mve_move")
 781 ])
 782
 783 ;;
 784 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 785 ;;
 786 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 787   [
 788    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 789         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 790                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 791          VCVTQ_N_FROM_F))
 792   ]
 793   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 794   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 795   [(set_attr "type" "mve_move")
 796 ])
 797
 798 ;;
 799 ;; [vaddlvq_p_s])
 800 ;;
 801 (define_insn "mve_vaddlvq_p_<supf>v4si"
 802   [
 803    (set (match_operand:DI 0 "s_register_operand" "=r")
 804         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 805                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 806          VADDLVQ_P))
 807   ]
 808   "TARGET_HAVE_MVE"
 809   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 810   [(set_attr "type" "mve_move")
 811    (set_attr "length""8")])
 812
 813 ;;
 814 ;; [vcmpneq_, vcmpcsq_, vcmpeqq_, vcmpgeq_, vcmpgtq_, vcmphiq_, vcmpleq_, vcmpltq_])
 815 ;;
 816 (define_insn "mve_vcmp<mve_cmp_op>q_<mode>"
 817   [
 818    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 819         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 820                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 821   ]
 822   "TARGET_HAVE_MVE"
 823   "vcmp.<mve_cmp_type>%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
 824   [(set_attr "type" "mve_move")
 825 ])
 826
 827 ;;
 828 ;; [vcmpcsq_n_, vcmpeqq_n_, vcmpgeq_n_, vcmpgtq_n_, vcmphiq_n_, vcmpleq_n_, vcmpltq_n_, vcmpneq_n_])
 829 ;;
 830 (define_insn "mve_vcmp<mve_cmp_op>q_n_<mode>"
 831   [
 832    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 833         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 834                     (match_operand:<V_elem> 2 "s_register_operand" "r")))
 835   ]
 836   "TARGET_HAVE_MVE"
 837   "vcmp.<mve_cmp_type>%#<V_sz_elem>     <mve_cmp_op>, %q1, %2"
 838   [(set_attr "type" "mve_move")
 839 ])
 840
 841 ;;
 842 ;; [vshlq_s, vshlq_u])
 843 ;; See vec-common.md
 844
 845 ;;
 846 ;; [vabdq_s, vabdq_u])
 847 ;;
 848 (define_insn "mve_vabdq_<supf><mode>"
 849   [
 850    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 851         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 852                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 853          VABDQ))
 854   ]
 855   "TARGET_HAVE_MVE"
 856   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 857   [(set_attr "type" "mve_move")
 858 ])
 859
 860 ;;
 861 ;; [vaddq_n_s, vaddq_n_u])
 862 ;;
 863 (define_insn "mve_vaddq_n_<supf><mode>"
 864   [
 865    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 866         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 867                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 868          VADDQ_N))
 869   ]
 870   "TARGET_HAVE_MVE"
 871   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 872   [(set_attr "type" "mve_move")
 873 ])
 874
 875 ;;
 876 ;; [vaddvaq_s, vaddvaq_u])
 877 ;;
 878 (define_insn "mve_vaddvaq_<supf><mode>"
 879   [
 880    (set (match_operand:SI 0 "s_register_operand" "=Te")
 881         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 882                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 883          VADDVAQ))
 884   ]
 885   "TARGET_HAVE_MVE"
 886   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 887   [(set_attr "type" "mve_move")
 888 ])
 889
 890 ;;
 891 ;; [vaddvq_p_u, vaddvq_p_s])
 892 ;;
 893 (define_insn "mve_vaddvq_p_<supf><mode>"
 894   [
 895    (set (match_operand:SI 0 "s_register_operand" "=Te")
 896         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 897                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 898          VADDVQ_P))
 899   ]
 900   "TARGET_HAVE_MVE"
 901   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 902   [(set_attr "type" "mve_move")
 903    (set_attr "length""8")])
 904
 905 ;;
 906 ;; [vandq_u, vandq_s])
 907 ;;
 908 ;; signed and unsigned versions are the same: define the unsigned
 909 ;; insn, and use an expander for the signed one as we still reference
 910 ;; both names from arm_mve.h.
 911 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 912 (define_insn "mve_vandq_u<mode>"
 913   [
 914    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 915         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 916                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 917   ]
 918   "TARGET_HAVE_MVE"
 919   "@
 920    vand\t%q0, %q1, %q2
 921    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 922   [(set_attr "type" "mve_move")
 923 ])
 924 (define_expand "mve_vandq_s<mode>"
 925   [
 926    (set (match_operand:MVE_2 0 "s_register_operand")
 927         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 928                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 929   ]
 930   "TARGET_HAVE_MVE"
 931 )
 932
 933 ;;
 934 ;; [vbicq_s, vbicq_u])
 935 ;;
 936 (define_insn "mve_vbicq_u<mode>"
 937   [
 938    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 939         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 940                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 941   ]
 942   "TARGET_HAVE_MVE"
 943   "vbic\t%q0, %q1, %q2"
 944   [(set_attr "type" "mve_move")
 945 ])
 946
 947 (define_expand "mve_vbicq_s<mode>"
 948   [
 949    (set (match_operand:MVE_2 0 "s_register_operand")
 950         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 951                    (match_operand:MVE_2 1 "s_register_operand")))
 952   ]
 953   "TARGET_HAVE_MVE"
 954 )
 955
 956 ;;
 957 ;; [vbrsrq_n_u, vbrsrq_n_s])
 958 ;;
 959 (define_insn "mve_vbrsrq_n_<supf><mode>"
 960   [
 961    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 962         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 963                        (match_operand:SI 2 "s_register_operand" "r")]
 964          VBRSRQ_N))
 965   ]
 966   "TARGET_HAVE_MVE"
 967   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 968   [(set_attr "type" "mve_move")
 969 ])
 970
 971 ;;
 972 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 973 ;;
 974 (define_insn "mve_vcaddq<mve_rot><mode>"
 975   [
 976    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 977         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 978                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 979          VCADD))
 980   ]
 981   "TARGET_HAVE_MVE"
 982   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 983   [(set_attr "type" "mve_move")
 984 ])
 985
 986 ;; Auto vectorizer pattern for int vcadd
 987 (define_expand "cadd<rot><mode>3"
 988   [(set (match_operand:MVE_2 0 "register_operand")
 989         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
 990                        (match_operand:MVE_2 2 "register_operand")]
 991           VCADD))]
 992   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
 993 )
 994
 995 ;;
 996 ;; [veorq_u, veorq_s])
 997 ;;
 998 (define_insn "mve_veorq_u<mode>"
 999   [
1000    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1001         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1002                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1003   ]
1004   "TARGET_HAVE_MVE"
1005   "veor\t%q0, %q1, %q2"
1006   [(set_attr "type" "mve_move")
1007 ])
1008 (define_expand "mve_veorq_s<mode>"
1009   [
1010    (set (match_operand:MVE_2 0 "s_register_operand")
1011         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1012                    (match_operand:MVE_2 2 "s_register_operand")))
1013   ]
1014   "TARGET_HAVE_MVE"
1015 )
1016
1017 ;;
1018 ;; [vhaddq_n_u, vhaddq_n_s])
1019 ;;
1020 (define_insn "mve_vhaddq_n_<supf><mode>"
1021   [
1022    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1023         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1024                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1025          VHADDQ_N))
1026   ]
1027   "TARGET_HAVE_MVE"
1028   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1029   [(set_attr "type" "mve_move")
1030 ])
1031
1032 ;;
1033 ;; [vhaddq_s, vhaddq_u])
1034 ;;
1035 (define_insn "mve_vhaddq_<supf><mode>"
1036   [
1037    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1038         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1039                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1040          VHADDQ))
1041   ]
1042   "TARGET_HAVE_MVE"
1043   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1044   [(set_attr "type" "mve_move")
1045 ])
1046
1047 ;;
1048 ;; [vhcaddq_rot270_s])
1049 ;;
1050 (define_insn "mve_vhcaddq_rot270_s<mode>"
1051   [
1052    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1053         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1054                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1055          VHCADDQ_ROT270_S))
1056   ]
1057   "TARGET_HAVE_MVE"
1058   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1059   [(set_attr "type" "mve_move")
1060 ])
1061
1062 ;;
1063 ;; [vhcaddq_rot90_s])
1064 ;;
1065 (define_insn "mve_vhcaddq_rot90_s<mode>"
1066   [
1067    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1068         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1069                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1070          VHCADDQ_ROT90_S))
1071   ]
1072   "TARGET_HAVE_MVE"
1073   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1074   [(set_attr "type" "mve_move")
1075 ])
1076
1077 ;;
1078 ;; [vhsubq_n_u, vhsubq_n_s])
1079 ;;
1080 (define_insn "mve_vhsubq_n_<supf><mode>"
1081   [
1082    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1083         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1084                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1085          VHSUBQ_N))
1086   ]
1087   "TARGET_HAVE_MVE"
1088   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1089   [(set_attr "type" "mve_move")
1090 ])
1091
1092 ;;
1093 ;; [vhsubq_s, vhsubq_u])
1094 ;;
1095 (define_insn "mve_vhsubq_<supf><mode>"
1096   [
1097    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1098         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1099                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1100          VHSUBQ))
1101   ]
1102   "TARGET_HAVE_MVE"
1103   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1104   [(set_attr "type" "mve_move")
1105 ])
1106
1107 ;;
1108 ;; [vmaxaq_s])
1109 ;;
1110 (define_insn "mve_vmaxaq_s<mode>"
1111   [
1112    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1113         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1114                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1115          VMAXAQ_S))
1116   ]
1117   "TARGET_HAVE_MVE"
1118   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1119   [(set_attr "type" "mve_move")
1120 ])
1121
1122 ;;
1123 ;; [vmaxavq_s])
1124 ;;
1125 (define_insn "mve_vmaxavq_s<mode>"
1126   [
1127    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1128         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1129                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1130          VMAXAVQ_S))
1131   ]
1132   "TARGET_HAVE_MVE"
1133   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1134   [(set_attr "type" "mve_move")
1135 ])
1136
1137 ;;
1138 ;; [vmaxq_u, vmaxq_s])
1139 ;;
1140 (define_insn "mve_vmaxq_s<mode>"
1141   [
1142    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1143         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1144                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1145   ]
1146   "TARGET_HAVE_MVE"
1147   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1148   [(set_attr "type" "mve_move")
1149 ])
1150
1151 (define_insn "mve_vmaxq_u<mode>"
1152   [
1153    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1154         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1155                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1156   ]
1157   "TARGET_HAVE_MVE"
1158   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1159   [(set_attr "type" "mve_move")
1160 ])
1161
1162 ;;
1163 ;; [vmaxvq_u, vmaxvq_s])
1164 ;;
1165 (define_insn "mve_vmaxvq_<supf><mode>"
1166   [
1167    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1168         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1169                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1170          VMAXVQ))
1171   ]
1172   "TARGET_HAVE_MVE"
1173   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1174   [(set_attr "type" "mve_move")
1175 ])
1176
1177 ;;
1178 ;; [vminaq_s])
1179 ;;
1180 (define_insn "mve_vminaq_s<mode>"
1181   [
1182    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1183         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1184                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1185          VMINAQ_S))
1186   ]
1187   "TARGET_HAVE_MVE"
1188   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1189   [(set_attr "type" "mve_move")
1190 ])
1191
1192 ;;
1193 ;; [vminavq_s])
1194 ;;
1195 (define_insn "mve_vminavq_s<mode>"
1196   [
1197    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1198         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1199                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1200          VMINAVQ_S))
1201   ]
1202   "TARGET_HAVE_MVE"
1203   "vminav.s%#<V_sz_elem>\t%0, %q2"
1204   [(set_attr "type" "mve_move")
1205 ])
1206
1207 ;;
1208 ;; [vminq_s, vminq_u])
1209 ;;
1210 (define_insn "mve_vminq_s<mode>"
1211   [
1212    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1213         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1214                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1215   ]
1216   "TARGET_HAVE_MVE"
1217   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1218   [(set_attr "type" "mve_move")
1219 ])
1220
1221 (define_insn "mve_vminq_u<mode>"
1222   [
1223    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1224         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1225                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1226   ]
1227   "TARGET_HAVE_MVE"
1228   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1229   [(set_attr "type" "mve_move")
1230 ])
1231
1232 ;;
1233 ;; [vminvq_u, vminvq_s])
1234 ;;
1235 (define_insn "mve_vminvq_<supf><mode>"
1236   [
1237    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1238         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1239                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1240          VMINVQ))
1241   ]
1242   "TARGET_HAVE_MVE"
1243   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1244   [(set_attr "type" "mve_move")
1245 ])
1246
1247 ;;
1248 ;; [vmladavq_u, vmladavq_s])
1249 ;;
1250 (define_insn "mve_vmladavq_<supf><mode>"
1251   [
1252    (set (match_operand:SI 0 "s_register_operand" "=Te")
1253         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1254                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1255          VMLADAVQ))
1256   ]
1257   "TARGET_HAVE_MVE"
1258   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1259   [(set_attr "type" "mve_move")
1260 ])
1261
1262 ;;
1263 ;; [vmladavxq_s])
1264 ;;
1265 (define_insn "mve_vmladavxq_s<mode>"
1266   [
1267    (set (match_operand:SI 0 "s_register_operand" "=Te")
1268         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1269                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1270          VMLADAVXQ_S))
1271   ]
1272   "TARGET_HAVE_MVE"
1273   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1274   [(set_attr "type" "mve_move")
1275 ])
1276
1277 ;;
1278 ;; [vmlsdavq_s])
1279 ;;
1280 (define_insn "mve_vmlsdavq_s<mode>"
1281   [
1282    (set (match_operand:SI 0 "s_register_operand" "=Te")
1283         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1284                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1285          VMLSDAVQ_S))
1286   ]
1287   "TARGET_HAVE_MVE"
1288   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1289   [(set_attr "type" "mve_move")
1290 ])
1291
1292 ;;
1293 ;; [vmlsdavxq_s])
1294 ;;
1295 (define_insn "mve_vmlsdavxq_s<mode>"
1296   [
1297    (set (match_operand:SI 0 "s_register_operand" "=Te")
1298         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1299                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1300          VMLSDAVXQ_S))
1301   ]
1302   "TARGET_HAVE_MVE"
1303   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1304   [(set_attr "type" "mve_move")
1305 ])
1306
1307 ;;
1308 ;; [vmulhq_s, vmulhq_u])
1309 ;;
1310 (define_insn "mve_vmulhq_<supf><mode>"
1311   [
1312    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1313         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1314                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1315          VMULHQ))
1316   ]
1317   "TARGET_HAVE_MVE"
1318   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1319   [(set_attr "type" "mve_move")
1320 ])
1321
1322 ;;
1323 ;; [vmullbq_int_u, vmullbq_int_s])
1324 ;;
1325 (define_insn "mve_vmullbq_int_<supf><mode>"
1326   [
1327    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1328         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1329                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1330          VMULLBQ_INT))
1331   ]
1332   "TARGET_HAVE_MVE"
1333   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1334   [(set_attr "type" "mve_move")
1335 ])
1336
1337 ;;
1338 ;; [vmulltq_int_u, vmulltq_int_s])
1339 ;;
1340 (define_insn "mve_vmulltq_int_<supf><mode>"
1341   [
1342    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1343         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1344                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1345          VMULLTQ_INT))
1346   ]
1347   "TARGET_HAVE_MVE"
1348   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1349   [(set_attr "type" "mve_move")
1350 ])
1351
1352 ;;
1353 ;; [vmulq_n_u, vmulq_n_s])
1354 ;;
1355 (define_insn "mve_vmulq_n_<supf><mode>"
1356   [
1357    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1358         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1359                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1360          VMULQ_N))
1361   ]
1362   "TARGET_HAVE_MVE"
1363   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1364   [(set_attr "type" "mve_move")
1365 ])
1366
1367 ;;
1368 ;; [vmulq_u, vmulq_s])
1369 ;;
1370 (define_insn "mve_vmulq_<supf><mode>"
1371   [
1372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1374                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1375          VMULQ))
1376   ]
1377   "TARGET_HAVE_MVE"
1378   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1379   [(set_attr "type" "mve_move")
1380 ])
1381
1382 (define_insn "mve_vmulq<mode>"
1383   [
1384    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1385         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1386                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1387   ]
1388   "TARGET_HAVE_MVE"
1389   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1390   [(set_attr "type" "mve_move")
1391 ])
1392
1393 ;;
1394 ;; [vornq_u, vornq_s])
1395 ;;
1396 (define_insn "mve_vornq_s<mode>"
1397   [
1398    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1399         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1400                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1401   ]
1402   "TARGET_HAVE_MVE"
1403    "vorn\t%q0, %q1, %q2"
1404   [(set_attr "type" "mve_move")
1405 ])
1406
1407 (define_expand "mve_vornq_u<mode>"
1408   [
1409    (set (match_operand:MVE_2 0 "s_register_operand")
1410         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1411                    (match_operand:MVE_2 1 "s_register_operand")))
1412   ]
1413   "TARGET_HAVE_MVE"
1414 )
1415
1416 ;;
1417 ;; [vorrq_s, vorrq_u])
1418 ;;
1419 ;; signed and unsigned versions are the same: define the unsigned
1420 ;; insn, and use an expander for the signed one as we still reference
1421 ;; both names from arm_mve.h.
1422 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1423 (define_insn "mve_vorrq_s<mode>"
1424   [
1425    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1426         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1427                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1428   ]
1429   "TARGET_HAVE_MVE"
1430   "@
1431    vorr\t%q0, %q1, %q2
1432    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1433   [(set_attr "type" "mve_move")
1434 ])
1435 (define_expand "mve_vorrq_u<mode>"
1436   [
1437    (set (match_operand:MVE_2 0 "s_register_operand")
1438         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1439                    (match_operand:MVE_2 2 "neon_logic_op2")))
1440   ]
1441   "TARGET_HAVE_MVE"
1442 )
1443
1444 ;;
1445 ;; [vqaddq_n_s, vqaddq_n_u])
1446 ;;
1447 (define_insn "mve_vqaddq_n_<supf><mode>"
1448   [
1449    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1450         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1451                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1452          VQADDQ_N))
1453   ]
1454   "TARGET_HAVE_MVE"
1455   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1456   [(set_attr "type" "mve_move")
1457 ])
1458
1459 ;;
1460 ;; [vqaddq_u, vqaddq_s])
1461 ;;
1462 (define_insn "mve_vqaddq_<supf><mode>"
1463   [
1464    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1465         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1466                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1467          VQADDQ))
1468   ]
1469   "TARGET_HAVE_MVE"
1470   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1471   [(set_attr "type" "mve_move")
1472 ])
1473
1474 ;;
1475 ;; [vqdmulhq_n_s])
1476 ;;
1477 (define_insn "mve_vqdmulhq_n_s<mode>"
1478   [
1479    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1480         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1481                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1482          VQDMULHQ_N_S))
1483   ]
1484   "TARGET_HAVE_MVE"
1485   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1486   [(set_attr "type" "mve_move")
1487 ])
1488
1489 ;;
1490 ;; [vqdmulhq_s])
1491 ;;
1492 (define_insn "mve_vqdmulhq_s<mode>"
1493   [
1494    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1495         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1496                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1497          VQDMULHQ_S))
1498   ]
1499   "TARGET_HAVE_MVE"
1500   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1501   [(set_attr "type" "mve_move")
1502 ])
1503
1504 ;;
1505 ;; [vqrdmulhq_n_s])
1506 ;;
1507 (define_insn "mve_vqrdmulhq_n_s<mode>"
1508   [
1509    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1510         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1511                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1512          VQRDMULHQ_N_S))
1513   ]
1514   "TARGET_HAVE_MVE"
1515   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1516   [(set_attr "type" "mve_move")
1517 ])
1518
1519 ;;
1520 ;; [vqrdmulhq_s])
1521 ;;
1522 (define_insn "mve_vqrdmulhq_s<mode>"
1523   [
1524    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1525         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1526                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1527          VQRDMULHQ_S))
1528   ]
1529   "TARGET_HAVE_MVE"
1530   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1531   [(set_attr "type" "mve_move")
1532 ])
1533
1534 ;;
1535 ;; [vqrshlq_n_s, vqrshlq_n_u])
1536 ;;
1537 (define_insn "mve_vqrshlq_n_<supf><mode>"
1538   [
1539    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1540         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1541                        (match_operand:SI 2 "s_register_operand" "r")]
1542          VQRSHLQ_N))
1543   ]
1544   "TARGET_HAVE_MVE"
1545   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1546   [(set_attr "type" "mve_move")
1547 ])
1548
1549 ;;
1550 ;; [vqrshlq_s, vqrshlq_u])
1551 ;;
1552 (define_insn "mve_vqrshlq_<supf><mode>"
1553   [
1554    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1555         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1556                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1557          VQRSHLQ))
1558   ]
1559   "TARGET_HAVE_MVE"
1560   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1561   [(set_attr "type" "mve_move")
1562 ])
1563
1564 ;;
1565 ;; [vqshlq_n_s, vqshlq_n_u])
1566 ;;
1567 (define_insn "mve_vqshlq_n_<supf><mode>"
1568   [
1569    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1570         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1571                        (match_operand:SI 2 "immediate_operand" "i")]
1572          VQSHLQ_N))
1573   ]
1574   "TARGET_HAVE_MVE"
1575   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1576   [(set_attr "type" "mve_move")
1577 ])
1578
1579 ;;
1580 ;; [vqshlq_r_u, vqshlq_r_s])
1581 ;;
1582 (define_insn "mve_vqshlq_r_<supf><mode>"
1583   [
1584    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1585         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1586                        (match_operand:SI 2 "s_register_operand" "r")]
1587          VQSHLQ_R))
1588   ]
1589   "TARGET_HAVE_MVE"
1590   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1591   [(set_attr "type" "mve_move")
1592 ])
1593
1594 ;;
1595 ;; [vqshlq_s, vqshlq_u])
1596 ;;
1597 (define_insn "mve_vqshlq_<supf><mode>"
1598   [
1599    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1600         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1601                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1602          VQSHLQ))
1603   ]
1604   "TARGET_HAVE_MVE"
1605   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1606   [(set_attr "type" "mve_move")
1607 ])
1608
1609 ;;
1610 ;; [vqshluq_n_s])
1611 ;;
1612 (define_insn "mve_vqshluq_n_s<mode>"
1613   [
1614    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1615         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1616                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1617          VQSHLUQ_N_S))
1618   ]
1619   "TARGET_HAVE_MVE"
1620   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1621   [(set_attr "type" "mve_move")
1622 ])
1623
1624 ;;
1625 ;; [vqsubq_n_s, vqsubq_n_u])
1626 ;;
1627 (define_insn "mve_vqsubq_n_<supf><mode>"
1628   [
1629    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1630         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1631                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1632          VQSUBQ_N))
1633   ]
1634   "TARGET_HAVE_MVE"
1635   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1636   [(set_attr "type" "mve_move")
1637 ])
1638
1639 ;;
1640 ;; [vqsubq_u, vqsubq_s])
1641 ;;
1642 (define_insn "mve_vqsubq_<supf><mode>"
1643   [
1644    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1645         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1646                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1647          VQSUBQ))
1648   ]
1649   "TARGET_HAVE_MVE"
1650   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1651   [(set_attr "type" "mve_move")
1652 ])
1653
1654 ;;
1655 ;; [vrhaddq_s, vrhaddq_u])
1656 ;;
1657 (define_insn "mve_vrhaddq_<supf><mode>"
1658   [
1659    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1660         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1661                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1662          VRHADDQ))
1663   ]
1664   "TARGET_HAVE_MVE"
1665   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1666   [(set_attr "type" "mve_move")
1667 ])
1668
1669 ;;
1670 ;; [vrmulhq_s, vrmulhq_u])
1671 ;;
1672 (define_insn "mve_vrmulhq_<supf><mode>"
1673   [
1674    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1675         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1676                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1677          VRMULHQ))
1678   ]
1679   "TARGET_HAVE_MVE"
1680   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1681   [(set_attr "type" "mve_move")
1682 ])
1683
1684 ;;
1685 ;; [vrshlq_n_u, vrshlq_n_s])
1686 ;;
1687 (define_insn "mve_vrshlq_n_<supf><mode>"
1688   [
1689    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1690         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1691                        (match_operand:SI 2 "s_register_operand" "r")]
1692          VRSHLQ_N))
1693   ]
1694   "TARGET_HAVE_MVE"
1695   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1696   [(set_attr "type" "mve_move")
1697 ])
1698
1699 ;;
1700 ;; [vrshlq_s, vrshlq_u])
1701 ;;
1702 (define_insn "mve_vrshlq_<supf><mode>"
1703   [
1704    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1705         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1706                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1707          VRSHLQ))
1708   ]
1709   "TARGET_HAVE_MVE"
1710   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1711   [(set_attr "type" "mve_move")
1712 ])
1713
1714 ;;
1715 ;; [vrshrq_n_s, vrshrq_n_u])
1716 ;;
1717 (define_insn "mve_vrshrq_n_<supf><mode>"
1718   [
1719    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1720         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1721                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1722          VRSHRQ_N))
1723   ]
1724   "TARGET_HAVE_MVE"
1725   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1726   [(set_attr "type" "mve_move")
1727 ])
1728
1729 ;;
1730 ;; [vshlq_n_u, vshlq_n_s])
1731 ;;
1732 (define_insn "mve_vshlq_n_<supf><mode>"
1733   [
1734    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1735         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1736                        (match_operand:SI 2 "immediate_operand" "i")]
1737          VSHLQ_N))
1738   ]
1739   "TARGET_HAVE_MVE"
1740   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1741   [(set_attr "type" "mve_move")
1742 ])
1743
1744 ;;
1745 ;; [vshlq_r_s, vshlq_r_u])
1746 ;;
1747 (define_insn "mve_vshlq_r_<supf><mode>"
1748   [
1749    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1750         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1751                        (match_operand:SI 2 "s_register_operand" "r")]
1752          VSHLQ_R))
1753   ]
1754   "TARGET_HAVE_MVE"
1755   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1756   [(set_attr "type" "mve_move")
1757 ])
1758
1759 ;;
1760 ;; [vsubq_n_s, vsubq_n_u])
1761 ;;
1762 (define_insn "mve_vsubq_n_<supf><mode>"
1763   [
1764    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1765         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1766                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1767          VSUBQ_N))
1768   ]
1769   "TARGET_HAVE_MVE"
1770   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1771   [(set_attr "type" "mve_move")
1772 ])
1773
1774 ;;
1775 ;; [vsubq_s, vsubq_u])
1776 ;;
1777 (define_insn "mve_vsubq_<supf><mode>"
1778   [
1779    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1780         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1781                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1782          VSUBQ))
1783   ]
1784   "TARGET_HAVE_MVE"
1785   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1786   [(set_attr "type" "mve_move")
1787 ])
1788
1789 (define_insn "mve_vsubq<mode>"
1790   [
1791    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1792         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1793                      (match_operand:MVE_2 2 "s_register_operand" "w")))
1794   ]
1795   "TARGET_HAVE_MVE"
1796   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1797   [(set_attr "type" "mve_move")
1798 ])
1799
1800 ;;
1801 ;; [vabdq_f])
1802 ;;
1803 (define_insn "mve_vabdq_f<mode>"
1804   [
1805    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1806         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1807                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1808          VABDQ_F))
1809   ]
1810   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1811   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
1812   [(set_attr "type" "mve_move")
1813 ])
1814
1815 ;;
1816 ;; [vaddlvaq_s vaddlvaq_u])
1817 ;;
1818 (define_insn "mve_vaddlvaq_<supf>v4si"
1819   [
1820    (set (match_operand:DI 0 "s_register_operand" "=r")
1821         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1822                     (match_operand:V4SI 2 "s_register_operand" "w")]
1823          VADDLVAQ))
1824   ]
1825   "TARGET_HAVE_MVE"
1826   "vaddlva.<supf>32 %Q0, %R0, %q2"
1827   [(set_attr "type" "mve_move")
1828 ])
1829
1830 ;;
1831 ;; [vaddq_n_f])
1832 ;;
1833 (define_insn "mve_vaddq_n_f<mode>"
1834   [
1835    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1836         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1837                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1838          VADDQ_N_F))
1839   ]
1840   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1841   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
1842   [(set_attr "type" "mve_move")
1843 ])
1844
1845 ;;
1846 ;; [vandq_f])
1847 ;;
1848 (define_insn "mve_vandq_f<mode>"
1849   [
1850    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1851         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1852                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1853   ]
1854   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1855   "vand %q0, %q1, %q2"
1856   [(set_attr "type" "mve_move")
1857 ])
1858
1859 ;;
1860 ;; [vbicq_f])
1861 ;;
1862 (define_insn "mve_vbicq_f<mode>"
1863   [
1864    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1865         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
1866                               (match_operand:MVE_0 2 "s_register_operand" "w")))
1867   ]
1868   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1869   "vbic %q0, %q1, %q2"
1870   [(set_attr "type" "mve_move")
1871 ])
1872
1873 ;;
1874 ;; [vbicq_n_s, vbicq_n_u])
1875 ;;
1876 (define_insn "mve_vbicq_n_<supf><mode>"
1877   [
1878    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1879         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1880                        (match_operand:SI 2 "immediate_operand" "i")]
1881          VBICQ_N))
1882   ]
1883   "TARGET_HAVE_MVE"
1884   "vbic.i%#<V_sz_elem>  %q0, %2"
1885   [(set_attr "type" "mve_move")
1886 ])
1887
1888 ;;
1889 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
1890 ;;
1891 (define_insn "mve_vcaddq<mve_rot><mode>"
1892   [
1893    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1894         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1895                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1896          VCADD))
1897   ]
1898   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1899   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1900   [(set_attr "type" "mve_move")
1901 ])
1902
1903 ;;
1904 ;; [vcmpeqq_f, vcmpgeq_f, vcmpgtq_f, vcmpleq_f, vcmpltq_f, vcmpneq_f])
1905 ;;
1906 (define_insn "mve_vcmp<mve_cmp_op>q_f<mode>"
1907   [
1908    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1909         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1910                                (match_operand:MVE_0 2 "s_register_operand" "w")))
1911   ]
1912   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1913   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
1914   [(set_attr "type" "mve_move")
1915 ])
1916
1917 ;;
1918 ;; [vcmpeqq_n_f, vcmpgeq_n_f, vcmpgtq_n_f, vcmpleq_n_f, vcmpltq_n_f, vcmpneq_n_f])
1919 ;;
1920 (define_insn "mve_vcmp<mve_cmp_op>q_n_f<mode>"
1921   [
1922    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1923         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1924                                (match_operand:<V_elem> 2 "s_register_operand" "r")))
1925   ]
1926   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1927   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %2"
1928   [(set_attr "type" "mve_move")
1929 ])
1930
1931 ;;
1932 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
1933 ;;
1934 (define_insn "mve_vcmulq<mve_rot><mode>"
1935   [
1936    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1937         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1938                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1939          VCMUL))
1940   ]
1941   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1942   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1943   [(set_attr "type" "mve_move")
1944 ])
1945
1946 ;;
1947 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
1948 ;;
1949 (define_insn "mve_vctp<mode1>q_mhi"
1950   [
1951    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1952         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
1953                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1954          VCTPQ_M))
1955   ]
1956   "TARGET_HAVE_MVE"
1957   "vpst\;vctpt.<mode1> %1"
1958   [(set_attr "type" "mve_move")
1959    (set_attr "length""8")])
1960
1961 ;;
1962 ;; [vcvtbq_f16_f32])
1963 ;;
1964 (define_insn "mve_vcvtbq_f16_f32v8hf"
1965   [
1966    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1967         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1968                       (match_operand:V4SF 2 "s_register_operand" "w")]
1969          VCVTBQ_F16_F32))
1970   ]
1971   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1972   "vcvtb.f16.f32 %q0, %q2"
1973   [(set_attr "type" "mve_move")
1974 ])
1975
1976 ;;
1977 ;; [vcvttq_f16_f32])
1978 ;;
1979 (define_insn "mve_vcvttq_f16_f32v8hf"
1980   [
1981    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1982         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1983                       (match_operand:V4SF 2 "s_register_operand" "w")]
1984          VCVTTQ_F16_F32))
1985   ]
1986   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1987   "vcvtt.f16.f32 %q0, %q2"
1988   [(set_attr "type" "mve_move")
1989 ])
1990
1991 ;;
1992 ;; [veorq_f])
1993 ;;
1994 (define_insn "mve_veorq_f<mode>"
1995   [
1996    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1997         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1998                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1999   ]
2000   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2001   "veor %q0, %q1, %q2"
2002   [(set_attr "type" "mve_move")
2003 ])
2004
2005 ;;
2006 ;; [vmaxnmaq_f])
2007 ;;
2008 (define_insn "mve_vmaxnmaq_f<mode>"
2009   [
2010    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2011         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2012                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2013          VMAXNMAQ_F))
2014   ]
2015   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2016   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2017   [(set_attr "type" "mve_move")
2018 ])
2019
2020 ;;
2021 ;; [vmaxnmavq_f])
2022 ;;
2023 (define_insn "mve_vmaxnmavq_f<mode>"
2024   [
2025    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2026         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2027                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2028          VMAXNMAVQ_F))
2029   ]
2030   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2031   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2032   [(set_attr "type" "mve_move")
2033 ])
2034
2035 ;;
2036 ;; [vmaxnmq_f])
2037 ;;
2038 (define_insn "mve_vmaxnmq_f<mode>"
2039   [
2040    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2041         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2042                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2043   ]
2044   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2045   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2046   [(set_attr "type" "mve_move")
2047 ])
2048
2049 ;;
2050 ;; [vmaxnmvq_f])
2051 ;;
2052 (define_insn "mve_vmaxnmvq_f<mode>"
2053   [
2054    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2055         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2056                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2057          VMAXNMVQ_F))
2058   ]
2059   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2060   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2061   [(set_attr "type" "mve_move")
2062 ])
2063
2064 ;;
2065 ;; [vminnmaq_f])
2066 ;;
2067 (define_insn "mve_vminnmaq_f<mode>"
2068   [
2069    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2070         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2071                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2072          VMINNMAQ_F))
2073   ]
2074   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2075   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2076   [(set_attr "type" "mve_move")
2077 ])
2078
2079 ;;
2080 ;; [vminnmavq_f])
2081 ;;
2082 (define_insn "mve_vminnmavq_f<mode>"
2083   [
2084    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2085         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2086                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2087          VMINNMAVQ_F))
2088   ]
2089   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2090   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2091   [(set_attr "type" "mve_move")
2092 ])
2093
2094 ;;
2095 ;; [vminnmq_f])
2096 ;;
2097 (define_insn "mve_vminnmq_f<mode>"
2098   [
2099    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2100         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2101                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2102   ]
2103   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2104   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2105   [(set_attr "type" "mve_move")
2106 ])
2107
2108 ;;
2109 ;; [vminnmvq_f])
2110 ;;
2111 (define_insn "mve_vminnmvq_f<mode>"
2112   [
2113    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2114         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2115                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2116          VMINNMVQ_F))
2117   ]
2118   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2119   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2120   [(set_attr "type" "mve_move")
2121 ])
2122
2123 ;;
2124 ;; [vmlaldavq_u, vmlaldavq_s])
2125 ;;
2126 (define_insn "mve_vmlaldavq_<supf><mode>"
2127   [
2128    (set (match_operand:DI 0 "s_register_operand" "=r")
2129         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2130                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2131          VMLALDAVQ))
2132   ]
2133   "TARGET_HAVE_MVE"
2134   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2135   [(set_attr "type" "mve_move")
2136 ])
2137
2138 ;;
2139 ;; [vmlaldavxq_s])
2140 ;;
2141 (define_insn "mve_vmlaldavxq_s<mode>"
2142   [
2143    (set (match_operand:DI 0 "s_register_operand" "=r")
2144         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2145                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2146          VMLALDAVXQ_S))
2147   ]
2148   "TARGET_HAVE_MVE"
2149   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2150   [(set_attr "type" "mve_move")
2151 ])
2152
2153 ;;
2154 ;; [vmlsldavq_s])
2155 ;;
2156 (define_insn "mve_vmlsldavq_s<mode>"
2157   [
2158    (set (match_operand:DI 0 "s_register_operand" "=r")
2159         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2160                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2161          VMLSLDAVQ_S))
2162   ]
2163   "TARGET_HAVE_MVE"
2164   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2165   [(set_attr "type" "mve_move")
2166 ])
2167
2168 ;;
2169 ;; [vmlsldavxq_s])
2170 ;;
2171 (define_insn "mve_vmlsldavxq_s<mode>"
2172   [
2173    (set (match_operand:DI 0 "s_register_operand" "=r")
2174         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2175                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2176          VMLSLDAVXQ_S))
2177   ]
2178   "TARGET_HAVE_MVE"
2179   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2180   [(set_attr "type" "mve_move")
2181 ])
2182
2183 ;;
2184 ;; [vmovnbq_u, vmovnbq_s])
2185 ;;
2186 (define_insn "mve_vmovnbq_<supf><mode>"
2187   [
2188    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2189         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2190                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2191          VMOVNBQ))
2192   ]
2193   "TARGET_HAVE_MVE"
2194   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2195   [(set_attr "type" "mve_move")
2196 ])
2197
2198 ;;
2199 ;; [vmovntq_s, vmovntq_u])
2200 ;;
2201 (define_insn "mve_vmovntq_<supf><mode>"
2202   [
2203    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2204         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2205                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2206          VMOVNTQ))
2207   ]
2208   "TARGET_HAVE_MVE"
2209   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2210   [(set_attr "type" "mve_move")
2211 ])
2212
2213 ;;
2214 ;; [vmulq_f])
2215 ;;
2216 (define_insn "mve_vmulq_f<mode>"
2217   [
2218    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2219         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2220                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2221   ]
2222   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2223   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2224   [(set_attr "type" "mve_move")
2225 ])
2226
2227 ;;
2228 ;; [vmulq_n_f])
2229 ;;
2230 (define_insn "mve_vmulq_n_f<mode>"
2231   [
2232    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2233         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2234                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2235          VMULQ_N_F))
2236   ]
2237   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2238   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2239   [(set_attr "type" "mve_move")
2240 ])
2241
2242 ;;
2243 ;; [vornq_f])
2244 ;;
2245 (define_insn "mve_vornq_f<mode>"
2246   [
2247    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2248         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
2249                    (match_operand:MVE_0 1 "s_register_operand" "w")))
2250   ]
2251   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2252   "vorn %q0, %q1, %q2"
2253   [(set_attr "type" "mve_move")
2254 ])
2255
2256 ;;
2257 ;; [vorrq_f])
2258 ;;
2259 (define_insn "mve_vorrq_f<mode>"
2260   [
2261    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2262         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2263                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2264   ]
2265   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2266   "vorr %q0, %q1, %q2"
2267   [(set_attr "type" "mve_move")
2268 ])
2269
2270 ;;
2271 ;; [vorrq_n_u, vorrq_n_s])
2272 ;;
2273 (define_insn "mve_vorrq_n_<supf><mode>"
2274   [
2275    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2276         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2277                        (match_operand:SI 2 "immediate_operand" "i")]
2278          VORRQ_N))
2279   ]
2280   "TARGET_HAVE_MVE"
2281   "vorr.i%#<V_sz_elem>  %q0, %2"
2282   [(set_attr "type" "mve_move")
2283 ])
2284
2285 ;;
2286 ;; [vqdmullbq_n_s])
2287 ;;
2288 (define_insn "mve_vqdmullbq_n_s<mode>"
2289   [
2290    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2291         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2292                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2293          VQDMULLBQ_N_S))
2294   ]
2295   "TARGET_HAVE_MVE"
2296   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2297   [(set_attr "type" "mve_move")
2298 ])
2299
2300 ;;
2301 ;; [vqdmullbq_s])
2302 ;;
2303 (define_insn "mve_vqdmullbq_s<mode>"
2304   [
2305    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2306         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2307                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2308          VQDMULLBQ_S))
2309   ]
2310   "TARGET_HAVE_MVE"
2311   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2312   [(set_attr "type" "mve_move")
2313 ])
2314
2315 ;;
2316 ;; [vqdmulltq_n_s])
2317 ;;
2318 (define_insn "mve_vqdmulltq_n_s<mode>"
2319   [
2320    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2321         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2322                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2323          VQDMULLTQ_N_S))
2324   ]
2325   "TARGET_HAVE_MVE"
2326   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2327   [(set_attr "type" "mve_move")
2328 ])
2329
2330 ;;
2331 ;; [vqdmulltq_s])
2332 ;;
2333 (define_insn "mve_vqdmulltq_s<mode>"
2334   [
2335    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2336         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2337                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2338          VQDMULLTQ_S))
2339   ]
2340   "TARGET_HAVE_MVE"
2341   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2342   [(set_attr "type" "mve_move")
2343 ])
2344
2345 ;;
2346 ;; [vqmovnbq_u, vqmovnbq_s])
2347 ;;
2348 (define_insn "mve_vqmovnbq_<supf><mode>"
2349   [
2350    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2351         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2352                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2353          VQMOVNBQ))
2354   ]
2355   "TARGET_HAVE_MVE"
2356   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2357   [(set_attr "type" "mve_move")
2358 ])
2359
2360 ;;
2361 ;; [vqmovntq_u, vqmovntq_s])
2362 ;;
2363 (define_insn "mve_vqmovntq_<supf><mode>"
2364   [
2365    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2366         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2367                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2368          VQMOVNTQ))
2369   ]
2370   "TARGET_HAVE_MVE"
2371   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2372   [(set_attr "type" "mve_move")
2373 ])
2374
2375 ;;
2376 ;; [vqmovunbq_s])
2377 ;;
2378 (define_insn "mve_vqmovunbq_s<mode>"
2379   [
2380    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2381         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2382                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2383          VQMOVUNBQ_S))
2384   ]
2385   "TARGET_HAVE_MVE"
2386   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2387   [(set_attr "type" "mve_move")
2388 ])
2389
2390 ;;
2391 ;; [vqmovuntq_s])
2392 ;;
2393 (define_insn "mve_vqmovuntq_s<mode>"
2394   [
2395    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2396         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2397                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2398          VQMOVUNTQ_S))
2399   ]
2400   "TARGET_HAVE_MVE"
2401   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2402   [(set_attr "type" "mve_move")
2403 ])
2404
2405 ;;
2406 ;; [vrmlaldavhxq_s])
2407 ;;
2408 (define_insn "mve_vrmlaldavhxq_sv4si"
2409   [
2410    (set (match_operand:DI 0 "s_register_operand" "=r")
2411         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2412                     (match_operand:V4SI 2 "s_register_operand" "w")]
2413          VRMLALDAVHXQ_S))
2414   ]
2415   "TARGET_HAVE_MVE"
2416   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2417   [(set_attr "type" "mve_move")
2418 ])
2419
2420 ;;
2421 ;; [vrmlsldavhq_s])
2422 ;;
2423 (define_insn "mve_vrmlsldavhq_sv4si"
2424   [
2425    (set (match_operand:DI 0 "s_register_operand" "=r")
2426         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2427                     (match_operand:V4SI 2 "s_register_operand" "w")]
2428          VRMLSLDAVHQ_S))
2429   ]
2430   "TARGET_HAVE_MVE"
2431   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2432   [(set_attr "type" "mve_move")
2433 ])
2434
2435 ;;
2436 ;; [vrmlsldavhxq_s])
2437 ;;
2438 (define_insn "mve_vrmlsldavhxq_sv4si"
2439   [
2440    (set (match_operand:DI 0 "s_register_operand" "=r")
2441         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2442                     (match_operand:V4SI 2 "s_register_operand" "w")]
2443          VRMLSLDAVHXQ_S))
2444   ]
2445   "TARGET_HAVE_MVE"
2446   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2447   [(set_attr "type" "mve_move")
2448 ])
2449
2450 ;;
2451 ;; [vshllbq_n_s, vshllbq_n_u])
2452 ;;
2453 (define_insn "mve_vshllbq_n_<supf><mode>"
2454   [
2455    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2456         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2457                                   (match_operand:SI 2 "immediate_operand" "i")]
2458          VSHLLBQ_N))
2459   ]
2460   "TARGET_HAVE_MVE"
2461   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2462   [(set_attr "type" "mve_move")
2463 ])
2464
2465 ;;
2466 ;; [vshlltq_n_u, vshlltq_n_s])
2467 ;;
2468 (define_insn "mve_vshlltq_n_<supf><mode>"
2469   [
2470    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2471         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2472                                   (match_operand:SI 2 "immediate_operand" "i")]
2473          VSHLLTQ_N))
2474   ]
2475   "TARGET_HAVE_MVE"
2476   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2477   [(set_attr "type" "mve_move")
2478 ])
2479
2480 ;;
2481 ;; [vsubq_f])
2482 ;;
2483 (define_insn "mve_vsubq_f<mode>"
2484   [
2485    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2486         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2487                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2488   ]
2489   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2490   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2491   [(set_attr "type" "mve_move")
2492 ])
2493
2494 ;;
2495 ;; [vmulltq_poly_p])
2496 ;;
2497 (define_insn "mve_vmulltq_poly_p<mode>"
2498   [
2499    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2500         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2501                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2502          VMULLTQ_POLY_P))
2503   ]
2504   "TARGET_HAVE_MVE"
2505   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2506   [(set_attr "type" "mve_move")
2507 ])
2508
2509 ;;
2510 ;; [vmullbq_poly_p])
2511 ;;
2512 (define_insn "mve_vmullbq_poly_p<mode>"
2513   [
2514    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2515         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2516                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2517          VMULLBQ_POLY_P))
2518   ]
2519   "TARGET_HAVE_MVE"
2520   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2521   [(set_attr "type" "mve_move")
2522 ])
2523
2524 ;;
2525 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2526 ;;
2527 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2528   [
2529    (set (match_operand:DI 0 "s_register_operand" "=r")
2530         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2531                     (match_operand:V4SI 2 "s_register_operand" "w")]
2532          VRMLALDAVHQ))
2533   ]
2534   "TARGET_HAVE_MVE"
2535   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2536   [(set_attr "type" "mve_move")
2537 ])
2538
2539 ;;
2540 ;; [vbicq_m_n_s, vbicq_m_n_u])
2541 ;;
2542 (define_insn "mve_vbicq_m_n_<supf><mode>"
2543   [
2544    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2545         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2546                        (match_operand:SI 2 "immediate_operand" "i")
2547                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2548          VBICQ_M_N))
2549   ]
2550   "TARGET_HAVE_MVE"
2551   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2552   [(set_attr "type" "mve_move")
2553    (set_attr "length""8")])
2554 ;;
2555 ;; [vcmpeqq_m_f])
2556 ;;
2557 (define_insn "mve_vcmpeqq_m_f<mode>"
2558   [
2559    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2560         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2561                     (match_operand:MVE_0 2 "s_register_operand" "w")
2562                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2563          VCMPEQQ_M_F))
2564   ]
2565   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2566   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2567   [(set_attr "type" "mve_move")
2568    (set_attr "length""8")])
2569 ;;
2570 ;; [vcvtaq_m_u, vcvtaq_m_s])
2571 ;;
2572 (define_insn "mve_vcvtaq_m_<supf><mode>"
2573   [
2574    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2575         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2576                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2577                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2578          VCVTAQ_M))
2579   ]
2580   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2581   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2582   [(set_attr "type" "mve_move")
2583    (set_attr "length""8")])
2584 ;;
2585 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2586 ;;
2587 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2588   [
2589    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2590         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2591                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2592                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2593          VCVTQ_M_TO_F))
2594   ]
2595   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2596   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2597   [(set_attr "type" "mve_move")
2598    (set_attr "length""8")])
2599 ;;
2600 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2601 ;;
2602 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2603   [
2604    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2605         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2606                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2607                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2608          VQRSHRNBQ_N))
2609   ]
2610   "TARGET_HAVE_MVE"
2611   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2612   [(set_attr "type" "mve_move")
2613 ])
2614 ;;
2615 ;; [vqrshrunbq_n_s])
2616 ;;
2617 (define_insn "mve_vqrshrunbq_n_s<mode>"
2618   [
2619    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2620         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2621                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2622                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2623          VQRSHRUNBQ_N_S))
2624   ]
2625   "TARGET_HAVE_MVE"
2626   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
2627   [(set_attr "type" "mve_move")
2628 ])
2629 ;;
2630 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
2631 ;;
2632 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
2633   [
2634    (set (match_operand:DI 0 "s_register_operand" "=r")
2635         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2636                     (match_operand:V4SI 2 "s_register_operand" "w")
2637                     (match_operand:V4SI 3 "s_register_operand" "w")]
2638          VRMLALDAVHAQ))
2639   ]
2640   "TARGET_HAVE_MVE"
2641   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
2642   [(set_attr "type" "mve_move")
2643 ])
2644
2645 ;;
2646 ;; [vabavq_s, vabavq_u])
2647 ;;
2648 (define_insn "mve_vabavq_<supf><mode>"
2649   [
2650    (set (match_operand:SI 0 "s_register_operand" "=r")
2651         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2652                     (match_operand:MVE_2 2 "s_register_operand" "w")
2653                     (match_operand:MVE_2 3 "s_register_operand" "w")]
2654          VABAVQ))
2655   ]
2656   "TARGET_HAVE_MVE"
2657   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2658   [(set_attr "type" "mve_move")
2659 ])
2660
2661 ;;
2662 ;; [vshlcq_u vshlcq_s]
2663 ;;
2664 (define_expand "mve_vshlcq_vec_<supf><mode>"
2665  [(match_operand:MVE_2 0 "s_register_operand")
2666   (match_operand:MVE_2 1 "s_register_operand")
2667   (match_operand:SI 2 "s_register_operand")
2668   (match_operand:SI 3 "mve_imm_32")
2669   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2670  "TARGET_HAVE_MVE"
2671 {
2672   rtx ignore_wb = gen_reg_rtx (SImode);
2673   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
2674                                       operands[2], operands[3]));
2675   DONE;
2676 })
2677
2678 (define_expand "mve_vshlcq_carry_<supf><mode>"
2679  [(match_operand:SI 0 "s_register_operand")
2680   (match_operand:MVE_2 1 "s_register_operand")
2681   (match_operand:SI 2 "s_register_operand")
2682   (match_operand:SI 3 "mve_imm_32")
2683   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2684  "TARGET_HAVE_MVE"
2685 {
2686   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
2687   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
2688                                       operands[2], operands[3]));
2689   DONE;
2690 })
2691
2692 (define_insn "mve_vshlcq_<supf><mode>"
2693  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
2694        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
2695                       (match_operand:SI 3 "s_register_operand" "1")
2696                       (match_operand:SI 4 "mve_imm_32" "Rf")]
2697         VSHLCQ))
2698   (set (match_operand:SI  1 "s_register_operand" "=r")
2699        (unspec:SI [(match_dup 2)
2700                    (match_dup 3)
2701                    (match_dup 4)]
2702         VSHLCQ))]
2703  "TARGET_HAVE_MVE"
2704  "vshlc %q0, %1, %4")
2705
2706 ;;
2707 ;; [vabsq_m_s])
2708 ;;
2709 (define_insn "mve_vabsq_m_s<mode>"
2710   [
2711    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2712         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2713                        (match_operand:MVE_2 2 "s_register_operand" "w")
2714                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2715          VABSQ_M_S))
2716   ]
2717   "TARGET_HAVE_MVE"
2718   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
2719   [(set_attr "type" "mve_move")
2720    (set_attr "length""8")])
2721
2722 ;;
2723 ;; [vaddvaq_p_u, vaddvaq_p_s])
2724 ;;
2725 (define_insn "mve_vaddvaq_p_<supf><mode>"
2726   [
2727    (set (match_operand:SI 0 "s_register_operand" "=Te")
2728         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2729                        (match_operand:MVE_2 2 "s_register_operand" "w")
2730                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2731          VADDVAQ_P))
2732   ]
2733   "TARGET_HAVE_MVE"
2734   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
2735   [(set_attr "type" "mve_move")
2736    (set_attr "length""8")])
2737
2738 ;;
2739 ;; [vclsq_m_s])
2740 ;;
2741 (define_insn "mve_vclsq_m_s<mode>"
2742   [
2743    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2744         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2745                        (match_operand:MVE_2 2 "s_register_operand" "w")
2746                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2747          VCLSQ_M_S))
2748   ]
2749   "TARGET_HAVE_MVE"
2750   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
2751   [(set_attr "type" "mve_move")
2752    (set_attr "length""8")])
2753
2754 ;;
2755 ;; [vclzq_m_s, vclzq_m_u])
2756 ;;
2757 (define_insn "mve_vclzq_m_<supf><mode>"
2758   [
2759    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2760         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2761                        (match_operand:MVE_2 2 "s_register_operand" "w")
2762                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2763          VCLZQ_M))
2764   ]
2765   "TARGET_HAVE_MVE"
2766   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
2767   [(set_attr "type" "mve_move")
2768    (set_attr "length""8")])
2769
2770 ;;
2771 ;; [vcmpcsq_m_n_u])
2772 ;;
2773 (define_insn "mve_vcmpcsq_m_n_u<mode>"
2774   [
2775    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2776         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2777                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2778                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2779          VCMPCSQ_M_N_U))
2780   ]
2781   "TARGET_HAVE_MVE"
2782   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
2783   [(set_attr "type" "mve_move")
2784    (set_attr "length""8")])
2785
2786 ;;
2787 ;; [vcmpcsq_m_u])
2788 ;;
2789 (define_insn "mve_vcmpcsq_m_u<mode>"
2790   [
2791    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2792         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2793                        (match_operand:MVE_2 2 "s_register_operand" "w")
2794                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2795          VCMPCSQ_M_U))
2796   ]
2797   "TARGET_HAVE_MVE"
2798   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
2799   [(set_attr "type" "mve_move")
2800    (set_attr "length""8")])
2801
2802 ;;
2803 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
2804 ;;
2805 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
2806   [
2807    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2808         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2809                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2810                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2811          VCMPEQQ_M_N))
2812   ]
2813   "TARGET_HAVE_MVE"
2814   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
2815   [(set_attr "type" "mve_move")
2816    (set_attr "length""8")])
2817
2818 ;;
2819 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
2820 ;;
2821 (define_insn "mve_vcmpeqq_m_<supf><mode>"
2822   [
2823    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2824         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2825                        (match_operand:MVE_2 2 "s_register_operand" "w")
2826                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2827          VCMPEQQ_M))
2828   ]
2829   "TARGET_HAVE_MVE"
2830   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
2831   [(set_attr "type" "mve_move")
2832    (set_attr "length""8")])
2833
2834 ;;
2835 ;; [vcmpgeq_m_n_s])
2836 ;;
2837 (define_insn "mve_vcmpgeq_m_n_s<mode>"
2838   [
2839    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2840         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2841                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2842                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2843          VCMPGEQ_M_N_S))
2844   ]
2845   "TARGET_HAVE_MVE"
2846   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
2847   [(set_attr "type" "mve_move")
2848    (set_attr "length""8")])
2849
2850 ;;
2851 ;; [vcmpgeq_m_s])
2852 ;;
2853 (define_insn "mve_vcmpgeq_m_s<mode>"
2854   [
2855    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2856         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2857                        (match_operand:MVE_2 2 "s_register_operand" "w")
2858                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2859          VCMPGEQ_M_S))
2860   ]
2861   "TARGET_HAVE_MVE"
2862   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
2863   [(set_attr "type" "mve_move")
2864    (set_attr "length""8")])
2865
2866 ;;
2867 ;; [vcmpgtq_m_n_s])
2868 ;;
2869 (define_insn "mve_vcmpgtq_m_n_s<mode>"
2870   [
2871    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2872         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2873                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2874                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2875          VCMPGTQ_M_N_S))
2876   ]
2877   "TARGET_HAVE_MVE"
2878   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
2879   [(set_attr "type" "mve_move")
2880    (set_attr "length""8")])
2881
2882 ;;
2883 ;; [vcmpgtq_m_s])
2884 ;;
2885 (define_insn "mve_vcmpgtq_m_s<mode>"
2886   [
2887    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2888         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2889                        (match_operand:MVE_2 2 "s_register_operand" "w")
2890                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2891          VCMPGTQ_M_S))
2892   ]
2893   "TARGET_HAVE_MVE"
2894   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
2895   [(set_attr "type" "mve_move")
2896    (set_attr "length""8")])
2897
2898 ;;
2899 ;; [vcmphiq_m_n_u])
2900 ;;
2901 (define_insn "mve_vcmphiq_m_n_u<mode>"
2902   [
2903    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2904         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2905                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2906                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2907          VCMPHIQ_M_N_U))
2908   ]
2909   "TARGET_HAVE_MVE"
2910   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
2911   [(set_attr "type" "mve_move")
2912    (set_attr "length""8")])
2913
2914 ;;
2915 ;; [vcmphiq_m_u])
2916 ;;
2917 (define_insn "mve_vcmphiq_m_u<mode>"
2918   [
2919    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2920         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2921                        (match_operand:MVE_2 2 "s_register_operand" "w")
2922                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2923          VCMPHIQ_M_U))
2924   ]
2925   "TARGET_HAVE_MVE"
2926   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
2927   [(set_attr "type" "mve_move")
2928    (set_attr "length""8")])
2929
2930 ;;
2931 ;; [vcmpleq_m_n_s])
2932 ;;
2933 (define_insn "mve_vcmpleq_m_n_s<mode>"
2934   [
2935    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2936         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2937                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2938                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2939          VCMPLEQ_M_N_S))
2940   ]
2941   "TARGET_HAVE_MVE"
2942   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
2943   [(set_attr "type" "mve_move")
2944    (set_attr "length""8")])
2945
2946 ;;
2947 ;; [vcmpleq_m_s])
2948 ;;
2949 (define_insn "mve_vcmpleq_m_s<mode>"
2950   [
2951    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2952         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2953                        (match_operand:MVE_2 2 "s_register_operand" "w")
2954                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2955          VCMPLEQ_M_S))
2956   ]
2957   "TARGET_HAVE_MVE"
2958   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
2959   [(set_attr "type" "mve_move")
2960    (set_attr "length""8")])
2961
2962 ;;
2963 ;; [vcmpltq_m_n_s])
2964 ;;
2965 (define_insn "mve_vcmpltq_m_n_s<mode>"
2966   [
2967    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2968         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2969                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2970                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2971          VCMPLTQ_M_N_S))
2972   ]
2973   "TARGET_HAVE_MVE"
2974   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
2975   [(set_attr "type" "mve_move")
2976    (set_attr "length""8")])
2977
2978 ;;
2979 ;; [vcmpltq_m_s])
2980 ;;
2981 (define_insn "mve_vcmpltq_m_s<mode>"
2982   [
2983    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2984         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2985                        (match_operand:MVE_2 2 "s_register_operand" "w")
2986                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2987          VCMPLTQ_M_S))
2988   ]
2989   "TARGET_HAVE_MVE"
2990   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
2991   [(set_attr "type" "mve_move")
2992    (set_attr "length""8")])
2993
2994 ;;
2995 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
2996 ;;
2997 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
2998   [
2999    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3000         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3001                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3002                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3003          VCMPNEQ_M_N))
3004   ]
3005   "TARGET_HAVE_MVE"
3006   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3007   [(set_attr "type" "mve_move")
3008    (set_attr "length""8")])
3009
3010 ;;
3011 ;; [vcmpneq_m_s, vcmpneq_m_u])
3012 ;;
3013 (define_insn "mve_vcmpneq_m_<supf><mode>"
3014   [
3015    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3016         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3017                        (match_operand:MVE_2 2 "s_register_operand" "w")
3018                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3019          VCMPNEQ_M))
3020   ]
3021   "TARGET_HAVE_MVE"
3022   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3023   [(set_attr "type" "mve_move")
3024    (set_attr "length""8")])
3025
3026 ;;
3027 ;; [vdupq_m_n_s, vdupq_m_n_u])
3028 ;;
3029 (define_insn "mve_vdupq_m_n_<supf><mode>"
3030   [
3031    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3032         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3033                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3034                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3035          VDUPQ_M_N))
3036   ]
3037   "TARGET_HAVE_MVE"
3038   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3039   [(set_attr "type" "mve_move")
3040    (set_attr "length""8")])
3041
3042 ;;
3043 ;; [vmaxaq_m_s])
3044 ;;
3045 (define_insn "mve_vmaxaq_m_s<mode>"
3046   [
3047    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3048         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3049                        (match_operand:MVE_2 2 "s_register_operand" "w")
3050                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3051          VMAXAQ_M_S))
3052   ]
3053   "TARGET_HAVE_MVE"
3054   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3055   [(set_attr "type" "mve_move")
3056    (set_attr "length""8")])
3057
3058 ;;
3059 ;; [vmaxavq_p_s])
3060 ;;
3061 (define_insn "mve_vmaxavq_p_s<mode>"
3062   [
3063    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3064         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3065                        (match_operand:MVE_2 2 "s_register_operand" "w")
3066                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3067          VMAXAVQ_P_S))
3068   ]
3069   "TARGET_HAVE_MVE"
3070   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3071   [(set_attr "type" "mve_move")
3072    (set_attr "length""8")])
3073
3074 ;;
3075 ;; [vmaxvq_p_u, vmaxvq_p_s])
3076 ;;
3077 (define_insn "mve_vmaxvq_p_<supf><mode>"
3078   [
3079    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3080         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3081                        (match_operand:MVE_2 2 "s_register_operand" "w")
3082                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3083          VMAXVQ_P))
3084   ]
3085   "TARGET_HAVE_MVE"
3086   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3087   [(set_attr "type" "mve_move")
3088    (set_attr "length""8")])
3089
3090 ;;
3091 ;; [vminaq_m_s])
3092 ;;
3093 (define_insn "mve_vminaq_m_s<mode>"
3094   [
3095    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3096         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3097                        (match_operand:MVE_2 2 "s_register_operand" "w")
3098                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3099          VMINAQ_M_S))
3100   ]
3101   "TARGET_HAVE_MVE"
3102   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3103   [(set_attr "type" "mve_move")
3104    (set_attr "length""8")])
3105
3106 ;;
3107 ;; [vminavq_p_s])
3108 ;;
3109 (define_insn "mve_vminavq_p_s<mode>"
3110   [
3111    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3112         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3113                        (match_operand:MVE_2 2 "s_register_operand" "w")
3114                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3115          VMINAVQ_P_S))
3116   ]
3117   "TARGET_HAVE_MVE"
3118   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3119   [(set_attr "type" "mve_move")
3120    (set_attr "length""8")])
3121
3122 ;;
3123 ;; [vminvq_p_s, vminvq_p_u])
3124 ;;
3125 (define_insn "mve_vminvq_p_<supf><mode>"
3126   [
3127    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3128         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3129                        (match_operand:MVE_2 2 "s_register_operand" "w")
3130                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3131          VMINVQ_P))
3132   ]
3133   "TARGET_HAVE_MVE"
3134   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3135   [(set_attr "type" "mve_move")
3136    (set_attr "length""8")])
3137
3138 ;;
3139 ;; [vmladavaq_u, vmladavaq_s])
3140 ;;
3141 (define_insn "mve_vmladavaq_<supf><mode>"
3142   [
3143    (set (match_operand:SI 0 "s_register_operand" "=Te")
3144         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3145                        (match_operand:MVE_2 2 "s_register_operand" "w")
3146                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3147          VMLADAVAQ))
3148   ]
3149   "TARGET_HAVE_MVE"
3150   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3151   [(set_attr "type" "mve_move")
3152 ])
3153
3154 ;;
3155 ;; [vmladavq_p_u, vmladavq_p_s])
3156 ;;
3157 (define_insn "mve_vmladavq_p_<supf><mode>"
3158   [
3159    (set (match_operand:SI 0 "s_register_operand" "=Te")
3160         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3161                        (match_operand:MVE_2 2 "s_register_operand" "w")
3162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3163          VMLADAVQ_P))
3164   ]
3165   "TARGET_HAVE_MVE"
3166   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3167   [(set_attr "type" "mve_move")
3168    (set_attr "length""8")])
3169
3170 ;;
3171 ;; [vmladavxq_p_s])
3172 ;;
3173 (define_insn "mve_vmladavxq_p_s<mode>"
3174   [
3175    (set (match_operand:SI 0 "s_register_operand" "=Te")
3176         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3177                        (match_operand:MVE_2 2 "s_register_operand" "w")
3178                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3179          VMLADAVXQ_P_S))
3180   ]
3181   "TARGET_HAVE_MVE"
3182   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3183   [(set_attr "type" "mve_move")
3184    (set_attr "length""8")])
3185
3186 ;;
3187 ;; [vmlaq_n_u, vmlaq_n_s])
3188 ;;
3189 (define_insn "mve_vmlaq_n_<supf><mode>"
3190   [
3191    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3192         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3193                        (match_operand:MVE_2 2 "s_register_operand" "w")
3194                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3195          VMLAQ_N))
3196   ]
3197   "TARGET_HAVE_MVE"
3198   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3199   [(set_attr "type" "mve_move")
3200 ])
3201
3202 ;;
3203 ;; [vmlasq_n_u, vmlasq_n_s])
3204 ;;
3205 (define_insn "mve_vmlasq_n_<supf><mode>"
3206   [
3207    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3208         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3209                        (match_operand:MVE_2 2 "s_register_operand" "w")
3210                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3211          VMLASQ_N))
3212   ]
3213   "TARGET_HAVE_MVE"
3214   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3215   [(set_attr "type" "mve_move")
3216 ])
3217
3218 ;;
3219 ;; [vmlsdavq_p_s])
3220 ;;
3221 (define_insn "mve_vmlsdavq_p_s<mode>"
3222   [
3223    (set (match_operand:SI 0 "s_register_operand" "=Te")
3224         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3225                        (match_operand:MVE_2 2 "s_register_operand" "w")
3226                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3227          VMLSDAVQ_P_S))
3228   ]
3229   "TARGET_HAVE_MVE"
3230   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3231   [(set_attr "type" "mve_move")
3232    (set_attr "length""8")])
3233
3234 ;;
3235 ;; [vmlsdavxq_p_s])
3236 ;;
3237 (define_insn "mve_vmlsdavxq_p_s<mode>"
3238   [
3239    (set (match_operand:SI 0 "s_register_operand" "=Te")
3240         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3241                        (match_operand:MVE_2 2 "s_register_operand" "w")
3242                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3243          VMLSDAVXQ_P_S))
3244   ]
3245   "TARGET_HAVE_MVE"
3246   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3247   [(set_attr "type" "mve_move")
3248    (set_attr "length""8")])
3249
3250 ;;
3251 ;; [vmvnq_m_s, vmvnq_m_u])
3252 ;;
3253 (define_insn "mve_vmvnq_m_<supf><mode>"
3254   [
3255    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3256         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3257                        (match_operand:MVE_2 2 "s_register_operand" "w")
3258                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3259          VMVNQ_M))
3260   ]
3261   "TARGET_HAVE_MVE"
3262   "vpst\;vmvnt %q0, %q2"
3263   [(set_attr "type" "mve_move")
3264    (set_attr "length""8")])
3265
3266 ;;
3267 ;; [vnegq_m_s])
3268 ;;
3269 (define_insn "mve_vnegq_m_s<mode>"
3270   [
3271    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3272         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3273                        (match_operand:MVE_2 2 "s_register_operand" "w")
3274                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3275          VNEGQ_M_S))
3276   ]
3277   "TARGET_HAVE_MVE"
3278   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3279   [(set_attr "type" "mve_move")
3280    (set_attr "length""8")])
3281
3282 ;;
3283 ;; [vpselq_u, vpselq_s])
3284 ;;
3285 (define_insn "mve_vpselq_<supf><mode>"
3286   [
3287    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3288         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3289                        (match_operand:MVE_1 2 "s_register_operand" "w")
3290                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3291          VPSELQ))
3292   ]
3293   "TARGET_HAVE_MVE"
3294   "vpsel %q0, %q1, %q2"
3295   [(set_attr "type" "mve_move")
3296 ])
3297
3298 ;;
3299 ;; [vqabsq_m_s])
3300 ;;
3301 (define_insn "mve_vqabsq_m_s<mode>"
3302   [
3303    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3304         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3305                        (match_operand:MVE_2 2 "s_register_operand" "w")
3306                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3307          VQABSQ_M_S))
3308   ]
3309   "TARGET_HAVE_MVE"
3310   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3311   [(set_attr "type" "mve_move")
3312    (set_attr "length""8")])
3313
3314 ;;
3315 ;; [vqdmlahq_n_s])
3316 ;;
3317 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3318   [
3319    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3320         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3321                        (match_operand:MVE_2 2 "s_register_operand" "w")
3322                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3323          VQDMLAHQ_N))
3324   ]
3325   "TARGET_HAVE_MVE"
3326   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3327   [(set_attr "type" "mve_move")
3328 ])
3329
3330 ;;
3331 ;; [vqdmlashq_n_s])
3332 ;;
3333 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3334   [
3335    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3336         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3337                        (match_operand:MVE_2 2 "s_register_operand" "w")
3338                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3339          VQDMLASHQ_N))
3340   ]
3341   "TARGET_HAVE_MVE"
3342   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3343   [(set_attr "type" "mve_move")
3344 ])
3345
3346 ;;
3347 ;; [vqnegq_m_s])
3348 ;;
3349 (define_insn "mve_vqnegq_m_s<mode>"
3350   [
3351    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3352         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3353                        (match_operand:MVE_2 2 "s_register_operand" "w")
3354                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3355          VQNEGQ_M_S))
3356   ]
3357   "TARGET_HAVE_MVE"
3358   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3359   [(set_attr "type" "mve_move")
3360    (set_attr "length""8")])
3361
3362 ;;
3363 ;; [vqrdmladhq_s])
3364 ;;
3365 (define_insn "mve_vqrdmladhq_s<mode>"
3366   [
3367    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3368         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3369                        (match_operand:MVE_2 2 "s_register_operand" "w")
3370                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3371          VQRDMLADHQ_S))
3372   ]
3373   "TARGET_HAVE_MVE"
3374   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3375   [(set_attr "type" "mve_move")
3376 ])
3377
3378 ;;
3379 ;; [vqrdmladhxq_s])
3380 ;;
3381 (define_insn "mve_vqrdmladhxq_s<mode>"
3382   [
3383    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3384         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3385                        (match_operand:MVE_2 2 "s_register_operand" "w")
3386                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3387          VQRDMLADHXQ_S))
3388   ]
3389   "TARGET_HAVE_MVE"
3390   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3391   [(set_attr "type" "mve_move")
3392 ])
3393
3394 ;;
3395 ;; [vqrdmlahq_n_s])
3396 ;;
3397 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3398   [
3399    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3400         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3401                        (match_operand:MVE_2 2 "s_register_operand" "w")
3402                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3403          VQRDMLAHQ_N))
3404   ]
3405   "TARGET_HAVE_MVE"
3406   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3407   [(set_attr "type" "mve_move")
3408 ])
3409
3410 ;;
3411 ;; [vqrdmlashq_n_s])
3412 ;;
3413 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3414   [
3415    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3416         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3417                        (match_operand:MVE_2 2 "s_register_operand" "w")
3418                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3419          VQRDMLASHQ_N))
3420   ]
3421   "TARGET_HAVE_MVE"
3422   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3423   [(set_attr "type" "mve_move")
3424 ])
3425
3426 ;;
3427 ;; [vqrdmlsdhq_s])
3428 ;;
3429 (define_insn "mve_vqrdmlsdhq_s<mode>"
3430   [
3431    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3432         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3433                        (match_operand:MVE_2 2 "s_register_operand" "w")
3434                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3435          VQRDMLSDHQ_S))
3436   ]
3437   "TARGET_HAVE_MVE"
3438   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3439   [(set_attr "type" "mve_move")
3440 ])
3441
3442 ;;
3443 ;; [vqrdmlsdhxq_s])
3444 ;;
3445 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3446   [
3447    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3448         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3449                        (match_operand:MVE_2 2 "s_register_operand" "w")
3450                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3451          VQRDMLSDHXQ_S))
3452   ]
3453   "TARGET_HAVE_MVE"
3454   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3455   [(set_attr "type" "mve_move")
3456 ])
3457
3458 ;;
3459 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3460 ;;
3461 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3462   [
3463    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3464         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3465                        (match_operand:SI 2 "s_register_operand" "r")
3466                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3467          VQRSHLQ_M_N))
3468   ]
3469   "TARGET_HAVE_MVE"
3470   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3471   [(set_attr "type" "mve_move")
3472    (set_attr "length""8")])
3473
3474 ;;
3475 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3476 ;;
3477 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3478   [
3479    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3480         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3481                        (match_operand:SI 2 "s_register_operand" "r")
3482                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3483          VQSHLQ_M_R))
3484   ]
3485   "TARGET_HAVE_MVE"
3486   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3487   [(set_attr "type" "mve_move")
3488    (set_attr "length""8")])
3489
3490 ;;
3491 ;; [vrev64q_m_u, vrev64q_m_s])
3492 ;;
3493 (define_insn "mve_vrev64q_m_<supf><mode>"
3494   [
3495    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3496         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3497                        (match_operand:MVE_2 2 "s_register_operand" "w")
3498                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3499          VREV64Q_M))
3500   ]
3501   "TARGET_HAVE_MVE"
3502   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3503   [(set_attr "type" "mve_move")
3504    (set_attr "length""8")])
3505
3506 ;;
3507 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3508 ;;
3509 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3510   [
3511    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3512         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3513                        (match_operand:SI 2 "s_register_operand" "r")
3514                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3515          VRSHLQ_M_N))
3516   ]
3517   "TARGET_HAVE_MVE"
3518   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3519   [(set_attr "type" "mve_move")
3520    (set_attr "length""8")])
3521
3522 ;;
3523 ;; [vshlq_m_r_u, vshlq_m_r_s])
3524 ;;
3525 (define_insn "mve_vshlq_m_r_<supf><mode>"
3526   [
3527    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3528         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3529                        (match_operand:SI 2 "s_register_operand" "r")
3530                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3531          VSHLQ_M_R))
3532   ]
3533   "TARGET_HAVE_MVE"
3534   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3535   [(set_attr "type" "mve_move")
3536    (set_attr "length""8")])
3537
3538 ;;
3539 ;; [vsliq_n_u, vsliq_n_s])
3540 ;;
3541 (define_insn "mve_vsliq_n_<supf><mode>"
3542   [
3543    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3544         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3545                        (match_operand:MVE_2 2 "s_register_operand" "w")
3546                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3547          VSLIQ_N))
3548   ]
3549   "TARGET_HAVE_MVE"
3550   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3551   [(set_attr "type" "mve_move")
3552 ])
3553
3554 ;;
3555 ;; [vsriq_n_u, vsriq_n_s])
3556 ;;
3557 (define_insn "mve_vsriq_n_<supf><mode>"
3558   [
3559    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3560         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3561                        (match_operand:MVE_2 2 "s_register_operand" "w")
3562                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3563          VSRIQ_N))
3564   ]
3565   "TARGET_HAVE_MVE"
3566   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3567   [(set_attr "type" "mve_move")
3568 ])
3569
3570 ;;
3571 ;; [vqdmlsdhxq_s])
3572 ;;
3573 (define_insn "mve_vqdmlsdhxq_s<mode>"
3574   [
3575    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3576         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3577                        (match_operand:MVE_2 2 "s_register_operand" "w")
3578                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3579          VQDMLSDHXQ_S))
3580   ]
3581   "TARGET_HAVE_MVE"
3582   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3583   [(set_attr "type" "mve_move")
3584 ])
3585
3586 ;;
3587 ;; [vqdmlsdhq_s])
3588 ;;
3589 (define_insn "mve_vqdmlsdhq_s<mode>"
3590   [
3591    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3592         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3593                        (match_operand:MVE_2 2 "s_register_operand" "w")
3594                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3595          VQDMLSDHQ_S))
3596   ]
3597   "TARGET_HAVE_MVE"
3598   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3599   [(set_attr "type" "mve_move")
3600 ])
3601
3602 ;;
3603 ;; [vqdmladhxq_s])
3604 ;;
3605 (define_insn "mve_vqdmladhxq_s<mode>"
3606   [
3607    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3608         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3609                        (match_operand:MVE_2 2 "s_register_operand" "w")
3610                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3611          VQDMLADHXQ_S))
3612   ]
3613   "TARGET_HAVE_MVE"
3614   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3615   [(set_attr "type" "mve_move")
3616 ])
3617
3618 ;;
3619 ;; [vqdmladhq_s])
3620 ;;
3621 (define_insn "mve_vqdmladhq_s<mode>"
3622   [
3623    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3624         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3625                        (match_operand:MVE_2 2 "s_register_operand" "w")
3626                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3627          VQDMLADHQ_S))
3628   ]
3629   "TARGET_HAVE_MVE"
3630   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3631   [(set_attr "type" "mve_move")
3632 ])
3633
3634 ;;
3635 ;; [vmlsdavaxq_s])
3636 ;;
3637 (define_insn "mve_vmlsdavaxq_s<mode>"
3638   [
3639    (set (match_operand:SI 0 "s_register_operand" "=Te")
3640         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3641                     (match_operand:MVE_2 2 "s_register_operand" "w")
3642                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3643          VMLSDAVAXQ_S))
3644   ]
3645   "TARGET_HAVE_MVE"
3646   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3647   [(set_attr "type" "mve_move")
3648 ])
3649
3650 ;;
3651 ;; [vmlsdavaq_s])
3652 ;;
3653 (define_insn "mve_vmlsdavaq_s<mode>"
3654   [
3655    (set (match_operand:SI 0 "s_register_operand" "=Te")
3656         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3657                     (match_operand:MVE_2 2 "s_register_operand" "w")
3658                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3659          VMLSDAVAQ_S))
3660   ]
3661   "TARGET_HAVE_MVE"
3662   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
3663   [(set_attr "type" "mve_move")
3664 ])
3665
3666 ;;
3667 ;; [vmladavaxq_s])
3668 ;;
3669 (define_insn "mve_vmladavaxq_s<mode>"
3670   [
3671    (set (match_operand:SI 0 "s_register_operand" "=Te")
3672         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3673                     (match_operand:MVE_2 2 "s_register_operand" "w")
3674                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3675          VMLADAVAXQ_S))
3676   ]
3677   "TARGET_HAVE_MVE"
3678   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3679   [(set_attr "type" "mve_move")
3680 ])
3681 ;;
3682 ;; [vabsq_m_f])
3683 ;;
3684 (define_insn "mve_vabsq_m_f<mode>"
3685   [
3686    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3687         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3688                        (match_operand:MVE_0 2 "s_register_operand" "w")
3689                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3690          VABSQ_M_F))
3691   ]
3692   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3693   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
3694   [(set_attr "type" "mve_move")
3695    (set_attr "length""8")])
3696
3697 ;;
3698 ;; [vaddlvaq_p_s vaddlvaq_p_u])
3699 ;;
3700 (define_insn "mve_vaddlvaq_p_<supf>v4si"
3701   [
3702    (set (match_operand:DI 0 "s_register_operand" "=r")
3703         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3704                        (match_operand:V4SI 2 "s_register_operand" "w")
3705                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3706          VADDLVAQ_P))
3707   ]
3708   "TARGET_HAVE_MVE"
3709   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
3710   [(set_attr "type" "mve_move")
3711    (set_attr "length""8")])
3712 ;;
3713 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
3714 ;;
3715 (define_insn "mve_vcmlaq<mve_rot><mode>"
3716   [
3717    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
3718         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
3719                     (unspec:MVE_0
3720                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
3721                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
3722                      VCMLA)))
3723   ]
3724   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3725   "@
3726    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
3727    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
3728   [(set_attr "type" "mve_move")
3729 ])
3730
3731 ;;
3732 ;; [vcmpeqq_m_n_f])
3733 ;;
3734 (define_insn "mve_vcmpeqq_m_n_f<mode>"
3735   [
3736    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3737         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3738                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3739                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3740          VCMPEQQ_M_N_F))
3741   ]
3742   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3743   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
3744   [(set_attr "type" "mve_move")
3745    (set_attr "length""8")])
3746
3747 ;;
3748 ;; [vcmpgeq_m_f])
3749 ;;
3750 (define_insn "mve_vcmpgeq_m_f<mode>"
3751   [
3752    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3753         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3754                        (match_operand:MVE_0 2 "s_register_operand" "w")
3755                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3756          VCMPGEQ_M_F))
3757   ]
3758   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3759   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
3760   [(set_attr "type" "mve_move")
3761    (set_attr "length""8")])
3762
3763 ;;
3764 ;; [vcmpgeq_m_n_f])
3765 ;;
3766 (define_insn "mve_vcmpgeq_m_n_f<mode>"
3767   [
3768    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3769         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3770                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3771                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3772          VCMPGEQ_M_N_F))
3773   ]
3774   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3775   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
3776   [(set_attr "type" "mve_move")
3777    (set_attr "length""8")])
3778
3779 ;;
3780 ;; [vcmpgtq_m_f])
3781 ;;
3782 (define_insn "mve_vcmpgtq_m_f<mode>"
3783   [
3784    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3785         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3786                        (match_operand:MVE_0 2 "s_register_operand" "w")
3787                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3788          VCMPGTQ_M_F))
3789   ]
3790   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3791   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
3792   [(set_attr "type" "mve_move")
3793    (set_attr "length""8")])
3794
3795 ;;
3796 ;; [vcmpgtq_m_n_f])
3797 ;;
3798 (define_insn "mve_vcmpgtq_m_n_f<mode>"
3799   [
3800    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3801         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3802                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3803                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3804          VCMPGTQ_M_N_F))
3805   ]
3806   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3807   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
3808   [(set_attr "type" "mve_move")
3809    (set_attr "length""8")])
3810
3811 ;;
3812 ;; [vcmpleq_m_f])
3813 ;;
3814 (define_insn "mve_vcmpleq_m_f<mode>"
3815   [
3816    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3817         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3818                        (match_operand:MVE_0 2 "s_register_operand" "w")
3819                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3820          VCMPLEQ_M_F))
3821   ]
3822   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3823   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
3824   [(set_attr "type" "mve_move")
3825    (set_attr "length""8")])
3826
3827 ;;
3828 ;; [vcmpleq_m_n_f])
3829 ;;
3830 (define_insn "mve_vcmpleq_m_n_f<mode>"
3831   [
3832    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3833         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3834                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3835                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3836          VCMPLEQ_M_N_F))
3837   ]
3838   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3839   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
3840   [(set_attr "type" "mve_move")
3841    (set_attr "length""8")])
3842
3843 ;;
3844 ;; [vcmpltq_m_f])
3845 ;;
3846 (define_insn "mve_vcmpltq_m_f<mode>"
3847   [
3848    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3849         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3850                        (match_operand:MVE_0 2 "s_register_operand" "w")
3851                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3852          VCMPLTQ_M_F))
3853   ]
3854   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3855   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
3856   [(set_attr "type" "mve_move")
3857    (set_attr "length""8")])
3858
3859 ;;
3860 ;; [vcmpltq_m_n_f])
3861 ;;
3862 (define_insn "mve_vcmpltq_m_n_f<mode>"
3863   [
3864    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3865         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3866                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3867                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3868          VCMPLTQ_M_N_F))
3869   ]
3870   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3871   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
3872   [(set_attr "type" "mve_move")
3873    (set_attr "length""8")])
3874
3875 ;;
3876 ;; [vcmpneq_m_f])
3877 ;;
3878 (define_insn "mve_vcmpneq_m_f<mode>"
3879   [
3880    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3881         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3882                        (match_operand:MVE_0 2 "s_register_operand" "w")
3883                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3884          VCMPNEQ_M_F))
3885   ]
3886   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3887   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
3888   [(set_attr "type" "mve_move")
3889    (set_attr "length""8")])
3890
3891 ;;
3892 ;; [vcmpneq_m_n_f])
3893 ;;
3894 (define_insn "mve_vcmpneq_m_n_f<mode>"
3895   [
3896    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3897         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3898                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3899                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3900          VCMPNEQ_M_N_F))
3901   ]
3902   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3903   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
3904   [(set_attr "type" "mve_move")
3905    (set_attr "length""8")])
3906
3907 ;;
3908 ;; [vcvtbq_m_f16_f32])
3909 ;;
3910 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
3911   [
3912    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3913         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3914                        (match_operand:V4SF 2 "s_register_operand" "w")
3915                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3916          VCVTBQ_M_F16_F32))
3917   ]
3918   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3919   "vpst\;vcvtbt.f16.f32 %q0, %q2"
3920   [(set_attr "type" "mve_move")
3921    (set_attr "length""8")])
3922
3923 ;;
3924 ;; [vcvtbq_m_f32_f16])
3925 ;;
3926 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
3927   [
3928    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3929         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3930                        (match_operand:V8HF 2 "s_register_operand" "w")
3931                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3932          VCVTBQ_M_F32_F16))
3933   ]
3934   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3935   "vpst\;vcvtbt.f32.f16 %q0, %q2"
3936   [(set_attr "type" "mve_move")
3937    (set_attr "length""8")])
3938
3939 ;;
3940 ;; [vcvttq_m_f16_f32])
3941 ;;
3942 (define_insn "mve_vcvttq_m_f16_f32v8hf"
3943   [
3944    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3945         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3946                        (match_operand:V4SF 2 "s_register_operand" "w")
3947                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3948          VCVTTQ_M_F16_F32))
3949   ]
3950   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3951   "vpst\;vcvttt.f16.f32 %q0, %q2"
3952   [(set_attr "type" "mve_move")
3953    (set_attr "length""8")])
3954
3955 ;;
3956 ;; [vcvttq_m_f32_f16])
3957 ;;
3958 (define_insn "mve_vcvttq_m_f32_f16v4sf"
3959   [
3960    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3961         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3962                        (match_operand:V8HF 2 "s_register_operand" "w")
3963                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3964          VCVTTQ_M_F32_F16))
3965   ]
3966   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3967   "vpst\;vcvttt.f32.f16 %q0, %q2"
3968   [(set_attr "type" "mve_move")
3969    (set_attr "length""8")])
3970
3971 ;;
3972 ;; [vdupq_m_n_f])
3973 ;;
3974 (define_insn "mve_vdupq_m_n_f<mode>"
3975   [
3976    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3977         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3978                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3979                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3980          VDUPQ_M_N_F))
3981   ]
3982   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3983   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3984   [(set_attr "type" "mve_move")
3985    (set_attr "length""8")])
3986
3987 ;;
3988 ;; [vfmaq_f])
3989 ;;
3990 (define_insn "mve_vfmaq_f<mode>"
3991   [
3992    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3993         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3994                        (match_operand:MVE_0 2 "s_register_operand" "w")
3995                        (match_operand:MVE_0 3 "s_register_operand" "w")]
3996          VFMAQ_F))
3997   ]
3998   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3999   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4000   [(set_attr "type" "mve_move")
4001 ])
4002
4003 ;;
4004 ;; [vfmaq_n_f])
4005 ;;
4006 (define_insn "mve_vfmaq_n_f<mode>"
4007   [
4008    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4009         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4010                        (match_operand:MVE_0 2 "s_register_operand" "w")
4011                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4012          VFMAQ_N_F))
4013   ]
4014   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4015   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4016   [(set_attr "type" "mve_move")
4017 ])
4018
4019 ;;
4020 ;; [vfmasq_n_f])
4021 ;;
4022 (define_insn "mve_vfmasq_n_f<mode>"
4023   [
4024    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4025         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4026                        (match_operand:MVE_0 2 "s_register_operand" "w")
4027                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4028          VFMASQ_N_F))
4029   ]
4030   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4031   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4032   [(set_attr "type" "mve_move")
4033 ])
4034 ;;
4035 ;; [vfmsq_f])
4036 ;;
4037 (define_insn "mve_vfmsq_f<mode>"
4038   [
4039    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4040         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4041                        (match_operand:MVE_0 2 "s_register_operand" "w")
4042                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4043          VFMSQ_F))
4044   ]
4045   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4046   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4047   [(set_attr "type" "mve_move")
4048 ])
4049
4050 ;;
4051 ;; [vmaxnmaq_m_f])
4052 ;;
4053 (define_insn "mve_vmaxnmaq_m_f<mode>"
4054   [
4055    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4056         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4057                        (match_operand:MVE_0 2 "s_register_operand" "w")
4058                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4059          VMAXNMAQ_M_F))
4060   ]
4061   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4062   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4063   [(set_attr "type" "mve_move")
4064    (set_attr "length""8")])
4065 ;;
4066 ;; [vmaxnmavq_p_f])
4067 ;;
4068 (define_insn "mve_vmaxnmavq_p_f<mode>"
4069   [
4070    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4071         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4072                        (match_operand:MVE_0 2 "s_register_operand" "w")
4073                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4074          VMAXNMAVQ_P_F))
4075   ]
4076   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4077   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4078   [(set_attr "type" "mve_move")
4079    (set_attr "length""8")])
4080
4081 ;;
4082 ;; [vmaxnmvq_p_f])
4083 ;;
4084 (define_insn "mve_vmaxnmvq_p_f<mode>"
4085   [
4086    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4087         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4088                        (match_operand:MVE_0 2 "s_register_operand" "w")
4089                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4090          VMAXNMVQ_P_F))
4091   ]
4092   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4093   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4094   [(set_attr "type" "mve_move")
4095    (set_attr "length""8")])
4096 ;;
4097 ;; [vminnmaq_m_f])
4098 ;;
4099 (define_insn "mve_vminnmaq_m_f<mode>"
4100   [
4101    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4102         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4103                        (match_operand:MVE_0 2 "s_register_operand" "w")
4104                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4105          VMINNMAQ_M_F))
4106   ]
4107   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4108   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4109   [(set_attr "type" "mve_move")
4110    (set_attr "length""8")])
4111
4112 ;;
4113 ;; [vminnmavq_p_f])
4114 ;;
4115 (define_insn "mve_vminnmavq_p_f<mode>"
4116   [
4117    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4118         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4119                        (match_operand:MVE_0 2 "s_register_operand" "w")
4120                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4121          VMINNMAVQ_P_F))
4122   ]
4123   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4124   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4125   [(set_attr "type" "mve_move")
4126    (set_attr "length""8")])
4127 ;;
4128 ;; [vminnmvq_p_f])
4129 ;;
4130 (define_insn "mve_vminnmvq_p_f<mode>"
4131   [
4132    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4133         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4134                        (match_operand:MVE_0 2 "s_register_operand" "w")
4135                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4136          VMINNMVQ_P_F))
4137   ]
4138   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4139   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4140   [(set_attr "type" "mve_move")
4141    (set_attr "length""8")])
4142
4143 ;;
4144 ;; [vmlaldavaq_s, vmlaldavaq_u])
4145 ;;
4146 (define_insn "mve_vmlaldavaq_<supf><mode>"
4147   [
4148    (set (match_operand:DI 0 "s_register_operand" "=r")
4149         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4150                        (match_operand:MVE_5 2 "s_register_operand" "w")
4151                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4152          VMLALDAVAQ))
4153   ]
4154   "TARGET_HAVE_MVE"
4155   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4156   [(set_attr "type" "mve_move")
4157 ])
4158
4159 ;;
4160 ;; [vmlaldavaxq_s])
4161 ;;
4162 (define_insn "mve_vmlaldavaxq_s<mode>"
4163   [
4164    (set (match_operand:DI 0 "s_register_operand" "=r")
4165         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4166                        (match_operand:MVE_5 2 "s_register_operand" "w")
4167                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4168          VMLALDAVAXQ_S))
4169   ]
4170   "TARGET_HAVE_MVE"
4171   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4172   [(set_attr "type" "mve_move")
4173 ])
4174
4175 ;;
4176 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4177 ;;
4178 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4179   [
4180    (set (match_operand:DI 0 "s_register_operand" "=r")
4181         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4182                        (match_operand:MVE_5 2 "s_register_operand" "w")
4183                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4184          VMLALDAVQ_P))
4185   ]
4186   "TARGET_HAVE_MVE"
4187   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4188   [(set_attr "type" "mve_move")
4189    (set_attr "length""8")])
4190
4191 ;;
4192 ;; [vmlaldavxq_p_s])
4193 ;;
4194 (define_insn "mve_vmlaldavxq_p_s<mode>"
4195   [
4196    (set (match_operand:DI 0 "s_register_operand" "=r")
4197         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4198                        (match_operand:MVE_5 2 "s_register_operand" "w")
4199                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4200          VMLALDAVXQ_P_S))
4201   ]
4202   "TARGET_HAVE_MVE"
4203   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4204   [(set_attr "type" "mve_move")
4205    (set_attr "length""8")])
4206 ;;
4207 ;; [vmlsldavaq_s])
4208 ;;
4209 (define_insn "mve_vmlsldavaq_s<mode>"
4210   [
4211    (set (match_operand:DI 0 "s_register_operand" "=r")
4212         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4213                        (match_operand:MVE_5 2 "s_register_operand" "w")
4214                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4215          VMLSLDAVAQ_S))
4216   ]
4217   "TARGET_HAVE_MVE"
4218   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4219   [(set_attr "type" "mve_move")
4220 ])
4221
4222 ;;
4223 ;; [vmlsldavaxq_s])
4224 ;;
4225 (define_insn "mve_vmlsldavaxq_s<mode>"
4226   [
4227    (set (match_operand:DI 0 "s_register_operand" "=r")
4228         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4229                        (match_operand:MVE_5 2 "s_register_operand" "w")
4230                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4231          VMLSLDAVAXQ_S))
4232   ]
4233   "TARGET_HAVE_MVE"
4234   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4235   [(set_attr "type" "mve_move")
4236 ])
4237
4238 ;;
4239 ;; [vmlsldavq_p_s])
4240 ;;
4241 (define_insn "mve_vmlsldavq_p_s<mode>"
4242   [
4243    (set (match_operand:DI 0 "s_register_operand" "=r")
4244         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4245                        (match_operand:MVE_5 2 "s_register_operand" "w")
4246                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4247          VMLSLDAVQ_P_S))
4248   ]
4249   "TARGET_HAVE_MVE"
4250   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4251   [(set_attr "type" "mve_move")
4252    (set_attr "length""8")])
4253
4254 ;;
4255 ;; [vmlsldavxq_p_s])
4256 ;;
4257 (define_insn "mve_vmlsldavxq_p_s<mode>"
4258   [
4259    (set (match_operand:DI 0 "s_register_operand" "=r")
4260         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4261                        (match_operand:MVE_5 2 "s_register_operand" "w")
4262                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4263          VMLSLDAVXQ_P_S))
4264   ]
4265   "TARGET_HAVE_MVE"
4266   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4267   [(set_attr "type" "mve_move")
4268    (set_attr "length""8")])
4269 ;;
4270 ;; [vmovlbq_m_u, vmovlbq_m_s])
4271 ;;
4272 (define_insn "mve_vmovlbq_m_<supf><mode>"
4273   [
4274    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4275         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4276                        (match_operand:MVE_3 2 "s_register_operand" "w")
4277                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4278          VMOVLBQ_M))
4279   ]
4280   "TARGET_HAVE_MVE"
4281   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4282   [(set_attr "type" "mve_move")
4283    (set_attr "length""8")])
4284 ;;
4285 ;; [vmovltq_m_u, vmovltq_m_s])
4286 ;;
4287 (define_insn "mve_vmovltq_m_<supf><mode>"
4288   [
4289    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4290         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4291                        (match_operand:MVE_3 2 "s_register_operand" "w")
4292                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4293          VMOVLTQ_M))
4294   ]
4295   "TARGET_HAVE_MVE"
4296   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4297   [(set_attr "type" "mve_move")
4298    (set_attr "length""8")])
4299 ;;
4300 ;; [vmovnbq_m_u, vmovnbq_m_s])
4301 ;;
4302 (define_insn "mve_vmovnbq_m_<supf><mode>"
4303   [
4304    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4305         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4306                        (match_operand:MVE_5 2 "s_register_operand" "w")
4307                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4308          VMOVNBQ_M))
4309   ]
4310   "TARGET_HAVE_MVE"
4311   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4312   [(set_attr "type" "mve_move")
4313    (set_attr "length""8")])
4314
4315 ;;
4316 ;; [vmovntq_m_u, vmovntq_m_s])
4317 ;;
4318 (define_insn "mve_vmovntq_m_<supf><mode>"
4319   [
4320    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4321         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4322                        (match_operand:MVE_5 2 "s_register_operand" "w")
4323                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4324          VMOVNTQ_M))
4325   ]
4326   "TARGET_HAVE_MVE"
4327   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4328   [(set_attr "type" "mve_move")
4329    (set_attr "length""8")])
4330
4331 ;;
4332 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4333 ;;
4334 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4335   [
4336    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4337         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4338                        (match_operand:SI 2 "immediate_operand" "i")
4339                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4340          VMVNQ_M_N))
4341   ]
4342   "TARGET_HAVE_MVE"
4343   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4344   [(set_attr "type" "mve_move")
4345    (set_attr "length""8")])
4346 ;;
4347 ;; [vnegq_m_f])
4348 ;;
4349 (define_insn "mve_vnegq_m_f<mode>"
4350   [
4351    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4352         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4353                        (match_operand:MVE_0 2 "s_register_operand" "w")
4354                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4355          VNEGQ_M_F))
4356   ]
4357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4358   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4359   [(set_attr "type" "mve_move")
4360    (set_attr "length""8")])
4361
4362 ;;
4363 ;; [vorrq_m_n_s, vorrq_m_n_u])
4364 ;;
4365 (define_insn "mve_vorrq_m_n_<supf><mode>"
4366   [
4367    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4368         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4369                        (match_operand:SI 2 "immediate_operand" "i")
4370                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4371          VORRQ_M_N))
4372   ]
4373   "TARGET_HAVE_MVE"
4374   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4375   [(set_attr "type" "mve_move")
4376    (set_attr "length""8")])
4377 ;;
4378 ;; [vpselq_f])
4379 ;;
4380 (define_insn "mve_vpselq_f<mode>"
4381   [
4382    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4383         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4384                        (match_operand:MVE_0 2 "s_register_operand" "w")
4385                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4386          VPSELQ_F))
4387   ]
4388   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4389   "vpsel %q0, %q1, %q2"
4390   [(set_attr "type" "mve_move")
4391 ])
4392
4393 ;;
4394 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4395 ;;
4396 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4397   [
4398    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4399         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4400                        (match_operand:MVE_5 2 "s_register_operand" "w")
4401                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4402          VQMOVNBQ_M))
4403   ]
4404   "TARGET_HAVE_MVE"
4405   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4406   [(set_attr "type" "mve_move")
4407    (set_attr "length""8")])
4408
4409 ;;
4410 ;; [vqmovntq_m_u, vqmovntq_m_s])
4411 ;;
4412 (define_insn "mve_vqmovntq_m_<supf><mode>"
4413   [
4414    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4415         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4416                        (match_operand:MVE_5 2 "s_register_operand" "w")
4417                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4418          VQMOVNTQ_M))
4419   ]
4420   "TARGET_HAVE_MVE"
4421   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4422   [(set_attr "type" "mve_move")
4423    (set_attr "length""8")])
4424
4425 ;;
4426 ;; [vqmovunbq_m_s])
4427 ;;
4428 (define_insn "mve_vqmovunbq_m_s<mode>"
4429   [
4430    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4431         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4432                        (match_operand:MVE_5 2 "s_register_operand" "w")
4433                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4434          VQMOVUNBQ_M_S))
4435   ]
4436   "TARGET_HAVE_MVE"
4437   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4438   [(set_attr "type" "mve_move")
4439    (set_attr "length""8")])
4440
4441 ;;
4442 ;; [vqmovuntq_m_s])
4443 ;;
4444 (define_insn "mve_vqmovuntq_m_s<mode>"
4445   [
4446    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4447         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4448                        (match_operand:MVE_5 2 "s_register_operand" "w")
4449                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4450          VQMOVUNTQ_M_S))
4451   ]
4452   "TARGET_HAVE_MVE"
4453   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4454   [(set_attr "type" "mve_move")
4455    (set_attr "length""8")])
4456
4457 ;;
4458 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4459 ;;
4460 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4461   [
4462    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4463         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4464                        (match_operand:MVE_5 2 "s_register_operand" "w")
4465                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4466          VQRSHRNTQ_N))
4467   ]
4468   "TARGET_HAVE_MVE"
4469   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4470   [(set_attr "type" "mve_move")
4471 ])
4472
4473 ;;
4474 ;; [vqrshruntq_n_s])
4475 ;;
4476 (define_insn "mve_vqrshruntq_n_s<mode>"
4477   [
4478    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4479         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4480                        (match_operand:MVE_5 2 "s_register_operand" "w")
4481                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4482          VQRSHRUNTQ_N_S))
4483   ]
4484   "TARGET_HAVE_MVE"
4485   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4486   [(set_attr "type" "mve_move")
4487 ])
4488
4489 ;;
4490 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4491 ;;
4492 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4493   [
4494    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4495         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4496                        (match_operand:MVE_5 2 "s_register_operand" "w")
4497                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4498          VQSHRNBQ_N))
4499   ]
4500   "TARGET_HAVE_MVE"
4501   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4502   [(set_attr "type" "mve_move")
4503 ])
4504
4505 ;;
4506 ;; [vqshrntq_n_u, vqshrntq_n_s])
4507 ;;
4508 (define_insn "mve_vqshrntq_n_<supf><mode>"
4509   [
4510    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4511         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4512                        (match_operand:MVE_5 2 "s_register_operand" "w")
4513                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4514          VQSHRNTQ_N))
4515   ]
4516   "TARGET_HAVE_MVE"
4517   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4518   [(set_attr "type" "mve_move")
4519 ])
4520
4521 ;;
4522 ;; [vqshrunbq_n_s])
4523 ;;
4524 (define_insn "mve_vqshrunbq_n_s<mode>"
4525   [
4526    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4527         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4528                        (match_operand:MVE_5 2 "s_register_operand" "w")
4529                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4530          VQSHRUNBQ_N_S))
4531   ]
4532   "TARGET_HAVE_MVE"
4533   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4534   [(set_attr "type" "mve_move")
4535 ])
4536
4537 ;;
4538 ;; [vqshruntq_n_s])
4539 ;;
4540 (define_insn "mve_vqshruntq_n_s<mode>"
4541   [
4542    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4543         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4544                        (match_operand:MVE_5 2 "s_register_operand" "w")
4545                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4546          VQSHRUNTQ_N_S))
4547   ]
4548   "TARGET_HAVE_MVE"
4549   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4550   [(set_attr "type" "mve_move")
4551 ])
4552
4553 ;;
4554 ;; [vrev32q_m_f])
4555 ;;
4556 (define_insn "mve_vrev32q_m_fv8hf"
4557   [
4558    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4559         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4560                        (match_operand:V8HF 2 "s_register_operand" "w")
4561                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4562          VREV32Q_M_F))
4563   ]
4564   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4565   "vpst\;vrev32t.16 %q0, %q2"
4566   [(set_attr "type" "mve_move")
4567    (set_attr "length""8")])
4568
4569 ;;
4570 ;; [vrev32q_m_s, vrev32q_m_u])
4571 ;;
4572 (define_insn "mve_vrev32q_m_<supf><mode>"
4573   [
4574    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4575         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4576                        (match_operand:MVE_3 2 "s_register_operand" "w")
4577                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4578          VREV32Q_M))
4579   ]
4580   "TARGET_HAVE_MVE"
4581   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4582   [(set_attr "type" "mve_move")
4583    (set_attr "length""8")])
4584
4585 ;;
4586 ;; [vrev64q_m_f])
4587 ;;
4588 (define_insn "mve_vrev64q_m_f<mode>"
4589   [
4590    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4591         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4592                        (match_operand:MVE_0 2 "s_register_operand" "w")
4593                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4594          VREV64Q_M_F))
4595   ]
4596   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4597   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4598   [(set_attr "type" "mve_move")
4599    (set_attr "length""8")])
4600
4601 ;;
4602 ;; [vrmlaldavhaxq_s])
4603 ;;
4604 (define_insn "mve_vrmlaldavhaxq_sv4si"
4605   [
4606    (set (match_operand:DI 0 "s_register_operand" "=r")
4607         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4608                        (match_operand:V4SI 2 "s_register_operand" "w")
4609                        (match_operand:V4SI 3 "s_register_operand" "w")]
4610          VRMLALDAVHAXQ_S))
4611   ]
4612   "TARGET_HAVE_MVE"
4613   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
4614   [(set_attr "type" "mve_move")
4615 ])
4616
4617 ;;
4618 ;; [vrmlaldavhxq_p_s])
4619 ;;
4620 (define_insn "mve_vrmlaldavhxq_p_sv4si"
4621   [
4622    (set (match_operand:DI 0 "s_register_operand" "=r")
4623         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4624                        (match_operand:V4SI 2 "s_register_operand" "w")
4625                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4626          VRMLALDAVHXQ_P_S))
4627   ]
4628   "TARGET_HAVE_MVE"
4629   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
4630   [(set_attr "type" "mve_move")
4631    (set_attr "length""8")])
4632
4633 ;;
4634 ;; [vrmlsldavhaxq_s])
4635 ;;
4636 (define_insn "mve_vrmlsldavhaxq_sv4si"
4637   [
4638    (set (match_operand:DI 0 "s_register_operand" "=r")
4639         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4640                        (match_operand:V4SI 2 "s_register_operand" "w")
4641                        (match_operand:V4SI 3 "s_register_operand" "w")]
4642          VRMLSLDAVHAXQ_S))
4643   ]
4644   "TARGET_HAVE_MVE"
4645   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
4646   [(set_attr "type" "mve_move")
4647 ])
4648
4649 ;;
4650 ;; [vrmlsldavhq_p_s])
4651 ;;
4652 (define_insn "mve_vrmlsldavhq_p_sv4si"
4653   [
4654    (set (match_operand:DI 0 "s_register_operand" "=r")
4655         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4656                        (match_operand:V4SI 2 "s_register_operand" "w")
4657                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4658          VRMLSLDAVHQ_P_S))
4659   ]
4660   "TARGET_HAVE_MVE"
4661   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
4662   [(set_attr "type" "mve_move")
4663    (set_attr "length""8")])
4664
4665 ;;
4666 ;; [vrmlsldavhxq_p_s])
4667 ;;
4668 (define_insn "mve_vrmlsldavhxq_p_sv4si"
4669   [
4670    (set (match_operand:DI 0 "s_register_operand" "=r")
4671         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4672                        (match_operand:V4SI 2 "s_register_operand" "w")
4673                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4674          VRMLSLDAVHXQ_P_S))
4675   ]
4676   "TARGET_HAVE_MVE"
4677   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
4678   [(set_attr "type" "mve_move")
4679    (set_attr "length""8")])
4680
4681 ;;
4682 ;; [vrndaq_m_f])
4683 ;;
4684 (define_insn "mve_vrndaq_m_f<mode>"
4685   [
4686    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4687         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4688                        (match_operand:MVE_0 2 "s_register_operand" "w")
4689                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4690          VRNDAQ_M_F))
4691   ]
4692   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4693   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
4694   [(set_attr "type" "mve_move")
4695    (set_attr "length""8")])
4696
4697 ;;
4698 ;; [vrndmq_m_f])
4699 ;;
4700 (define_insn "mve_vrndmq_m_f<mode>"
4701   [
4702    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4703         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4704                        (match_operand:MVE_0 2 "s_register_operand" "w")
4705                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4706          VRNDMQ_M_F))
4707   ]
4708   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4709   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
4710   [(set_attr "type" "mve_move")
4711    (set_attr "length""8")])
4712
4713 ;;
4714 ;; [vrndnq_m_f])
4715 ;;
4716 (define_insn "mve_vrndnq_m_f<mode>"
4717   [
4718    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4719         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4720                        (match_operand:MVE_0 2 "s_register_operand" "w")
4721                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4722          VRNDNQ_M_F))
4723   ]
4724   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4725   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
4726   [(set_attr "type" "mve_move")
4727    (set_attr "length""8")])
4728
4729 ;;
4730 ;; [vrndpq_m_f])
4731 ;;
4732 (define_insn "mve_vrndpq_m_f<mode>"
4733   [
4734    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4735         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4736                        (match_operand:MVE_0 2 "s_register_operand" "w")
4737                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4738          VRNDPQ_M_F))
4739   ]
4740   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4741   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
4742   [(set_attr "type" "mve_move")
4743    (set_attr "length""8")])
4744
4745 ;;
4746 ;; [vrndxq_m_f])
4747 ;;
4748 (define_insn "mve_vrndxq_m_f<mode>"
4749   [
4750    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4751         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4752                        (match_operand:MVE_0 2 "s_register_operand" "w")
4753                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4754          VRNDXQ_M_F))
4755   ]
4756   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4757   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
4758   [(set_attr "type" "mve_move")
4759    (set_attr "length""8")])
4760
4761 ;;
4762 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
4763 ;;
4764 (define_insn "mve_vrshrnbq_n_<supf><mode>"
4765   [
4766    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4767         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4768                        (match_operand:MVE_5 2 "s_register_operand" "w")
4769                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4770          VRSHRNBQ_N))
4771   ]
4772   "TARGET_HAVE_MVE"
4773   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
4774   [(set_attr "type" "mve_move")
4775 ])
4776
4777 ;;
4778 ;; [vrshrntq_n_u, vrshrntq_n_s])
4779 ;;
4780 (define_insn "mve_vrshrntq_n_<supf><mode>"
4781   [
4782    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4783         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4784                        (match_operand:MVE_5 2 "s_register_operand" "w")
4785                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4786          VRSHRNTQ_N))
4787   ]
4788   "TARGET_HAVE_MVE"
4789   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
4790   [(set_attr "type" "mve_move")
4791 ])
4792
4793 ;;
4794 ;; [vshrnbq_n_u, vshrnbq_n_s])
4795 ;;
4796 (define_insn "mve_vshrnbq_n_<supf><mode>"
4797   [
4798    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4799         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4800                        (match_operand:MVE_5 2 "s_register_operand" "w")
4801                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4802          VSHRNBQ_N))
4803   ]
4804   "TARGET_HAVE_MVE"
4805   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
4806   [(set_attr "type" "mve_move")
4807 ])
4808
4809 ;;
4810 ;; [vshrntq_n_s, vshrntq_n_u])
4811 ;;
4812 (define_insn "mve_vshrntq_n_<supf><mode>"
4813   [
4814    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4815         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4816                                  (match_operand:MVE_5 2 "s_register_operand" "w")
4817                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4818          VSHRNTQ_N))
4819   ]
4820   "TARGET_HAVE_MVE"
4821   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
4822   [(set_attr "type" "mve_move")
4823 ])
4824
4825 ;;
4826 ;; [vcvtmq_m_s, vcvtmq_m_u])
4827 ;;
4828 (define_insn "mve_vcvtmq_m_<supf><mode>"
4829   [
4830    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4831         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4832                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4833                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4834          VCVTMQ_M))
4835   ]
4836   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4837   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4838   [(set_attr "type" "mve_move")
4839    (set_attr "length""8")])
4840
4841 ;;
4842 ;; [vcvtpq_m_u, vcvtpq_m_s])
4843 ;;
4844 (define_insn "mve_vcvtpq_m_<supf><mode>"
4845   [
4846    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4847         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4848                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4849                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4850          VCVTPQ_M))
4851   ]
4852   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4853   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4854   [(set_attr "type" "mve_move")
4855    (set_attr "length""8")])
4856
4857 ;;
4858 ;; [vcvtnq_m_s, vcvtnq_m_u])
4859 ;;
4860 (define_insn "mve_vcvtnq_m_<supf><mode>"
4861   [
4862    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4863         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4864                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4865                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4866          VCVTNQ_M))
4867   ]
4868   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4869   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4870   [(set_attr "type" "mve_move")
4871    (set_attr "length""8")])
4872
4873 ;;
4874 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
4875 ;;
4876 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
4877   [
4878    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4879         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4880                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4881                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
4882                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4883          VCVTQ_M_N_FROM_F))
4884   ]
4885   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4886   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
4887   [(set_attr "type" "mve_move")
4888    (set_attr "length""8")])
4889
4890 ;;
4891 ;; [vrev16q_m_u, vrev16q_m_s])
4892 ;;
4893 (define_insn "mve_vrev16q_m_<supf>v16qi"
4894   [
4895    (set (match_operand:V16QI 0 "s_register_operand" "=w")
4896         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
4897                        (match_operand:V16QI 2 "s_register_operand" "w")
4898                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4899          VREV16Q_M))
4900   ]
4901   "TARGET_HAVE_MVE"
4902   "vpst\;vrev16t.8 %q0, %q2"
4903   [(set_attr "type" "mve_move")
4904    (set_attr "length""8")])
4905
4906 ;;
4907 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
4908 ;;
4909 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
4910   [
4911    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4912         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4913                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4914                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4915          VCVTQ_M_FROM_F))
4916   ]
4917   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4918   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4919   [(set_attr "type" "mve_move")
4920    (set_attr "length""8")])
4921
4922 ;;
4923 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
4924 ;;
4925 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
4926   [
4927    (set (match_operand:DI 0 "s_register_operand" "=r")
4928         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4929                     (match_operand:V4SI 2 "s_register_operand" "w")
4930                     (match_operand:HI 3 "vpr_register_operand" "Up")]
4931          VRMLALDAVHQ_P))
4932   ]
4933   "TARGET_HAVE_MVE"
4934   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
4935   [(set_attr "type" "mve_move")
4936    (set_attr "length""8")])
4937
4938 ;;
4939 ;; [vrmlsldavhaq_s])
4940 ;;
4941 (define_insn "mve_vrmlsldavhaq_sv4si"
4942   [
4943    (set (match_operand:DI 0 "s_register_operand" "=r")
4944         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4945                     (match_operand:V4SI 2 "s_register_operand" "w")
4946                     (match_operand:V4SI 3 "s_register_operand" "w")]
4947          VRMLSLDAVHAQ_S))
4948   ]
4949   "TARGET_HAVE_MVE"
4950   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
4951   [(set_attr "type" "mve_move")
4952 ])
4953
4954 ;;
4955 ;; [vabavq_p_s, vabavq_p_u])
4956 ;;
4957 (define_insn "mve_vabavq_p_<supf><mode>"
4958   [
4959    (set (match_operand:SI 0 "s_register_operand" "=r")
4960         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4961                     (match_operand:MVE_2 2 "s_register_operand" "w")
4962                     (match_operand:MVE_2 3 "s_register_operand" "w")
4963                     (match_operand:HI 4 "vpr_register_operand" "Up")]
4964          VABAVQ_P))
4965   ]
4966   "TARGET_HAVE_MVE"
4967   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
4968   [(set_attr "type" "mve_move")
4969 ])
4970
4971 ;;
4972 ;; [vqshluq_m_n_s])
4973 ;;
4974 (define_insn "mve_vqshluq_m_n_s<mode>"
4975   [
4976    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4977         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4978                        (match_operand:MVE_2 2 "s_register_operand" "w")
4979                        (match_operand:SI 3 "mve_imm_7" "Ra")
4980                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4981          VQSHLUQ_M_N_S))
4982   ]
4983   "TARGET_HAVE_MVE"
4984   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
4985   [(set_attr "type" "mve_move")])
4986
4987 ;;
4988 ;; [vshlq_m_s, vshlq_m_u])
4989 ;;
4990 (define_insn "mve_vshlq_m_<supf><mode>"
4991   [
4992    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4993         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4994                        (match_operand:MVE_2 2 "s_register_operand" "w")
4995                        (match_operand:MVE_2 3 "s_register_operand" "w")
4996                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4997          VSHLQ_M))
4998   ]
4999   "TARGET_HAVE_MVE"
5000   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5001   [(set_attr "type" "mve_move")])
5002
5003 ;;
5004 ;; [vsriq_m_n_s, vsriq_m_n_u])
5005 ;;
5006 (define_insn "mve_vsriq_m_n_<supf><mode>"
5007   [
5008    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5009         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5010                        (match_operand:MVE_2 2 "s_register_operand" "w")
5011                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5012                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5013          VSRIQ_M_N))
5014   ]
5015   "TARGET_HAVE_MVE"
5016   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5017   [(set_attr "type" "mve_move")])
5018
5019 ;;
5020 ;; [vsubq_m_u, vsubq_m_s])
5021 ;;
5022 (define_insn "mve_vsubq_m_<supf><mode>"
5023   [
5024    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5025         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5026                        (match_operand:MVE_2 2 "s_register_operand" "w")
5027                        (match_operand:MVE_2 3 "s_register_operand" "w")
5028                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5029          VSUBQ_M))
5030   ]
5031   "TARGET_HAVE_MVE"
5032   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5033   [(set_attr "type" "mve_move")])
5034
5035 ;;
5036 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5037 ;;
5038 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5039   [
5040    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5041         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5042                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5043                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5044                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5045          VCVTQ_M_N_TO_F))
5046   ]
5047   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5048   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5049   [(set_attr "type" "mve_move")
5050    (set_attr "length""8")])
5051 ;;
5052 ;; [vabdq_m_s, vabdq_m_u])
5053 ;;
5054 (define_insn "mve_vabdq_m_<supf><mode>"
5055   [
5056    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5057         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5058                        (match_operand:MVE_2 2 "s_register_operand" "w")
5059                        (match_operand:MVE_2 3 "s_register_operand" "w")
5060                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5061          VABDQ_M))
5062   ]
5063   "TARGET_HAVE_MVE"
5064   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5065   [(set_attr "type" "mve_move")
5066    (set_attr "length""8")])
5067
5068 ;;
5069 ;; [vaddq_m_n_s, vaddq_m_n_u])
5070 ;;
5071 (define_insn "mve_vaddq_m_n_<supf><mode>"
5072   [
5073    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5074         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5075                        (match_operand:MVE_2 2 "s_register_operand" "w")
5076                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5077                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5078          VADDQ_M_N))
5079   ]
5080   "TARGET_HAVE_MVE"
5081   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5082   [(set_attr "type" "mve_move")
5083    (set_attr "length""8")])
5084
5085 ;;
5086 ;; [vaddq_m_u, vaddq_m_s])
5087 ;;
5088 (define_insn "mve_vaddq_m_<supf><mode>"
5089   [
5090    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5091         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5092                        (match_operand:MVE_2 2 "s_register_operand" "w")
5093                        (match_operand:MVE_2 3 "s_register_operand" "w")
5094                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5095          VADDQ_M))
5096   ]
5097   "TARGET_HAVE_MVE"
5098   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5099   [(set_attr "type" "mve_move")
5100    (set_attr "length""8")])
5101
5102 ;;
5103 ;; [vandq_m_u, vandq_m_s])
5104 ;;
5105 (define_insn "mve_vandq_m_<supf><mode>"
5106   [
5107    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5108         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5109                        (match_operand:MVE_2 2 "s_register_operand" "w")
5110                        (match_operand:MVE_2 3 "s_register_operand" "w")
5111                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5112          VANDQ_M))
5113   ]
5114   "TARGET_HAVE_MVE"
5115   "vpst\;vandt %q0, %q2, %q3"
5116   [(set_attr "type" "mve_move")
5117    (set_attr "length""8")])
5118
5119 ;;
5120 ;; [vbicq_m_u, vbicq_m_s])
5121 ;;
5122 (define_insn "mve_vbicq_m_<supf><mode>"
5123   [
5124    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5125         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5126                        (match_operand:MVE_2 2 "s_register_operand" "w")
5127                        (match_operand:MVE_2 3 "s_register_operand" "w")
5128                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5129          VBICQ_M))
5130   ]
5131   "TARGET_HAVE_MVE"
5132   "vpst\;vbict %q0, %q2, %q3"
5133   [(set_attr "type" "mve_move")
5134    (set_attr "length""8")])
5135
5136 ;;
5137 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5138 ;;
5139 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5140   [
5141    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5142         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5143                        (match_operand:MVE_2 2 "s_register_operand" "w")
5144                        (match_operand:SI 3 "s_register_operand" "r")
5145                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5146          VBRSRQ_M_N))
5147   ]
5148   "TARGET_HAVE_MVE"
5149   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5150   [(set_attr "type" "mve_move")
5151    (set_attr "length""8")])
5152
5153 ;;
5154 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5155 ;;
5156 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5157   [
5158    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5159         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5160                        (match_operand:MVE_2 2 "s_register_operand" "w")
5161                        (match_operand:MVE_2 3 "s_register_operand" "w")
5162                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5163          VCADDQ_ROT270_M))
5164   ]
5165   "TARGET_HAVE_MVE"
5166   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5167   [(set_attr "type" "mve_move")
5168    (set_attr "length""8")])
5169
5170 ;;
5171 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5172 ;;
5173 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5174   [
5175    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5176         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5177                        (match_operand:MVE_2 2 "s_register_operand" "w")
5178                        (match_operand:MVE_2 3 "s_register_operand" "w")
5179                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5180          VCADDQ_ROT90_M))
5181   ]
5182   "TARGET_HAVE_MVE"
5183   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5184   [(set_attr "type" "mve_move")
5185    (set_attr "length""8")])
5186
5187 ;;
5188 ;; [veorq_m_s, veorq_m_u])
5189 ;;
5190 (define_insn "mve_veorq_m_<supf><mode>"
5191   [
5192    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5193         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5194                        (match_operand:MVE_2 2 "s_register_operand" "w")
5195                        (match_operand:MVE_2 3 "s_register_operand" "w")
5196                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5197          VEORQ_M))
5198   ]
5199   "TARGET_HAVE_MVE"
5200   "vpst\;veort %q0, %q2, %q3"
5201   [(set_attr "type" "mve_move")
5202    (set_attr "length""8")])
5203
5204 ;;
5205 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5206 ;;
5207 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5208   [
5209    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5210         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5211                        (match_operand:MVE_2 2 "s_register_operand" "w")
5212                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5213                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5214          VHADDQ_M_N))
5215   ]
5216   "TARGET_HAVE_MVE"
5217   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5218   [(set_attr "type" "mve_move")
5219    (set_attr "length""8")])
5220
5221 ;;
5222 ;; [vhaddq_m_s, vhaddq_m_u])
5223 ;;
5224 (define_insn "mve_vhaddq_m_<supf><mode>"
5225   [
5226    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5227         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5228                        (match_operand:MVE_2 2 "s_register_operand" "w")
5229                        (match_operand:MVE_2 3 "s_register_operand" "w")
5230                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5231          VHADDQ_M))
5232   ]
5233   "TARGET_HAVE_MVE"
5234   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5235   [(set_attr "type" "mve_move")
5236    (set_attr "length""8")])
5237
5238 ;;
5239 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5240 ;;
5241 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5242   [
5243    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5244         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5245                        (match_operand:MVE_2 2 "s_register_operand" "w")
5246                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5247                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5248          VHSUBQ_M_N))
5249   ]
5250   "TARGET_HAVE_MVE"
5251   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5252   [(set_attr "type" "mve_move")
5253    (set_attr "length""8")])
5254
5255 ;;
5256 ;; [vhsubq_m_s, vhsubq_m_u])
5257 ;;
5258 (define_insn "mve_vhsubq_m_<supf><mode>"
5259   [
5260    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5261         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5262                        (match_operand:MVE_2 2 "s_register_operand" "w")
5263                        (match_operand:MVE_2 3 "s_register_operand" "w")
5264                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5265          VHSUBQ_M))
5266   ]
5267   "TARGET_HAVE_MVE"
5268   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5269   [(set_attr "type" "mve_move")
5270    (set_attr "length""8")])
5271
5272 ;;
5273 ;; [vmaxq_m_s, vmaxq_m_u])
5274 ;;
5275 (define_insn "mve_vmaxq_m_<supf><mode>"
5276   [
5277    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5278         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5279                        (match_operand:MVE_2 2 "s_register_operand" "w")
5280                        (match_operand:MVE_2 3 "s_register_operand" "w")
5281                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5282          VMAXQ_M))
5283   ]
5284   "TARGET_HAVE_MVE"
5285   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5286   [(set_attr "type" "mve_move")
5287    (set_attr "length""8")])
5288
5289 ;;
5290 ;; [vminq_m_s, vminq_m_u])
5291 ;;
5292 (define_insn "mve_vminq_m_<supf><mode>"
5293   [
5294    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5295         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5296                        (match_operand:MVE_2 2 "s_register_operand" "w")
5297                        (match_operand:MVE_2 3 "s_register_operand" "w")
5298                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5299          VMINQ_M))
5300   ]
5301   "TARGET_HAVE_MVE"
5302   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5303   [(set_attr "type" "mve_move")
5304    (set_attr "length""8")])
5305
5306 ;;
5307 ;; [vmladavaq_p_u, vmladavaq_p_s])
5308 ;;
5309 (define_insn "mve_vmladavaq_p_<supf><mode>"
5310   [
5311    (set (match_operand:SI 0 "s_register_operand" "=Te")
5312         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5313                     (match_operand:MVE_2 2 "s_register_operand" "w")
5314                     (match_operand:MVE_2 3 "s_register_operand" "w")
5315                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5316          VMLADAVAQ_P))
5317   ]
5318   "TARGET_HAVE_MVE"
5319   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5320   [(set_attr "type" "mve_move")
5321    (set_attr "length""8")])
5322
5323 ;;
5324 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5325 ;;
5326 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5327   [
5328    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5329         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5330                        (match_operand:MVE_2 2 "s_register_operand" "w")
5331                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5332                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5333          VMLAQ_M_N))
5334   ]
5335   "TARGET_HAVE_MVE"
5336   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5337   [(set_attr "type" "mve_move")
5338    (set_attr "length""8")])
5339
5340 ;;
5341 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5342 ;;
5343 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5344   [
5345    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5346         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5347                        (match_operand:MVE_2 2 "s_register_operand" "w")
5348                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5349                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5350          VMLASQ_M_N))
5351   ]
5352   "TARGET_HAVE_MVE"
5353   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5354   [(set_attr "type" "mve_move")
5355    (set_attr "length""8")])
5356
5357 ;;
5358 ;; [vmulhq_m_s, vmulhq_m_u])
5359 ;;
5360 (define_insn "mve_vmulhq_m_<supf><mode>"
5361   [
5362    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5363         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5364                        (match_operand:MVE_2 2 "s_register_operand" "w")
5365                        (match_operand:MVE_2 3 "s_register_operand" "w")
5366                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5367          VMULHQ_M))
5368   ]
5369   "TARGET_HAVE_MVE"
5370   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5371   [(set_attr "type" "mve_move")
5372    (set_attr "length""8")])
5373
5374 ;;
5375 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5376 ;;
5377 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5378   [
5379    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5380         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5381                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5382                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5383                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5384          VMULLBQ_INT_M))
5385   ]
5386   "TARGET_HAVE_MVE"
5387   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5388   [(set_attr "type" "mve_move")
5389    (set_attr "length""8")])
5390
5391 ;;
5392 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5393 ;;
5394 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5395   [
5396    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5397         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5398                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5399                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5400                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5401          VMULLTQ_INT_M))
5402   ]
5403   "TARGET_HAVE_MVE"
5404   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5405   [(set_attr "type" "mve_move")
5406    (set_attr "length""8")])
5407
5408 ;;
5409 ;; [vmulq_m_n_u, vmulq_m_n_s])
5410 ;;
5411 (define_insn "mve_vmulq_m_n_<supf><mode>"
5412   [
5413    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5414         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5415                        (match_operand:MVE_2 2 "s_register_operand" "w")
5416                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5417                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5418          VMULQ_M_N))
5419   ]
5420   "TARGET_HAVE_MVE"
5421   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5422   [(set_attr "type" "mve_move")
5423    (set_attr "length""8")])
5424
5425 ;;
5426 ;; [vmulq_m_s, vmulq_m_u])
5427 ;;
5428 (define_insn "mve_vmulq_m_<supf><mode>"
5429   [
5430    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5431         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5432                        (match_operand:MVE_2 2 "s_register_operand" "w")
5433                        (match_operand:MVE_2 3 "s_register_operand" "w")
5434                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5435          VMULQ_M))
5436   ]
5437   "TARGET_HAVE_MVE"
5438   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5439   [(set_attr "type" "mve_move")
5440    (set_attr "length""8")])
5441
5442 ;;
5443 ;; [vornq_m_u, vornq_m_s])
5444 ;;
5445 (define_insn "mve_vornq_m_<supf><mode>"
5446   [
5447    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5448         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5449                        (match_operand:MVE_2 2 "s_register_operand" "w")
5450                        (match_operand:MVE_2 3 "s_register_operand" "w")
5451                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5452          VORNQ_M))
5453   ]
5454   "TARGET_HAVE_MVE"
5455   "vpst\;vornt %q0, %q2, %q3"
5456   [(set_attr "type" "mve_move")
5457    (set_attr "length""8")])
5458
5459 ;;
5460 ;; [vorrq_m_s, vorrq_m_u])
5461 ;;
5462 (define_insn "mve_vorrq_m_<supf><mode>"
5463   [
5464    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5465         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5466                        (match_operand:MVE_2 2 "s_register_operand" "w")
5467                        (match_operand:MVE_2 3 "s_register_operand" "w")
5468                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5469          VORRQ_M))
5470   ]
5471   "TARGET_HAVE_MVE"
5472   "vpst\;vorrt %q0, %q2, %q3"
5473   [(set_attr "type" "mve_move")
5474    (set_attr "length""8")])
5475
5476 ;;
5477 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5478 ;;
5479 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5480   [
5481    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5482         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5483                        (match_operand:MVE_2 2 "s_register_operand" "w")
5484                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5485                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5486          VQADDQ_M_N))
5487   ]
5488   "TARGET_HAVE_MVE"
5489   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5490   [(set_attr "type" "mve_move")
5491    (set_attr "length""8")])
5492
5493 ;;
5494 ;; [vqaddq_m_u, vqaddq_m_s])
5495 ;;
5496 (define_insn "mve_vqaddq_m_<supf><mode>"
5497   [
5498    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5499         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5500                        (match_operand:MVE_2 2 "s_register_operand" "w")
5501                        (match_operand:MVE_2 3 "s_register_operand" "w")
5502                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5503          VQADDQ_M))
5504   ]
5505   "TARGET_HAVE_MVE"
5506   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5507   [(set_attr "type" "mve_move")
5508    (set_attr "length""8")])
5509
5510 ;;
5511 ;; [vqdmlahq_m_n_s])
5512 ;;
5513 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5514   [
5515    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5516         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5517                        (match_operand:MVE_2 2 "s_register_operand" "w")
5518                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5519                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5520          VQDMLAHQ_M_N_S))
5521   ]
5522   "TARGET_HAVE_MVE"
5523   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5524   [(set_attr "type" "mve_move")
5525    (set_attr "length""8")])
5526
5527 ;;
5528 ;; [vqdmlashq_m_n_s])
5529 ;;
5530 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5531   [
5532    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5533         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5534                        (match_operand:MVE_2 2 "s_register_operand" "w")
5535                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5536                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5537          VQDMLASHQ_M_N_S))
5538   ]
5539   "TARGET_HAVE_MVE"
5540   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5541   [(set_attr "type" "mve_move")
5542    (set_attr "length""8")])
5543
5544 ;;
5545 ;; [vqrdmlahq_m_n_s])
5546 ;;
5547 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5548   [
5549    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5550         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5551                        (match_operand:MVE_2 2 "s_register_operand" "w")
5552                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5553                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5554          VQRDMLAHQ_M_N_S))
5555   ]
5556   "TARGET_HAVE_MVE"
5557   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5558   [(set_attr "type" "mve_move")
5559    (set_attr "length""8")])
5560
5561 ;;
5562 ;; [vqrdmlashq_m_n_s])
5563 ;;
5564 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5565   [
5566    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5567         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5568                        (match_operand:MVE_2 2 "s_register_operand" "w")
5569                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5570                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5571          VQRDMLASHQ_M_N_S))
5572   ]
5573   "TARGET_HAVE_MVE"
5574   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5575   [(set_attr "type" "mve_move")
5576    (set_attr "length""8")])
5577
5578 ;;
5579 ;; [vqrshlq_m_u, vqrshlq_m_s])
5580 ;;
5581 (define_insn "mve_vqrshlq_m_<supf><mode>"
5582   [
5583    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5584         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5585                        (match_operand:MVE_2 2 "s_register_operand" "w")
5586                        (match_operand:MVE_2 3 "s_register_operand" "w")
5587                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5588          VQRSHLQ_M))
5589   ]
5590   "TARGET_HAVE_MVE"
5591   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5592   [(set_attr "type" "mve_move")
5593    (set_attr "length""8")])
5594
5595 ;;
5596 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5597 ;;
5598 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5599   [
5600    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5601         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5602                        (match_operand:MVE_2 2 "s_register_operand" "w")
5603                        (match_operand:SI 3 "immediate_operand" "i")
5604                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5605          VQSHLQ_M_N))
5606   ]
5607   "TARGET_HAVE_MVE"
5608   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5609   [(set_attr "type" "mve_move")
5610    (set_attr "length""8")])
5611
5612 ;;
5613 ;; [vqshlq_m_u, vqshlq_m_s])
5614 ;;
5615 (define_insn "mve_vqshlq_m_<supf><mode>"
5616   [
5617    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5618         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5619                        (match_operand:MVE_2 2 "s_register_operand" "w")
5620                        (match_operand:MVE_2 3 "s_register_operand" "w")
5621                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5622          VQSHLQ_M))
5623   ]
5624   "TARGET_HAVE_MVE"
5625   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5626   [(set_attr "type" "mve_move")
5627    (set_attr "length""8")])
5628
5629 ;;
5630 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
5631 ;;
5632 (define_insn "mve_vqsubq_m_n_<supf><mode>"
5633   [
5634    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5635         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5636                        (match_operand:MVE_2 2 "s_register_operand" "w")
5637                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5638                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5639          VQSUBQ_M_N))
5640   ]
5641   "TARGET_HAVE_MVE"
5642   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5643   [(set_attr "type" "mve_move")
5644    (set_attr "length""8")])
5645
5646 ;;
5647 ;; [vqsubq_m_u, vqsubq_m_s])
5648 ;;
5649 (define_insn "mve_vqsubq_m_<supf><mode>"
5650   [
5651    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5652         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5653                        (match_operand:MVE_2 2 "s_register_operand" "w")
5654                        (match_operand:MVE_2 3 "s_register_operand" "w")
5655                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5656          VQSUBQ_M))
5657   ]
5658   "TARGET_HAVE_MVE"
5659   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5660   [(set_attr "type" "mve_move")
5661    (set_attr "length""8")])
5662
5663 ;;
5664 ;; [vrhaddq_m_u, vrhaddq_m_s])
5665 ;;
5666 (define_insn "mve_vrhaddq_m_<supf><mode>"
5667   [
5668    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5669         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5670                        (match_operand:MVE_2 2 "s_register_operand" "w")
5671                        (match_operand:MVE_2 3 "s_register_operand" "w")
5672                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5673          VRHADDQ_M))
5674   ]
5675   "TARGET_HAVE_MVE"
5676   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5677   [(set_attr "type" "mve_move")
5678    (set_attr "length""8")])
5679
5680 ;;
5681 ;; [vrmulhq_m_u, vrmulhq_m_s])
5682 ;;
5683 (define_insn "mve_vrmulhq_m_<supf><mode>"
5684   [
5685    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5686         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5687                        (match_operand:MVE_2 2 "s_register_operand" "w")
5688                        (match_operand:MVE_2 3 "s_register_operand" "w")
5689                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5690          VRMULHQ_M))
5691   ]
5692   "TARGET_HAVE_MVE"
5693   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5694   [(set_attr "type" "mve_move")
5695    (set_attr "length""8")])
5696
5697 ;;
5698 ;; [vrshlq_m_s, vrshlq_m_u])
5699 ;;
5700 (define_insn "mve_vrshlq_m_<supf><mode>"
5701   [
5702    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5703         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5704                        (match_operand:MVE_2 2 "s_register_operand" "w")
5705                        (match_operand:MVE_2 3 "s_register_operand" "w")
5706                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5707          VRSHLQ_M))
5708   ]
5709   "TARGET_HAVE_MVE"
5710   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5711   [(set_attr "type" "mve_move")
5712    (set_attr "length""8")])
5713
5714 ;;
5715 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
5716 ;;
5717 (define_insn "mve_vrshrq_m_n_<supf><mode>"
5718   [
5719    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5720         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5721                        (match_operand:MVE_2 2 "s_register_operand" "w")
5722                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5723                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5724          VRSHRQ_M_N))
5725   ]
5726   "TARGET_HAVE_MVE"
5727   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5728   [(set_attr "type" "mve_move")
5729    (set_attr "length""8")])
5730
5731 ;;
5732 ;; [vshlq_m_n_s, vshlq_m_n_u])
5733 ;;
5734 (define_insn "mve_vshlq_m_n_<supf><mode>"
5735   [
5736    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5737         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5738                        (match_operand:MVE_2 2 "s_register_operand" "w")
5739                        (match_operand:SI 3 "immediate_operand" "i")
5740                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5741          VSHLQ_M_N))
5742   ]
5743   "TARGET_HAVE_MVE"
5744   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5745   [(set_attr "type" "mve_move")
5746    (set_attr "length""8")])
5747
5748 ;;
5749 ;; [vshrq_m_n_s, vshrq_m_n_u])
5750 ;;
5751 (define_insn "mve_vshrq_m_n_<supf><mode>"
5752   [
5753    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5754         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5755                        (match_operand:MVE_2 2 "s_register_operand" "w")
5756                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5757                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5758          VSHRQ_M_N))
5759   ]
5760   "TARGET_HAVE_MVE"
5761   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5762   [(set_attr "type" "mve_move")
5763    (set_attr "length""8")])
5764
5765 ;;
5766 ;; [vsliq_m_n_u, vsliq_m_n_s])
5767 ;;
5768 (define_insn "mve_vsliq_m_n_<supf><mode>"
5769    [
5770    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5771        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5772                        (match_operand:MVE_2 2 "s_register_operand" "w")
5773                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
5774                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5775          VSLIQ_M_N))
5776   ]
5777   "TARGET_HAVE_MVE"
5778   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
5779   [(set_attr "type" "mve_move")
5780    (set_attr "length""8")])
5781
5782 ;;
5783 ;; [vsubq_m_n_s, vsubq_m_n_u])
5784 ;;
5785 (define_insn "mve_vsubq_m_n_<supf><mode>"
5786   [
5787    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5788         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5789                        (match_operand:MVE_2 2 "s_register_operand" "w")
5790                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5791                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5792          VSUBQ_M_N))
5793   ]
5794   "TARGET_HAVE_MVE"
5795   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
5796   [(set_attr "type" "mve_move")
5797    (set_attr "length""8")])
5798
5799 ;;
5800 ;; [vhcaddq_rot270_m_s])
5801 ;;
5802 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
5803   [
5804    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5805         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5806                        (match_operand:MVE_2 2 "s_register_operand" "w")
5807                        (match_operand:MVE_2 3 "s_register_operand" "w")
5808                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5809          VHCADDQ_ROT270_M_S))
5810   ]
5811   "TARGET_HAVE_MVE"
5812   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
5813   [(set_attr "type" "mve_move")
5814    (set_attr "length""8")])
5815
5816 ;;
5817 ;; [vhcaddq_rot90_m_s])
5818 ;;
5819 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
5820   [
5821    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5822         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5823                        (match_operand:MVE_2 2 "s_register_operand" "w")
5824                        (match_operand:MVE_2 3 "s_register_operand" "w")
5825                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5826          VHCADDQ_ROT90_M_S))
5827   ]
5828   "TARGET_HAVE_MVE"
5829   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
5830   [(set_attr "type" "mve_move")
5831    (set_attr "length""8")])
5832
5833 ;;
5834 ;; [vmladavaxq_p_s])
5835 ;;
5836 (define_insn "mve_vmladavaxq_p_s<mode>"
5837   [
5838    (set (match_operand:SI 0 "s_register_operand" "=Te")
5839         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5840                        (match_operand:MVE_2 2 "s_register_operand" "w")
5841                        (match_operand:MVE_2 3 "s_register_operand" "w")
5842                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5843          VMLADAVAXQ_P_S))
5844   ]
5845   "TARGET_HAVE_MVE"
5846   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5847   [(set_attr "type" "mve_move")
5848    (set_attr "length""8")])
5849
5850 ;;
5851 ;; [vmlsdavaq_p_s])
5852 ;;
5853 (define_insn "mve_vmlsdavaq_p_s<mode>"
5854   [
5855    (set (match_operand:SI 0 "s_register_operand" "=Te")
5856         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5857                        (match_operand:MVE_2 2 "s_register_operand" "w")
5858                        (match_operand:MVE_2 3 "s_register_operand" "w")
5859                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5860          VMLSDAVAQ_P_S))
5861   ]
5862   "TARGET_HAVE_MVE"
5863   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
5864   [(set_attr "type" "mve_move")
5865    (set_attr "length""8")])
5866
5867 ;;
5868 ;; [vmlsdavaxq_p_s])
5869 ;;
5870 (define_insn "mve_vmlsdavaxq_p_s<mode>"
5871   [
5872    (set (match_operand:SI 0 "s_register_operand" "=Te")
5873         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5874                        (match_operand:MVE_2 2 "s_register_operand" "w")
5875                        (match_operand:MVE_2 3 "s_register_operand" "w")
5876                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5877          VMLSDAVAXQ_P_S))
5878   ]
5879   "TARGET_HAVE_MVE"
5880   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5881   [(set_attr "type" "mve_move")
5882    (set_attr "length""8")])
5883
5884 ;;
5885 ;; [vqdmladhq_m_s])
5886 ;;
5887 (define_insn "mve_vqdmladhq_m_s<mode>"
5888   [
5889    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5890         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5891                        (match_operand:MVE_2 2 "s_register_operand" "w")
5892                        (match_operand:MVE_2 3 "s_register_operand" "w")
5893                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5894          VQDMLADHQ_M_S))
5895   ]
5896   "TARGET_HAVE_MVE"
5897   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5898   [(set_attr "type" "mve_move")
5899    (set_attr "length""8")])
5900
5901 ;;
5902 ;; [vqdmladhxq_m_s])
5903 ;;
5904 (define_insn "mve_vqdmladhxq_m_s<mode>"
5905   [
5906    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5907         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5908                        (match_operand:MVE_2 2 "s_register_operand" "w")
5909                        (match_operand:MVE_2 3 "s_register_operand" "w")
5910                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5911          VQDMLADHXQ_M_S))
5912   ]
5913   "TARGET_HAVE_MVE"
5914   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5915   [(set_attr "type" "mve_move")
5916    (set_attr "length""8")])
5917
5918 ;;
5919 ;; [vqdmlsdhq_m_s])
5920 ;;
5921 (define_insn "mve_vqdmlsdhq_m_s<mode>"
5922   [
5923    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5924         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5925                        (match_operand:MVE_2 2 "s_register_operand" "w")
5926                        (match_operand:MVE_2 3 "s_register_operand" "w")
5927                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5928          VQDMLSDHQ_M_S))
5929   ]
5930   "TARGET_HAVE_MVE"
5931   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5932   [(set_attr "type" "mve_move")
5933    (set_attr "length""8")])
5934
5935 ;;
5936 ;; [vqdmlsdhxq_m_s])
5937 ;;
5938 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
5939   [
5940    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5941         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5942                        (match_operand:MVE_2 2 "s_register_operand" "w")
5943                        (match_operand:MVE_2 3 "s_register_operand" "w")
5944                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5945          VQDMLSDHXQ_M_S))
5946   ]
5947   "TARGET_HAVE_MVE"
5948   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5949   [(set_attr "type" "mve_move")
5950    (set_attr "length""8")])
5951
5952 ;;
5953 ;; [vqdmulhq_m_n_s])
5954 ;;
5955 (define_insn "mve_vqdmulhq_m_n_s<mode>"
5956   [
5957    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5958         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5959                        (match_operand:MVE_2 2 "s_register_operand" "w")
5960                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5961                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5962          VQDMULHQ_M_N_S))
5963   ]
5964   "TARGET_HAVE_MVE"
5965   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
5966   [(set_attr "type" "mve_move")
5967    (set_attr "length""8")])
5968
5969 ;;
5970 ;; [vqdmulhq_m_s])
5971 ;;
5972 (define_insn "mve_vqdmulhq_m_s<mode>"
5973   [
5974    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5975         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5976                        (match_operand:MVE_2 2 "s_register_operand" "w")
5977                        (match_operand:MVE_2 3 "s_register_operand" "w")
5978                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5979          VQDMULHQ_M_S))
5980   ]
5981   "TARGET_HAVE_MVE"
5982   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5983   [(set_attr "type" "mve_move")
5984    (set_attr "length""8")])
5985
5986 ;;
5987 ;; [vqrdmladhq_m_s])
5988 ;;
5989 (define_insn "mve_vqrdmladhq_m_s<mode>"
5990   [
5991    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5992         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5993                        (match_operand:MVE_2 2 "s_register_operand" "w")
5994                        (match_operand:MVE_2 3 "s_register_operand" "w")
5995                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5996          VQRDMLADHQ_M_S))
5997   ]
5998   "TARGET_HAVE_MVE"
5999   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6000   [(set_attr "type" "mve_move")
6001    (set_attr "length""8")])
6002
6003 ;;
6004 ;; [vqrdmladhxq_m_s])
6005 ;;
6006 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6007   [
6008    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6009         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6010                        (match_operand:MVE_2 2 "s_register_operand" "w")
6011                        (match_operand:MVE_2 3 "s_register_operand" "w")
6012                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6013          VQRDMLADHXQ_M_S))
6014   ]
6015   "TARGET_HAVE_MVE"
6016   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6017   [(set_attr "type" "mve_move")
6018    (set_attr "length""8")])
6019
6020 ;;
6021 ;; [vqrdmlsdhq_m_s])
6022 ;;
6023 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6024   [
6025    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6026         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6027                        (match_operand:MVE_2 2 "s_register_operand" "w")
6028                        (match_operand:MVE_2 3 "s_register_operand" "w")
6029                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6030          VQRDMLSDHQ_M_S))
6031   ]
6032   "TARGET_HAVE_MVE"
6033   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6034   [(set_attr "type" "mve_move")
6035    (set_attr "length""8")])
6036
6037 ;;
6038 ;; [vqrdmlsdhxq_m_s])
6039 ;;
6040 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6041   [
6042    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6043         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6044                        (match_operand:MVE_2 2 "s_register_operand" "w")
6045                        (match_operand:MVE_2 3 "s_register_operand" "w")
6046                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6047          VQRDMLSDHXQ_M_S))
6048   ]
6049   "TARGET_HAVE_MVE"
6050   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6051   [(set_attr "type" "mve_move")
6052    (set_attr "length""8")])
6053
6054 ;;
6055 ;; [vqrdmulhq_m_n_s])
6056 ;;
6057 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6058   [
6059    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6060         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6061                        (match_operand:MVE_2 2 "s_register_operand" "w")
6062                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6063                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6064          VQRDMULHQ_M_N_S))
6065   ]
6066   "TARGET_HAVE_MVE"
6067   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6068   [(set_attr "type" "mve_move")
6069    (set_attr "length""8")])
6070
6071 ;;
6072 ;; [vqrdmulhq_m_s])
6073 ;;
6074 (define_insn "mve_vqrdmulhq_m_s<mode>"
6075   [
6076    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6077         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6078                        (match_operand:MVE_2 2 "s_register_operand" "w")
6079                        (match_operand:MVE_2 3 "s_register_operand" "w")
6080                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6081          VQRDMULHQ_M_S))
6082   ]
6083   "TARGET_HAVE_MVE"
6084   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6085   [(set_attr "type" "mve_move")
6086    (set_attr "length""8")])
6087
6088 ;;
6089 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6090 ;;
6091 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6092   [
6093    (set (match_operand:DI 0 "s_register_operand" "=r")
6094         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6095                        (match_operand:MVE_5 2 "s_register_operand" "w")
6096                        (match_operand:MVE_5 3 "s_register_operand" "w")
6097                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6098          VMLALDAVAQ_P))
6099   ]
6100   "TARGET_HAVE_MVE"
6101   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6102   [(set_attr "type" "mve_move")
6103    (set_attr "length""8")])
6104
6105 ;;
6106 ;; [vmlaldavaxq_p_s])
6107 ;;
6108 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6109   [
6110    (set (match_operand:DI 0 "s_register_operand" "=r")
6111         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6112                        (match_operand:MVE_5 2 "s_register_operand" "w")
6113                        (match_operand:MVE_5 3 "s_register_operand" "w")
6114                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6115          VMLALDAVAXQ_P))
6116   ]
6117   "TARGET_HAVE_MVE"
6118   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6119   [(set_attr "type" "mve_move")
6120    (set_attr "length""8")])
6121
6122 ;;
6123 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6124 ;;
6125 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6126   [
6127    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6128         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6129                        (match_operand:MVE_5 2 "s_register_operand" "w")
6130                        (match_operand:SI 3 "mve_imm_8" "Rb")
6131                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6132          VQRSHRNBQ_M_N))
6133   ]
6134   "TARGET_HAVE_MVE"
6135   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6136   [(set_attr "type" "mve_move")
6137    (set_attr "length""8")])
6138
6139 ;;
6140 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6141 ;;
6142 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6143   [
6144    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6145         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6146                        (match_operand:MVE_5 2 "s_register_operand" "w")
6147                        (match_operand:SI 3 "mve_imm_8" "Rb")
6148                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6149          VQRSHRNTQ_M_N))
6150   ]
6151   "TARGET_HAVE_MVE"
6152   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6153   [(set_attr "type" "mve_move")
6154    (set_attr "length""8")])
6155
6156 ;;
6157 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6158 ;;
6159 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6160   [
6161    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6162         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6163                        (match_operand:MVE_5 2 "s_register_operand" "w")
6164                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6165                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6166          VQSHRNBQ_M_N))
6167   ]
6168   "TARGET_HAVE_MVE"
6169   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6170   [(set_attr "type" "mve_move")
6171    (set_attr "length""8")])
6172
6173 ;;
6174 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6175 ;;
6176 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6177   [
6178    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6179         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6180                        (match_operand:MVE_5 2 "s_register_operand" "w")
6181                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6182                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6183          VQSHRNTQ_M_N))
6184   ]
6185   "TARGET_HAVE_MVE"
6186   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6187   [(set_attr "type" "mve_move")
6188    (set_attr "length""8")])
6189
6190 ;;
6191 ;; [vrmlaldavhaq_p_s])
6192 ;;
6193 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6194   [
6195    (set (match_operand:DI 0 "s_register_operand" "=r")
6196         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6197                        (match_operand:V4SI 2 "s_register_operand" "w")
6198                        (match_operand:V4SI 3 "s_register_operand" "w")
6199                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6200          VRMLALDAVHAQ_P_S))
6201   ]
6202   "TARGET_HAVE_MVE"
6203   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6204   [(set_attr "type" "mve_move")
6205    (set_attr "length""8")])
6206
6207 ;;
6208 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6209 ;;
6210 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6211   [
6212    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6213         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6214                        (match_operand:MVE_5 2 "s_register_operand" "w")
6215                        (match_operand:SI 3 "mve_imm_8" "Rb")
6216                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6217          VRSHRNBQ_M_N))
6218   ]
6219   "TARGET_HAVE_MVE"
6220   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6221   [(set_attr "type" "mve_move")
6222    (set_attr "length""8")])
6223
6224 ;;
6225 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6226 ;;
6227 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6228   [
6229    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6230         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6231                        (match_operand:MVE_5 2 "s_register_operand" "w")
6232                        (match_operand:SI 3 "mve_imm_8" "Rb")
6233                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6234          VRSHRNTQ_M_N))
6235   ]
6236   "TARGET_HAVE_MVE"
6237   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6238   [(set_attr "type" "mve_move")
6239    (set_attr "length""8")])
6240
6241 ;;
6242 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6243 ;;
6244 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6245   [
6246    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6247         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6248                        (match_operand:MVE_3 2 "s_register_operand" "w")
6249                        (match_operand:SI 3 "immediate_operand" "i")
6250                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6251          VSHLLBQ_M_N))
6252   ]
6253   "TARGET_HAVE_MVE"
6254   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6255   [(set_attr "type" "mve_move")
6256    (set_attr "length""8")])
6257
6258 ;;
6259 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6260 ;;
6261 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6262   [
6263    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6264         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6265                        (match_operand:MVE_3 2 "s_register_operand" "w")
6266                        (match_operand:SI 3 "immediate_operand" "i")
6267                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6268          VSHLLTQ_M_N))
6269   ]
6270   "TARGET_HAVE_MVE"
6271   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6272   [(set_attr "type" "mve_move")
6273    (set_attr "length""8")])
6274
6275 ;;
6276 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6277 ;;
6278 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6279   [
6280    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6281         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6282                        (match_operand:MVE_5 2 "s_register_operand" "w")
6283                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6284                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6285          VSHRNBQ_M_N))
6286   ]
6287   "TARGET_HAVE_MVE"
6288   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6289   [(set_attr "type" "mve_move")
6290    (set_attr "length""8")])
6291
6292 ;;
6293 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6294 ;;
6295 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6296   [
6297    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6298         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6299                        (match_operand:MVE_5 2 "s_register_operand" "w")
6300                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6301                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6302          VSHRNTQ_M_N))
6303   ]
6304   "TARGET_HAVE_MVE"
6305   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6306   [(set_attr "type" "mve_move")
6307    (set_attr "length""8")])
6308
6309 ;;
6310 ;; [vmlsldavaq_p_s])
6311 ;;
6312 (define_insn "mve_vmlsldavaq_p_s<mode>"
6313   [
6314    (set (match_operand:DI 0 "s_register_operand" "=r")
6315         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6316                        (match_operand:MVE_5 2 "s_register_operand" "w")
6317                        (match_operand:MVE_5 3 "s_register_operand" "w")
6318                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6319          VMLSLDAVAQ_P_S))
6320   ]
6321   "TARGET_HAVE_MVE"
6322   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6323   [(set_attr "type" "mve_move")
6324    (set_attr "length""8")])
6325
6326 ;;
6327 ;; [vmlsldavaxq_p_s])
6328 ;;
6329 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6330   [
6331    (set (match_operand:DI 0 "s_register_operand" "=r")
6332         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6333                        (match_operand:MVE_5 2 "s_register_operand" "w")
6334                        (match_operand:MVE_5 3 "s_register_operand" "w")
6335                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6336          VMLSLDAVAXQ_P_S))
6337   ]
6338   "TARGET_HAVE_MVE"
6339   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6340   [(set_attr "type" "mve_move")
6341    (set_attr "length""8")])
6342
6343 ;;
6344 ;; [vmullbq_poly_m_p])
6345 ;;
6346 (define_insn "mve_vmullbq_poly_m_p<mode>"
6347   [
6348    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6349         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6350                        (match_operand:MVE_3 2 "s_register_operand" "w")
6351                        (match_operand:MVE_3 3 "s_register_operand" "w")
6352                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6353          VMULLBQ_POLY_M_P))
6354   ]
6355   "TARGET_HAVE_MVE"
6356   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6357   [(set_attr "type" "mve_move")
6358    (set_attr "length""8")])
6359
6360 ;;
6361 ;; [vmulltq_poly_m_p])
6362 ;;
6363 (define_insn "mve_vmulltq_poly_m_p<mode>"
6364   [
6365    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6366         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6367                        (match_operand:MVE_3 2 "s_register_operand" "w")
6368                        (match_operand:MVE_3 3 "s_register_operand" "w")
6369                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6370          VMULLTQ_POLY_M_P))
6371   ]
6372   "TARGET_HAVE_MVE"
6373   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6374   [(set_attr "type" "mve_move")
6375    (set_attr "length""8")])
6376
6377 ;;
6378 ;; [vqdmullbq_m_n_s])
6379 ;;
6380 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6381   [
6382    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6383         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6384                        (match_operand:MVE_5 2 "s_register_operand" "w")
6385                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6386                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6387          VQDMULLBQ_M_N_S))
6388   ]
6389   "TARGET_HAVE_MVE"
6390   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6391   [(set_attr "type" "mve_move")
6392    (set_attr "length""8")])
6393
6394 ;;
6395 ;; [vqdmullbq_m_s])
6396 ;;
6397 (define_insn "mve_vqdmullbq_m_s<mode>"
6398   [
6399    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6400         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6401                        (match_operand:MVE_5 2 "s_register_operand" "w")
6402                        (match_operand:MVE_5 3 "s_register_operand" "w")
6403                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6404          VQDMULLBQ_M_S))
6405   ]
6406   "TARGET_HAVE_MVE"
6407   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6408   [(set_attr "type" "mve_move")
6409    (set_attr "length""8")])
6410
6411 ;;
6412 ;; [vqdmulltq_m_n_s])
6413 ;;
6414 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6415   [
6416    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6417         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6418                        (match_operand:MVE_5 2 "s_register_operand" "w")
6419                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6420                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6421          VQDMULLTQ_M_N_S))
6422   ]
6423   "TARGET_HAVE_MVE"
6424   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6425   [(set_attr "type" "mve_move")
6426    (set_attr "length""8")])
6427
6428 ;;
6429 ;; [vqdmulltq_m_s])
6430 ;;
6431 (define_insn "mve_vqdmulltq_m_s<mode>"
6432   [
6433    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6434         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6435                        (match_operand:MVE_5 2 "s_register_operand" "w")
6436                        (match_operand:MVE_5 3 "s_register_operand" "w")
6437                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6438          VQDMULLTQ_M_S))
6439   ]
6440   "TARGET_HAVE_MVE"
6441   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6442   [(set_attr "type" "mve_move")
6443    (set_attr "length""8")])
6444
6445 ;;
6446 ;; [vqrshrunbq_m_n_s])
6447 ;;
6448 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6449   [
6450    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6451         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6452                        (match_operand:MVE_5 2 "s_register_operand" "w")
6453                        (match_operand:SI 3 "mve_imm_8" "Rb")
6454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6455          VQRSHRUNBQ_M_N_S))
6456   ]
6457   "TARGET_HAVE_MVE"
6458   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6459   [(set_attr "type" "mve_move")
6460    (set_attr "length""8")])
6461
6462 ;;
6463 ;; [vqrshruntq_m_n_s])
6464 ;;
6465 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6466   [
6467    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6468         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6469                        (match_operand:MVE_5 2 "s_register_operand" "w")
6470                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6471                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6472          VQRSHRUNTQ_M_N_S))
6473   ]
6474   "TARGET_HAVE_MVE"
6475   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6476   [(set_attr "type" "mve_move")
6477    (set_attr "length""8")])
6478
6479 ;;
6480 ;; [vqshrunbq_m_n_s])
6481 ;;
6482 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6483   [
6484    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6485         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6486                        (match_operand:MVE_5 2 "s_register_operand" "w")
6487                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6489          VQSHRUNBQ_M_N_S))
6490   ]
6491   "TARGET_HAVE_MVE"
6492   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6493   [(set_attr "type" "mve_move")
6494    (set_attr "length""8")])
6495
6496 ;;
6497 ;; [vqshruntq_m_n_s])
6498 ;;
6499 (define_insn "mve_vqshruntq_m_n_s<mode>"
6500   [
6501    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6502         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6503                        (match_operand:MVE_5 2 "s_register_operand" "w")
6504                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6505                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6506          VQSHRUNTQ_M_N_S))
6507   ]
6508   "TARGET_HAVE_MVE"
6509   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6510   [(set_attr "type" "mve_move")
6511    (set_attr "length""8")])
6512
6513 ;;
6514 ;; [vrmlaldavhaq_p_u])
6515 ;;
6516 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6517   [
6518    (set (match_operand:DI 0 "s_register_operand" "=r")
6519         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6520                        (match_operand:V4SI 2 "s_register_operand" "w")
6521                        (match_operand:V4SI 3 "s_register_operand" "w")
6522                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6523          VRMLALDAVHAQ_P_U))
6524   ]
6525   "TARGET_HAVE_MVE"
6526   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6527   [(set_attr "type" "mve_move")
6528    (set_attr "length""8")])
6529
6530 ;;
6531 ;; [vrmlaldavhaxq_p_s])
6532 ;;
6533 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6534   [
6535    (set (match_operand:DI 0 "s_register_operand" "=r")
6536         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6537                        (match_operand:V4SI 2 "s_register_operand" "w")
6538                        (match_operand:V4SI 3 "s_register_operand" "w")
6539                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6540          VRMLALDAVHAXQ_P_S))
6541   ]
6542   "TARGET_HAVE_MVE"
6543   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6544   [(set_attr "type" "mve_move")
6545    (set_attr "length""8")])
6546
6547 ;;
6548 ;; [vrmlsldavhaq_p_s])
6549 ;;
6550 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6551   [
6552    (set (match_operand:DI 0 "s_register_operand" "=r")
6553         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6554                        (match_operand:V4SI 2 "s_register_operand" "w")
6555                        (match_operand:V4SI 3 "s_register_operand" "w")
6556                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6557          VRMLSLDAVHAQ_P_S))
6558   ]
6559   "TARGET_HAVE_MVE"
6560   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6561   [(set_attr "type" "mve_move")
6562    (set_attr "length""8")])
6563
6564 ;;
6565 ;; [vrmlsldavhaxq_p_s])
6566 ;;
6567 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6568   [
6569    (set (match_operand:DI 0 "s_register_operand" "=r")
6570         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6571                        (match_operand:V4SI 2 "s_register_operand" "w")
6572                        (match_operand:V4SI 3 "s_register_operand" "w")
6573                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6574          VRMLSLDAVHAXQ_P_S))
6575   ]
6576   "TARGET_HAVE_MVE"
6577   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6578   [(set_attr "type" "mve_move")
6579    (set_attr "length""8")])
6580 ;;
6581 ;; [vabdq_m_f])
6582 ;;
6583 (define_insn "mve_vabdq_m_f<mode>"
6584   [
6585    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6586         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6587                        (match_operand:MVE_0 2 "s_register_operand" "w")
6588                        (match_operand:MVE_0 3 "s_register_operand" "w")
6589                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6590          VABDQ_M_F))
6591   ]
6592   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6593   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6594   [(set_attr "type" "mve_move")
6595    (set_attr "length""8")])
6596
6597 ;;
6598 ;; [vaddq_m_f])
6599 ;;
6600 (define_insn "mve_vaddq_m_f<mode>"
6601   [
6602    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6603         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6604                        (match_operand:MVE_0 2 "s_register_operand" "w")
6605                        (match_operand:MVE_0 3 "s_register_operand" "w")
6606                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6607          VADDQ_M_F))
6608   ]
6609   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6610   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6611   [(set_attr "type" "mve_move")
6612    (set_attr "length""8")])
6613
6614 ;;
6615 ;; [vaddq_m_n_f])
6616 ;;
6617 (define_insn "mve_vaddq_m_n_f<mode>"
6618   [
6619    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6620         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6621                        (match_operand:MVE_0 2 "s_register_operand" "w")
6622                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6623                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6624          VADDQ_M_N_F))
6625   ]
6626   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6627   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
6628   [(set_attr "type" "mve_move")
6629    (set_attr "length""8")])
6630
6631 ;;
6632 ;; [vandq_m_f])
6633 ;;
6634 (define_insn "mve_vandq_m_f<mode>"
6635   [
6636    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6637         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6638                        (match_operand:MVE_0 2 "s_register_operand" "w")
6639                        (match_operand:MVE_0 3 "s_register_operand" "w")
6640                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6641          VANDQ_M_F))
6642   ]
6643   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6644   "vpst\;vandt %q0, %q2, %q3"
6645   [(set_attr "type" "mve_move")
6646    (set_attr "length""8")])
6647
6648 ;;
6649 ;; [vbicq_m_f])
6650 ;;
6651 (define_insn "mve_vbicq_m_f<mode>"
6652   [
6653    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6654         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6655                        (match_operand:MVE_0 2 "s_register_operand" "w")
6656                        (match_operand:MVE_0 3 "s_register_operand" "w")
6657                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6658          VBICQ_M_F))
6659   ]
6660   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6661   "vpst\;vbict %q0, %q2, %q3"
6662   [(set_attr "type" "mve_move")
6663    (set_attr "length""8")])
6664
6665 ;;
6666 ;; [vbrsrq_m_n_f])
6667 ;;
6668 (define_insn "mve_vbrsrq_m_n_f<mode>"
6669   [
6670    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6671         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6672                        (match_operand:MVE_0 2 "s_register_operand" "w")
6673                        (match_operand:SI 3 "s_register_operand" "r")
6674                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6675          VBRSRQ_M_N_F))
6676   ]
6677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6678   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6679   [(set_attr "type" "mve_move")
6680    (set_attr "length""8")])
6681
6682 ;;
6683 ;; [vcaddq_rot270_m_f])
6684 ;;
6685 (define_insn "mve_vcaddq_rot270_m_f<mode>"
6686   [
6687    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6688         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6689                        (match_operand:MVE_0 2 "s_register_operand" "w")
6690                        (match_operand:MVE_0 3 "s_register_operand" "w")
6691                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6692          VCADDQ_ROT270_M_F))
6693   ]
6694   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6695   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6696   [(set_attr "type" "mve_move")
6697    (set_attr "length""8")])
6698
6699 ;;
6700 ;; [vcaddq_rot90_m_f])
6701 ;;
6702 (define_insn "mve_vcaddq_rot90_m_f<mode>"
6703   [
6704    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6705         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6706                        (match_operand:MVE_0 2 "s_register_operand" "w")
6707                        (match_operand:MVE_0 3 "s_register_operand" "w")
6708                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6709          VCADDQ_ROT90_M_F))
6710   ]
6711   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6712   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6713   [(set_attr "type" "mve_move")
6714    (set_attr "length""8")])
6715
6716 ;;
6717 ;; [vcmlaq_m_f])
6718 ;;
6719 (define_insn "mve_vcmlaq_m_f<mode>"
6720   [
6721    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6722         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6723                        (match_operand:MVE_0 2 "s_register_operand" "w")
6724                        (match_operand:MVE_0 3 "s_register_operand" "w")
6725                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6726          VCMLAQ_M_F))
6727   ]
6728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6729   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6730   [(set_attr "type" "mve_move")
6731    (set_attr "length""8")])
6732
6733 ;;
6734 ;; [vcmlaq_rot180_m_f])
6735 ;;
6736 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
6737   [
6738    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6739         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6740                        (match_operand:MVE_0 2 "s_register_operand" "w")
6741                        (match_operand:MVE_0 3 "s_register_operand" "w")
6742                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6743          VCMLAQ_ROT180_M_F))
6744   ]
6745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6746   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6747   [(set_attr "type" "mve_move")
6748    (set_attr "length""8")])
6749
6750 ;;
6751 ;; [vcmlaq_rot270_m_f])
6752 ;;
6753 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
6754   [
6755    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6756         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6757                        (match_operand:MVE_0 2 "s_register_operand" "w")
6758                        (match_operand:MVE_0 3 "s_register_operand" "w")
6759                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6760          VCMLAQ_ROT270_M_F))
6761   ]
6762   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6763   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6764   [(set_attr "type" "mve_move")
6765    (set_attr "length""8")])
6766
6767 ;;
6768 ;; [vcmlaq_rot90_m_f])
6769 ;;
6770 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
6771   [
6772    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6773         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6774                        (match_operand:MVE_0 2 "s_register_operand" "w")
6775                        (match_operand:MVE_0 3 "s_register_operand" "w")
6776                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6777          VCMLAQ_ROT90_M_F))
6778   ]
6779   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6780   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6781   [(set_attr "type" "mve_move")
6782    (set_attr "length""8")])
6783
6784 ;;
6785 ;; [vcmulq_m_f])
6786 ;;
6787 (define_insn "mve_vcmulq_m_f<mode>"
6788   [
6789    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6790         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6791                        (match_operand:MVE_0 2 "s_register_operand" "w")
6792                        (match_operand:MVE_0 3 "s_register_operand" "w")
6793                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6794          VCMULQ_M_F))
6795   ]
6796   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6797   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6798   [(set_attr "type" "mve_move")
6799    (set_attr "length""8")])
6800
6801 ;;
6802 ;; [vcmulq_rot180_m_f])
6803 ;;
6804 (define_insn "mve_vcmulq_rot180_m_f<mode>"
6805   [
6806    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6807         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6808                        (match_operand:MVE_0 2 "s_register_operand" "w")
6809                        (match_operand:MVE_0 3 "s_register_operand" "w")
6810                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6811          VCMULQ_ROT180_M_F))
6812   ]
6813   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6814   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6815   [(set_attr "type" "mve_move")
6816    (set_attr "length""8")])
6817
6818 ;;
6819 ;; [vcmulq_rot270_m_f])
6820 ;;
6821 (define_insn "mve_vcmulq_rot270_m_f<mode>"
6822   [
6823    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6824         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6825                        (match_operand:MVE_0 2 "s_register_operand" "w")
6826                        (match_operand:MVE_0 3 "s_register_operand" "w")
6827                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6828          VCMULQ_ROT270_M_F))
6829   ]
6830   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6831   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6832   [(set_attr "type" "mve_move")
6833    (set_attr "length""8")])
6834
6835 ;;
6836 ;; [vcmulq_rot90_m_f])
6837 ;;
6838 (define_insn "mve_vcmulq_rot90_m_f<mode>"
6839   [
6840    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6841         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6842                        (match_operand:MVE_0 2 "s_register_operand" "w")
6843                        (match_operand:MVE_0 3 "s_register_operand" "w")
6844                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6845          VCMULQ_ROT90_M_F))
6846   ]
6847   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6848   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6849   [(set_attr "type" "mve_move")
6850    (set_attr "length""8")])
6851
6852 ;;
6853 ;; [veorq_m_f])
6854 ;;
6855 (define_insn "mve_veorq_m_f<mode>"
6856   [
6857    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6858         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6859                        (match_operand:MVE_0 2 "s_register_operand" "w")
6860                        (match_operand:MVE_0 3 "s_register_operand" "w")
6861                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6862          VEORQ_M_F))
6863   ]
6864   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6865   "vpst\;veort %q0, %q2, %q3"
6866   [(set_attr "type" "mve_move")
6867    (set_attr "length""8")])
6868
6869 ;;
6870 ;; [vfmaq_m_f])
6871 ;;
6872 (define_insn "mve_vfmaq_m_f<mode>"
6873   [
6874    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6875         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6876                        (match_operand:MVE_0 2 "s_register_operand" "w")
6877                        (match_operand:MVE_0 3 "s_register_operand" "w")
6878                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6879          VFMAQ_M_F))
6880   ]
6881   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6882   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
6883   [(set_attr "type" "mve_move")
6884    (set_attr "length""8")])
6885
6886 ;;
6887 ;; [vfmaq_m_n_f])
6888 ;;
6889 (define_insn "mve_vfmaq_m_n_f<mode>"
6890   [
6891    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6892         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6893                        (match_operand:MVE_0 2 "s_register_operand" "w")
6894                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6895                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6896          VFMAQ_M_N_F))
6897   ]
6898   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6899   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
6900   [(set_attr "type" "mve_move")
6901    (set_attr "length""8")])
6902
6903 ;;
6904 ;; [vfmasq_m_n_f])
6905 ;;
6906 (define_insn "mve_vfmasq_m_n_f<mode>"
6907   [
6908    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6909         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6910                        (match_operand:MVE_0 2 "s_register_operand" "w")
6911                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6912                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6913          VFMASQ_M_N_F))
6914   ]
6915   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6916   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
6917   [(set_attr "type" "mve_move")
6918    (set_attr "length""8")])
6919
6920 ;;
6921 ;; [vfmsq_m_f])
6922 ;;
6923 (define_insn "mve_vfmsq_m_f<mode>"
6924   [
6925    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6926         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6927                        (match_operand:MVE_0 2 "s_register_operand" "w")
6928                        (match_operand:MVE_0 3 "s_register_operand" "w")
6929                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6930          VFMSQ_M_F))
6931   ]
6932   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6933   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
6934   [(set_attr "type" "mve_move")
6935    (set_attr "length""8")])
6936
6937 ;;
6938 ;; [vmaxnmq_m_f])
6939 ;;
6940 (define_insn "mve_vmaxnmq_m_f<mode>"
6941   [
6942    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6943         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6944                        (match_operand:MVE_0 2 "s_register_operand" "w")
6945                        (match_operand:MVE_0 3 "s_register_operand" "w")
6946                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6947          VMAXNMQ_M_F))
6948   ]
6949   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6950   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6951   [(set_attr "type" "mve_move")
6952    (set_attr "length""8")])
6953
6954 ;;
6955 ;; [vminnmq_m_f])
6956 ;;
6957 (define_insn "mve_vminnmq_m_f<mode>"
6958   [
6959    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6960         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6961                        (match_operand:MVE_0 2 "s_register_operand" "w")
6962                        (match_operand:MVE_0 3 "s_register_operand" "w")
6963                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6964          VMINNMQ_M_F))
6965   ]
6966   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6967   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6968   [(set_attr "type" "mve_move")
6969    (set_attr "length""8")])
6970
6971 ;;
6972 ;; [vmulq_m_f])
6973 ;;
6974 (define_insn "mve_vmulq_m_f<mode>"
6975   [
6976    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6977         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6978                        (match_operand:MVE_0 2 "s_register_operand" "w")
6979                        (match_operand:MVE_0 3 "s_register_operand" "w")
6980                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6981          VMULQ_M_F))
6982   ]
6983   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6984   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
6985   [(set_attr "type" "mve_move")
6986    (set_attr "length""8")])
6987
6988 ;;
6989 ;; [vmulq_m_n_f])
6990 ;;
6991 (define_insn "mve_vmulq_m_n_f<mode>"
6992   [
6993    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6994         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6995                        (match_operand:MVE_0 2 "s_register_operand" "w")
6996                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6997                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6998          VMULQ_M_N_F))
6999   ]
7000   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7001   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7002   [(set_attr "type" "mve_move")
7003    (set_attr "length""8")])
7004
7005 ;;
7006 ;; [vornq_m_f])
7007 ;;
7008 (define_insn "mve_vornq_m_f<mode>"
7009   [
7010    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7011         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7012                        (match_operand:MVE_0 2 "s_register_operand" "w")
7013                        (match_operand:MVE_0 3 "s_register_operand" "w")
7014                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7015          VORNQ_M_F))
7016   ]
7017   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7018   "vpst\;vornt %q0, %q2, %q3"
7019   [(set_attr "type" "mve_move")
7020    (set_attr "length""8")])
7021
7022 ;;
7023 ;; [vorrq_m_f])
7024 ;;
7025 (define_insn "mve_vorrq_m_f<mode>"
7026   [
7027    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7028         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7029                        (match_operand:MVE_0 2 "s_register_operand" "w")
7030                        (match_operand:MVE_0 3 "s_register_operand" "w")
7031                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7032          VORRQ_M_F))
7033   ]
7034   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7035   "vpst\;vorrt %q0, %q2, %q3"
7036   [(set_attr "type" "mve_move")
7037    (set_attr "length""8")])
7038
7039 ;;
7040 ;; [vsubq_m_f])
7041 ;;
7042 (define_insn "mve_vsubq_m_f<mode>"
7043   [
7044    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7045         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7046                        (match_operand:MVE_0 2 "s_register_operand" "w")
7047                        (match_operand:MVE_0 3 "s_register_operand" "w")
7048                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7049          VSUBQ_M_F))
7050   ]
7051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7052   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7053   [(set_attr "type" "mve_move")
7054    (set_attr "length""8")])
7055
7056 ;;
7057 ;; [vsubq_m_n_f])
7058 ;;
7059 (define_insn "mve_vsubq_m_n_f<mode>"
7060   [
7061    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7062         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7063                        (match_operand:MVE_0 2 "s_register_operand" "w")
7064                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7065                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7066          VSUBQ_M_N_F))
7067   ]
7068   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7069   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7070   [(set_attr "type" "mve_move")
7071    (set_attr "length""8")])
7072
7073 ;;
7074 ;; [vstrbq_s vstrbq_u]
7075 ;;
7076 (define_insn "mve_vstrbq_<supf><mode>"
7077   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7078         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7079          VSTRBQ))
7080   ]
7081   "TARGET_HAVE_MVE"
7082 {
7083    rtx ops[2];
7084    int regno = REGNO (operands[1]);
7085    ops[1] = gen_rtx_REG (TImode, regno);
7086    ops[0]  = operands[0];
7087    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7088    return "";
7089 }
7090   [(set_attr "length" "4")])
7091
7092 ;;
7093 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7094 ;;
7095 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7096   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7097    (match_operand:MVE_2 1 "s_register_operand")
7098    (match_operand:MVE_2 2 "s_register_operand")
7099    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7100   "TARGET_HAVE_MVE"
7101 {
7102   rtx ind = XEXP (operands[0], 0);
7103   gcc_assert (REG_P (ind));
7104   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7105                                                               operands[2]));
7106   DONE;
7107 })
7108
7109 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7110   [(set (mem:BLK (scratch))
7111         (unspec:BLK
7112           [(match_operand:SI 0 "register_operand" "r")
7113            (match_operand:MVE_2 1 "s_register_operand" "w")
7114            (match_operand:MVE_2 2 "s_register_operand" "w")]
7115           VSTRBSOQ))]
7116   "TARGET_HAVE_MVE"
7117   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7118   [(set_attr "length" "4")])
7119
7120 ;;
7121 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7122 ;;
7123 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7124   [(set (mem:BLK (scratch))
7125         (unspec:BLK
7126                 [(match_operand:V4SI 0 "s_register_operand" "w")
7127                  (match_operand:SI 1 "immediate_operand" "i")
7128                  (match_operand:V4SI 2 "s_register_operand" "w")]
7129          VSTRWSBQ))
7130   ]
7131   "TARGET_HAVE_MVE"
7132 {
7133    rtx ops[3];
7134    ops[0] = operands[0];
7135    ops[1] = operands[1];
7136    ops[2] = operands[2];
7137    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7138    return "";
7139 }
7140   [(set_attr "length" "4")])
7141
7142 ;;
7143 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7144 ;;
7145 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7146   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7147         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7148                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7149          VLDRBGOQ))
7150   ]
7151   "TARGET_HAVE_MVE"
7152 {
7153    rtx ops[3];
7154    ops[0] = operands[0];
7155    ops[1] = operands[1];
7156    ops[2] = operands[2];
7157    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7158      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7159    else
7160      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7161    return "";
7162 }
7163   [(set_attr "length" "4")])
7164
7165 ;;
7166 ;; [vldrbq_s vldrbq_u]
7167 ;;
7168 (define_insn "mve_vldrbq_<supf><mode>"
7169   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7170         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7171          VLDRBQ))
7172   ]
7173   "TARGET_HAVE_MVE"
7174 {
7175    rtx ops[2];
7176    int regno = REGNO (operands[0]);
7177    ops[0] = gen_rtx_REG (TImode, regno);
7178    ops[1]  = operands[1];
7179    if (<V_sz_elem> == 8)
7180      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7181    else
7182      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7183    return "";
7184 }
7185   [(set_attr "length" "4")])
7186
7187 ;;
7188 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7189 ;;
7190 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7191   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7192         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7193                       (match_operand:SI 2 "immediate_operand" "i")]
7194          VLDRWGBQ))
7195   ]
7196   "TARGET_HAVE_MVE"
7197 {
7198    rtx ops[3];
7199    ops[0] = operands[0];
7200    ops[1] = operands[1];
7201    ops[2] = operands[2];
7202    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7203    return "";
7204 }
7205   [(set_attr "length" "4")])
7206
7207 ;;
7208 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7209 ;;
7210 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7211   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7212    (match_operand:MVE_2 1 "s_register_operand")
7213    (match_operand:MVE_2 2 "s_register_operand")
7214    (match_operand:HI 3 "vpr_register_operand" "Up")
7215    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7216   "TARGET_HAVE_MVE"
7217 {
7218   rtx ind = XEXP (operands[0], 0);
7219   gcc_assert (REG_P (ind));
7220   emit_insn (
7221     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7222                                                        operands[2],
7223                                                        operands[3]));
7224   DONE;
7225 })
7226
7227 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7228   [(set (mem:BLK (scratch))
7229         (unspec:BLK
7230           [(match_operand:SI 0 "register_operand" "r")
7231            (match_operand:MVE_2 1 "s_register_operand" "w")
7232            (match_operand:MVE_2 2 "s_register_operand" "w")
7233            (match_operand:HI 3 "vpr_register_operand" "Up")]
7234           VSTRBSOQ))]
7235   "TARGET_HAVE_MVE"
7236   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7237   [(set_attr "length" "8")])
7238
7239 ;;
7240 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7241 ;;
7242 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7243   [(set (mem:BLK (scratch))
7244         (unspec:BLK
7245                 [(match_operand:V4SI 0 "s_register_operand" "w")
7246                  (match_operand:SI 1 "immediate_operand" "i")
7247                  (match_operand:V4SI 2 "s_register_operand" "w")
7248                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7249          VSTRWSBQ))
7250   ]
7251   "TARGET_HAVE_MVE"
7252 {
7253    rtx ops[3];
7254    ops[0] = operands[0];
7255    ops[1] = operands[1];
7256    ops[2] = operands[2];
7257    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7258    return "";
7259 }
7260   [(set_attr "length" "8")])
7261
7262 ;;
7263 ;; [vstrbq_p_s vstrbq_p_u]
7264 ;;
7265 (define_insn "mve_vstrbq_p_<supf><mode>"
7266   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7267         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7268                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7269          VSTRBQ))
7270   ]
7271   "TARGET_HAVE_MVE"
7272 {
7273    rtx ops[2];
7274    int regno = REGNO (operands[1]);
7275    ops[1] = gen_rtx_REG (TImode, regno);
7276    ops[0]  = operands[0];
7277    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7278    return "";
7279 }
7280   [(set_attr "length" "8")])
7281
7282 ;;
7283 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7284 ;;
7285 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7286   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7287         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7288                        (match_operand:MVE_2 2 "s_register_operand" "w")
7289                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7290          VLDRBGOQ))
7291   ]
7292   "TARGET_HAVE_MVE"
7293 {
7294    rtx ops[4];
7295    ops[0] = operands[0];
7296    ops[1] = operands[1];
7297    ops[2] = operands[2];
7298    ops[3] = operands[3];
7299    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7300      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7301    else
7302      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7303    return "";
7304 }
7305   [(set_attr "length" "8")])
7306
7307 ;;
7308 ;; [vldrbq_z_s vldrbq_z_u]
7309 ;;
7310 (define_insn "mve_vldrbq_z_<supf><mode>"
7311   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7312         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7313                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7314          VLDRBQ))
7315   ]
7316   "TARGET_HAVE_MVE"
7317 {
7318    rtx ops[2];
7319    int regno = REGNO (operands[0]);
7320    ops[0] = gen_rtx_REG (TImode, regno);
7321    ops[1]  = operands[1];
7322    if (<V_sz_elem> == 8)
7323      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7324    else
7325      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7326    return "";
7327 }
7328   [(set_attr "length" "8")])
7329
7330 ;;
7331 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7332 ;;
7333 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7334   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7335         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7336                       (match_operand:SI 2 "immediate_operand" "i")
7337                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7338          VLDRWGBQ))
7339   ]
7340   "TARGET_HAVE_MVE"
7341 {
7342    rtx ops[3];
7343    ops[0] = operands[0];
7344    ops[1] = operands[1];
7345    ops[2] = operands[2];
7346    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7347    return "";
7348 }
7349   [(set_attr "length" "8")])
7350
7351 ;;
7352 ;; [vldrhq_f]
7353 ;;
7354 (define_insn "mve_vldrhq_fv8hf"
7355   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7356         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7357          VLDRHQ_F))
7358   ]
7359   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7360 {
7361    rtx ops[2];
7362    int regno = REGNO (operands[0]);
7363    ops[0] = gen_rtx_REG (TImode, regno);
7364    ops[1]  = operands[1];
7365    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7366    return "";
7367 }
7368   [(set_attr "length" "4")])
7369
7370 ;;
7371 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7372 ;;
7373 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7374   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7375         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7376                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7377         VLDRHGOQ))
7378   ]
7379   "TARGET_HAVE_MVE"
7380 {
7381    rtx ops[3];
7382    ops[0] = operands[0];
7383    ops[1] = operands[1];
7384    ops[2] = operands[2];
7385    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7386      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7387    else
7388      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7389    return "";
7390 }
7391   [(set_attr "length" "4")])
7392
7393 ;;
7394 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7395 ;;
7396 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7397   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7398         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7399                        (match_operand:MVE_6 2 "s_register_operand" "w")
7400                        (match_operand:HI 3 "vpr_register_operand" "Up")
7401         ]VLDRHGOQ))
7402   ]
7403   "TARGET_HAVE_MVE"
7404 {
7405    rtx ops[4];
7406    ops[0] = operands[0];
7407    ops[1] = operands[1];
7408    ops[2] = operands[2];
7409    ops[3] = operands[3];
7410    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7411      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7412    else
7413      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7414    return "";
7415 }
7416  [(set_attr "length" "8")])
7417
7418 ;;
7419 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7420 ;;
7421 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7422   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7423         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7424                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7425         VLDRHGSOQ))
7426   ]
7427   "TARGET_HAVE_MVE"
7428 {
7429    rtx ops[3];
7430    ops[0] = operands[0];
7431    ops[1] = operands[1];
7432    ops[2] = operands[2];
7433       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7434      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7435    else
7436      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7437    return "";
7438 }
7439   [(set_attr "length" "4")])
7440
7441 ;;
7442 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7443 ;;
7444 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7445   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7446         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7447                        (match_operand:MVE_6 2 "s_register_operand" "w")
7448                        (match_operand:HI 3 "vpr_register_operand" "Up")
7449         ]VLDRHGSOQ))
7450   ]
7451   "TARGET_HAVE_MVE"
7452 {
7453    rtx ops[4];
7454    ops[0] = operands[0];
7455    ops[1] = operands[1];
7456    ops[2] = operands[2];
7457    ops[3] = operands[3];
7458    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7459      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7460    else
7461      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7462    return "";
7463 }
7464   [(set_attr "length" "8")])
7465
7466 ;;
7467 ;; [vldrhq_s, vldrhq_u]
7468 ;;
7469 (define_insn "mve_vldrhq_<supf><mode>"
7470   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7471         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7472          VLDRHQ))
7473   ]
7474   "TARGET_HAVE_MVE"
7475 {
7476    rtx ops[2];
7477    int regno = REGNO (operands[0]);
7478    ops[0] = gen_rtx_REG (TImode, regno);
7479    ops[1]  = operands[1];
7480    if (<V_sz_elem> == 16)
7481      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7482    else
7483      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7484    return "";
7485 }
7486   [(set_attr "length" "4")])
7487
7488 ;;
7489 ;; [vldrhq_z_f]
7490 ;;
7491 (define_insn "mve_vldrhq_z_fv8hf"
7492   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7493         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7494         (match_operand:HI 2 "vpr_register_operand" "Up")]
7495          VLDRHQ_F))
7496   ]
7497   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7498 {
7499    rtx ops[2];
7500    int regno = REGNO (operands[0]);
7501    ops[0] = gen_rtx_REG (TImode, regno);
7502    ops[1]  = operands[1];
7503    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7504    return "";
7505 }
7506   [(set_attr "length" "8")])
7507
7508 ;;
7509 ;; [vldrhq_z_s vldrhq_z_u]
7510 ;;
7511 (define_insn "mve_vldrhq_z_<supf><mode>"
7512   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7513         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7514         (match_operand:HI 2 "vpr_register_operand" "Up")]
7515          VLDRHQ))
7516   ]
7517   "TARGET_HAVE_MVE"
7518 {
7519    rtx ops[2];
7520    int regno = REGNO (operands[0]);
7521    ops[0] = gen_rtx_REG (TImode, regno);
7522    ops[1]  = operands[1];
7523    if (<V_sz_elem> == 16)
7524      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7525    else
7526      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7527    return "";
7528 }
7529   [(set_attr "length" "8")])
7530
7531 ;;
7532 ;; [vldrwq_f]
7533 ;;
7534 (define_insn "mve_vldrwq_fv4sf"
7535   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7536         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7537          VLDRWQ_F))
7538   ]
7539   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7540 {
7541    rtx ops[2];
7542    int regno = REGNO (operands[0]);
7543    ops[0] = gen_rtx_REG (TImode, regno);
7544    ops[1]  = operands[1];
7545    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7546    return "";
7547 }
7548   [(set_attr "length" "4")])
7549
7550 ;;
7551 ;; [vldrwq_s vldrwq_u]
7552 ;;
7553 (define_insn "mve_vldrwq_<supf>v4si"
7554   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7555         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
7556          VLDRWQ))
7557   ]
7558   "TARGET_HAVE_MVE"
7559 {
7560    rtx ops[2];
7561    int regno = REGNO (operands[0]);
7562    ops[0] = gen_rtx_REG (TImode, regno);
7563    ops[1]  = operands[1];
7564    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7565    return "";
7566 }
7567   [(set_attr "length" "4")])
7568
7569 ;;
7570 ;; [vldrwq_z_f]
7571 ;;
7572 (define_insn "mve_vldrwq_z_fv4sf"
7573   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7574         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
7575         (match_operand:HI 2 "vpr_register_operand" "Up")]
7576          VLDRWQ_F))
7577   ]
7578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7579 {
7580    rtx ops[2];
7581    int regno = REGNO (operands[0]);
7582    ops[0] = gen_rtx_REG (TImode, regno);
7583    ops[1]  = operands[1];
7584    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7585    return "";
7586 }
7587   [(set_attr "length" "8")])
7588
7589 ;;
7590 ;; [vldrwq_z_s vldrwq_z_u]
7591 ;;
7592 (define_insn "mve_vldrwq_z_<supf>v4si"
7593   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7594         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
7595         (match_operand:HI 2 "vpr_register_operand" "Up")]
7596          VLDRWQ))
7597   ]
7598   "TARGET_HAVE_MVE"
7599 {
7600    rtx ops[2];
7601    int regno = REGNO (operands[0]);
7602    ops[0] = gen_rtx_REG (TImode, regno);
7603    ops[1]  = operands[1];
7604    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7605    return "";
7606 }
7607   [(set_attr "length" "8")])
7608
7609 (define_expand "mve_vld1q_f<mode>"
7610   [(match_operand:MVE_0 0 "s_register_operand")
7611    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7612   ]
7613   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
7614 {
7615   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
7616   DONE;
7617 })
7618
7619 (define_expand "mve_vld1q_<supf><mode>"
7620   [(match_operand:MVE_2 0 "s_register_operand")
7621    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
7622   ]
7623   "TARGET_HAVE_MVE"
7624 {
7625   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
7626   DONE;
7627 })
7628
7629 ;;
7630 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
7631 ;;
7632 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
7633   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7634         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7635                       (match_operand:SI 2 "immediate_operand" "i")]
7636          VLDRDGBQ))
7637   ]
7638   "TARGET_HAVE_MVE"
7639 {
7640    rtx ops[3];
7641    ops[0] = operands[0];
7642    ops[1] = operands[1];
7643    ops[2] = operands[2];
7644    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
7645    return "";
7646 }
7647   [(set_attr "length" "4")])
7648
7649 ;;
7650 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
7651 ;;
7652 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
7653   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7654         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7655                       (match_operand:SI 2 "immediate_operand" "i")
7656                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7657          VLDRDGBQ))
7658   ]
7659   "TARGET_HAVE_MVE"
7660 {
7661    rtx ops[3];
7662    ops[0] = operands[0];
7663    ops[1] = operands[1];
7664    ops[2] = operands[2];
7665    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
7666    return "";
7667 }
7668   [(set_attr "length" "8")])
7669
7670 ;;
7671 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
7672 ;;
7673 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
7674  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7675        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7676                      (match_operand:V2DI 2 "s_register_operand" "w")]
7677         VLDRDGOQ))
7678  ]
7679  "TARGET_HAVE_MVE"
7680 {
7681   rtx ops[3];
7682   ops[0] = operands[0];
7683   ops[1] = operands[1];
7684   ops[2] = operands[2];
7685   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
7686   return "";
7687 }
7688  [(set_attr "length" "4")])
7689
7690 ;;
7691 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
7692 ;;
7693 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
7694  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7695        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7696                      (match_operand:V2DI 2 "s_register_operand" "w")
7697                      (match_operand:HI 3 "vpr_register_operand" "Up")]
7698         VLDRDGOQ))
7699  ]
7700  "TARGET_HAVE_MVE"
7701 {
7702   rtx ops[3];
7703   ops[0] = operands[0];
7704   ops[1] = operands[1];
7705   ops[2] = operands[2];
7706   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
7707   return "";
7708 }
7709  [(set_attr "length" "8")])
7710
7711 ;;
7712 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
7713 ;;
7714 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
7715   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7716         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7717                       (match_operand:V2DI 2 "s_register_operand" "w")]
7718          VLDRDGSOQ))
7719   ]
7720   "TARGET_HAVE_MVE"
7721 {
7722    rtx ops[3];
7723    ops[0] = operands[0];
7724    ops[1] = operands[1];
7725    ops[2] = operands[2];
7726    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7727    return "";
7728 }
7729   [(set_attr "length" "4")])
7730
7731 ;;
7732 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
7733 ;;
7734 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
7735   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7736         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7737                       (match_operand:V2DI 2 "s_register_operand" "w")
7738                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7739          VLDRDGSOQ))
7740   ]
7741   "TARGET_HAVE_MVE"
7742 {
7743    rtx ops[3];
7744    ops[0] = operands[0];
7745    ops[1] = operands[1];
7746    ops[2] = operands[2];
7747    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7748    return "";
7749 }
7750   [(set_attr "length" "8")])
7751
7752 ;;
7753 ;; [vldrhq_gather_offset_f]
7754 ;;
7755 (define_insn "mve_vldrhq_gather_offset_fv8hf"
7756   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7757         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7758                       (match_operand:V8HI 2 "s_register_operand" "w")]
7759          VLDRHQGO_F))
7760   ]
7761   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7762 {
7763    rtx ops[3];
7764    ops[0] = operands[0];
7765    ops[1] = operands[1];
7766    ops[2] = operands[2];
7767    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
7768    return "";
7769 }
7770   [(set_attr "length" "4")])
7771
7772 ;;
7773 ;; [vldrhq_gather_offset_z_f]
7774 ;;
7775 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
7776   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7777         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7778                       (match_operand:V8HI 2 "s_register_operand" "w")
7779                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7780          VLDRHQGO_F))
7781   ]
7782   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7783 {
7784    rtx ops[4];
7785    ops[0] = operands[0];
7786    ops[1] = operands[1];
7787    ops[2] = operands[2];
7788    ops[3] = operands[3];
7789    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
7790    return "";
7791 }
7792   [(set_attr "length" "8")])
7793
7794 ;;
7795 ;; [vldrhq_gather_shifted_offset_f]
7796 ;;
7797 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
7798   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7799         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7800                       (match_operand:V8HI 2 "s_register_operand" "w")]
7801          VLDRHQGSO_F))
7802   ]
7803   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7804 {
7805    rtx ops[3];
7806    ops[0] = operands[0];
7807    ops[1] = operands[1];
7808    ops[2] = operands[2];
7809    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7810    return "";
7811 }
7812   [(set_attr "length" "4")])
7813
7814 ;;
7815 ;; [vldrhq_gather_shifted_offset_z_f]
7816 ;;
7817 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
7818   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7819         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7820                       (match_operand:V8HI 2 "s_register_operand" "w")
7821                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7822          VLDRHQGSO_F))
7823   ]
7824   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7825 {
7826    rtx ops[4];
7827    ops[0] = operands[0];
7828    ops[1] = operands[1];
7829    ops[2] = operands[2];
7830    ops[3] = operands[3];
7831    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7832    return "";
7833 }
7834   [(set_attr "length" "8")])
7835
7836 ;;
7837 ;; [vldrwq_gather_base_f]
7838 ;;
7839 (define_insn "mve_vldrwq_gather_base_fv4sf"
7840   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7841         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7842                       (match_operand:SI 2 "immediate_operand" "i")]
7843          VLDRWQGB_F))
7844   ]
7845   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7846 {
7847    rtx ops[3];
7848    ops[0] = operands[0];
7849    ops[1] = operands[1];
7850    ops[2] = operands[2];
7851    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7852    return "";
7853 }
7854   [(set_attr "length" "4")])
7855
7856 ;;
7857 ;; [vldrwq_gather_base_z_f]
7858 ;;
7859 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
7860   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7861         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7862                       (match_operand:SI 2 "immediate_operand" "i")
7863                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7864          VLDRWQGB_F))
7865   ]
7866   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7867 {
7868    rtx ops[3];
7869    ops[0] = operands[0];
7870    ops[1] = operands[1];
7871    ops[2] = operands[2];
7872    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7873    return "";
7874 }
7875   [(set_attr "length" "8")])
7876
7877 ;;
7878 ;; [vldrwq_gather_offset_f]
7879 ;;
7880 (define_insn "mve_vldrwq_gather_offset_fv4sf"
7881   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7882         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7883                        (match_operand:V4SI 2 "s_register_operand" "w")]
7884          VLDRWQGO_F))
7885   ]
7886   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7887 {
7888    rtx ops[3];
7889    ops[0] = operands[0];
7890    ops[1] = operands[1];
7891    ops[2] = operands[2];
7892    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7893    return "";
7894 }
7895   [(set_attr "length" "4")])
7896
7897 ;;
7898 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
7899 ;;
7900 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
7901   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7902         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7903                        (match_operand:V4SI 2 "s_register_operand" "w")]
7904          VLDRWGOQ))
7905   ]
7906   "TARGET_HAVE_MVE"
7907 {
7908    rtx ops[3];
7909    ops[0] = operands[0];
7910    ops[1] = operands[1];
7911    ops[2] = operands[2];
7912    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7913    return "";
7914 }
7915   [(set_attr "length" "4")])
7916
7917 ;;
7918 ;; [vldrwq_gather_offset_z_f]
7919 ;;
7920 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
7921   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7922         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7923                       (match_operand:V4SI 2 "s_register_operand" "w")
7924                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7925          VLDRWQGO_F))
7926   ]
7927   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7928 {
7929    rtx ops[4];
7930    ops[0] = operands[0];
7931    ops[1] = operands[1];
7932    ops[2] = operands[2];
7933    ops[3] = operands[3];
7934    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7935    return "";
7936 }
7937   [(set_attr "length" "8")])
7938
7939 ;;
7940 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
7941 ;;
7942 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
7943   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7944         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7945                       (match_operand:V4SI 2 "s_register_operand" "w")
7946                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7947          VLDRWGOQ))
7948   ]
7949   "TARGET_HAVE_MVE"
7950 {
7951    rtx ops[4];
7952    ops[0] = operands[0];
7953    ops[1] = operands[1];
7954    ops[2] = operands[2];
7955    ops[3] = operands[3];
7956    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7957    return "";
7958 }
7959   [(set_attr "length" "8")])
7960
7961 ;;
7962 ;; [vldrwq_gather_shifted_offset_f]
7963 ;;
7964 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
7965   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7966         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7967                       (match_operand:V4SI 2 "s_register_operand" "w")]
7968          VLDRWQGSO_F))
7969   ]
7970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7971 {
7972    rtx ops[3];
7973    ops[0] = operands[0];
7974    ops[1] = operands[1];
7975    ops[2] = operands[2];
7976    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
7977    return "";
7978 }
7979   [(set_attr "length" "4")])
7980
7981 ;;
7982 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
7983 ;;
7984 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
7985   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7986         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7987                       (match_operand:V4SI 2 "s_register_operand" "w")]
7988          VLDRWGSOQ))
7989   ]
7990   "TARGET_HAVE_MVE"
7991 {
7992    rtx ops[3];
7993    ops[0] = operands[0];
7994    ops[1] = operands[1];
7995    ops[2] = operands[2];
7996    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
7997    return "";
7998 }
7999   [(set_attr "length" "4")])
8000
8001 ;;
8002 ;; [vldrwq_gather_shifted_offset_z_f]
8003 ;;
8004 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8005   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8006         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8007                       (match_operand:V4SI 2 "s_register_operand" "w")
8008                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8009          VLDRWQGSO_F))
8010   ]
8011   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8012 {
8013    rtx ops[4];
8014    ops[0] = operands[0];
8015    ops[1] = operands[1];
8016    ops[2] = operands[2];
8017    ops[3] = operands[3];
8018    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8019    return "";
8020 }
8021   [(set_attr "length" "8")])
8022
8023 ;;
8024 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8025 ;;
8026 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8027   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8028         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8029                       (match_operand:V4SI 2 "s_register_operand" "w")
8030                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8031          VLDRWGSOQ))
8032   ]
8033   "TARGET_HAVE_MVE"
8034 {
8035    rtx ops[4];
8036    ops[0] = operands[0];
8037    ops[1] = operands[1];
8038    ops[2] = operands[2];
8039    ops[3] = operands[3];
8040    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8041    return "";
8042 }
8043   [(set_attr "length" "8")])
8044
8045 ;;
8046 ;; [vstrhq_f]
8047 ;;
8048 (define_insn "mve_vstrhq_fv8hf"
8049   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8050         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8051          VSTRHQ_F))
8052   ]
8053   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8054 {
8055    rtx ops[2];
8056    int regno = REGNO (operands[1]);
8057    ops[1] = gen_rtx_REG (TImode, regno);
8058    ops[0]  = operands[0];
8059    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8060    return "";
8061 }
8062   [(set_attr "length" "4")])
8063
8064 ;;
8065 ;; [vstrhq_p_f]
8066 ;;
8067 (define_insn "mve_vstrhq_p_fv8hf"
8068   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8069         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8070                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8071          VSTRHQ_F))
8072   ]
8073   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8074 {
8075    rtx ops[2];
8076    int regno = REGNO (operands[1]);
8077    ops[1] = gen_rtx_REG (TImode, regno);
8078    ops[0]  = operands[0];
8079    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8080    return "";
8081 }
8082   [(set_attr "length" "8")])
8083
8084 ;;
8085 ;; [vstrhq_p_s vstrhq_p_u]
8086 ;;
8087 (define_insn "mve_vstrhq_p_<supf><mode>"
8088   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8089         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8090                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8091          VSTRHQ))
8092   ]
8093   "TARGET_HAVE_MVE"
8094 {
8095    rtx ops[2];
8096    int regno = REGNO (operands[1]);
8097    ops[1] = gen_rtx_REG (TImode, regno);
8098    ops[0]  = operands[0];
8099    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8100    return "";
8101 }
8102   [(set_attr "length" "8")])
8103
8104 ;;
8105 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8106 ;;
8107 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8108   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8109    (match_operand:MVE_6 1 "s_register_operand")
8110    (match_operand:MVE_6 2 "s_register_operand")
8111    (match_operand:HI 3 "vpr_register_operand")
8112    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8113   "TARGET_HAVE_MVE"
8114 {
8115   rtx ind = XEXP (operands[0], 0);
8116   gcc_assert (REG_P (ind));
8117   emit_insn (
8118     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8119                                                        operands[2],
8120                                                        operands[3]));
8121   DONE;
8122 })
8123
8124 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8125   [(set (mem:BLK (scratch))
8126         (unspec:BLK
8127           [(match_operand:SI 0 "register_operand" "r")
8128            (match_operand:MVE_6 1 "s_register_operand" "w")
8129            (match_operand:MVE_6 2 "s_register_operand" "w")
8130            (match_operand:HI 3 "vpr_register_operand" "Up")]
8131           VSTRHSOQ))]
8132   "TARGET_HAVE_MVE"
8133   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8134   [(set_attr "length" "8")])
8135
8136 ;;
8137 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8138 ;;
8139 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8140   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8141    (match_operand:MVE_6 1 "s_register_operand")
8142    (match_operand:MVE_6 2 "s_register_operand")
8143    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8144   "TARGET_HAVE_MVE"
8145 {
8146   rtx ind = XEXP (operands[0], 0);
8147   gcc_assert (REG_P (ind));
8148   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8149                                                               operands[2]));
8150   DONE;
8151 })
8152
8153 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8154   [(set (mem:BLK (scratch))
8155         (unspec:BLK
8156           [(match_operand:SI 0 "register_operand" "r")
8157            (match_operand:MVE_6 1 "s_register_operand" "w")
8158            (match_operand:MVE_6 2 "s_register_operand" "w")]
8159           VSTRHSOQ))]
8160   "TARGET_HAVE_MVE"
8161   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8162   [(set_attr "length" "4")])
8163
8164 ;;
8165 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8166 ;;
8167 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8168   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8169    (match_operand:MVE_6 1 "s_register_operand")
8170    (match_operand:MVE_6 2 "s_register_operand")
8171    (match_operand:HI 3 "vpr_register_operand")
8172    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8173   "TARGET_HAVE_MVE"
8174 {
8175   rtx ind = XEXP (operands[0], 0);
8176   gcc_assert (REG_P (ind));
8177   emit_insn (
8178     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8179                                                                operands[2],
8180                                                                operands[3]));
8181   DONE;
8182 })
8183
8184 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8185   [(set (mem:BLK (scratch))
8186         (unspec:BLK
8187           [(match_operand:SI 0 "register_operand" "r")
8188            (match_operand:MVE_6 1 "s_register_operand" "w")
8189            (match_operand:MVE_6 2 "s_register_operand" "w")
8190            (match_operand:HI 3 "vpr_register_operand" "Up")]
8191           VSTRHSSOQ))]
8192   "TARGET_HAVE_MVE"
8193   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8194   [(set_attr "length" "8")])
8195
8196 ;;
8197 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8198 ;;
8199 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8200   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8201    (match_operand:MVE_6 1 "s_register_operand")
8202    (match_operand:MVE_6 2 "s_register_operand")
8203    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8204   "TARGET_HAVE_MVE"
8205 {
8206   rtx ind = XEXP (operands[0], 0);
8207   gcc_assert (REG_P (ind));
8208   emit_insn (
8209     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8210                                                              operands[2]));
8211   DONE;
8212 })
8213
8214 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8215   [(set (mem:BLK (scratch))
8216         (unspec:BLK
8217           [(match_operand:SI 0 "register_operand" "r")
8218            (match_operand:MVE_6 1 "s_register_operand" "w")
8219            (match_operand:MVE_6 2 "s_register_operand" "w")]
8220           VSTRHSSOQ))]
8221   "TARGET_HAVE_MVE"
8222   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8223   [(set_attr "length" "4")])
8224
8225 ;;
8226 ;; [vstrhq_s, vstrhq_u]
8227 ;;
8228 (define_insn "mve_vstrhq_<supf><mode>"
8229   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8230         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8231          VSTRHQ))
8232   ]
8233   "TARGET_HAVE_MVE"
8234 {
8235    rtx ops[2];
8236    int regno = REGNO (operands[1]);
8237    ops[1] = gen_rtx_REG (TImode, regno);
8238    ops[0]  = operands[0];
8239    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8240    return "";
8241 }
8242   [(set_attr "length" "4")])
8243
8244 ;;
8245 ;; [vstrwq_f]
8246 ;;
8247 (define_insn "mve_vstrwq_fv4sf"
8248   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8249         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8250          VSTRWQ_F))
8251   ]
8252   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8253 {
8254    rtx ops[2];
8255    int regno = REGNO (operands[1]);
8256    ops[1] = gen_rtx_REG (TImode, regno);
8257    ops[0]  = operands[0];
8258    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8259    return "";
8260 }
8261   [(set_attr "length" "4")])
8262
8263 ;;
8264 ;; [vstrwq_p_f]
8265 ;;
8266 (define_insn "mve_vstrwq_p_fv4sf"
8267   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8268         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8269                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8270          VSTRWQ_F))
8271   ]
8272   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8273 {
8274    rtx ops[2];
8275    int regno = REGNO (operands[1]);
8276    ops[1] = gen_rtx_REG (TImode, regno);
8277    ops[0]  = operands[0];
8278    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8279    return "";
8280 }
8281   [(set_attr "length" "8")])
8282
8283 ;;
8284 ;; [vstrwq_p_s vstrwq_p_u]
8285 ;;
8286 (define_insn "mve_vstrwq_p_<supf>v4si"
8287   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8288         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8289                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8290          VSTRWQ))
8291   ]
8292   "TARGET_HAVE_MVE"
8293 {
8294    rtx ops[2];
8295    int regno = REGNO (operands[1]);
8296    ops[1] = gen_rtx_REG (TImode, regno);
8297    ops[0]  = operands[0];
8298    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8299    return "";
8300 }
8301   [(set_attr "length" "8")])
8302
8303 ;;
8304 ;; [vstrwq_s vstrwq_u]
8305 ;;
8306 (define_insn "mve_vstrwq_<supf>v4si"
8307   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8308         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8309          VSTRWQ))
8310   ]
8311   "TARGET_HAVE_MVE"
8312 {
8313    rtx ops[2];
8314    int regno = REGNO (operands[1]);
8315    ops[1] = gen_rtx_REG (TImode, regno);
8316    ops[0]  = operands[0];
8317    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8318    return "";
8319 }
8320   [(set_attr "length" "4")])
8321
8322 (define_expand "mve_vst1q_f<mode>"
8323   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8324    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8325   ]
8326   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8327 {
8328   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8329   DONE;
8330 })
8331
8332 (define_expand "mve_vst1q_<supf><mode>"
8333   [(match_operand:MVE_2 0 "mve_memory_operand")
8334    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8335   ]
8336   "TARGET_HAVE_MVE"
8337 {
8338   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8339   DONE;
8340 })
8341
8342 ;;
8343 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8344 ;;
8345 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8346   [(set (mem:BLK (scratch))
8347         (unspec:BLK
8348                 [(match_operand:V2DI 0 "s_register_operand" "w")
8349                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8350                  (match_operand:V2DI 2 "s_register_operand" "w")
8351                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8352          VSTRDSBQ))
8353   ]
8354   "TARGET_HAVE_MVE"
8355 {
8356    rtx ops[3];
8357    ops[0] = operands[0];
8358    ops[1] = operands[1];
8359    ops[2] = operands[2];
8360    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8361    return "";
8362 }
8363   [(set_attr "length" "8")])
8364
8365 ;;
8366 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8367 ;;
8368 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8369   [(set (mem:BLK (scratch))
8370         (unspec:BLK
8371                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8372                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8373                  (match_operand:V2DI 2 "s_register_operand" "w")]
8374          VSTRDSBQ))
8375   ]
8376   "TARGET_HAVE_MVE"
8377 {
8378    rtx ops[3];
8379    ops[0] = operands[0];
8380    ops[1] = operands[1];
8381    ops[2] = operands[2];
8382    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8383    return "";
8384 }
8385   [(set_attr "length" "4")])
8386
8387 ;;
8388 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8389 ;;
8390 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8391   [(match_operand:V2DI 0 "mve_scatter_memory")
8392    (match_operand:V2DI 1 "s_register_operand")
8393    (match_operand:V2DI 2 "s_register_operand")
8394    (match_operand:HI 3 "vpr_register_operand")
8395    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8396   "TARGET_HAVE_MVE"
8397 {
8398   rtx ind = XEXP (operands[0], 0);
8399   gcc_assert (REG_P (ind));
8400   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8401                                                               operands[2],
8402                                                               operands[3]));
8403   DONE;
8404 })
8405
8406 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8407   [(set (mem:BLK (scratch))
8408         (unspec:BLK
8409           [(match_operand:SI 0 "register_operand" "r")
8410            (match_operand:V2DI 1 "s_register_operand" "w")
8411            (match_operand:V2DI 2 "s_register_operand" "w")
8412            (match_operand:HI 3 "vpr_register_operand" "Up")]
8413           VSTRDSOQ))]
8414   "TARGET_HAVE_MVE"
8415   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8416   [(set_attr "length" "8")])
8417
8418 ;;
8419 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8420 ;;
8421 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8422   [(match_operand:V2DI 0 "mve_scatter_memory")
8423    (match_operand:V2DI 1 "s_register_operand")
8424    (match_operand:V2DI 2 "s_register_operand")
8425    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8426   "TARGET_HAVE_MVE"
8427 {
8428   rtx ind = XEXP (operands[0], 0);
8429   gcc_assert (REG_P (ind));
8430   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8431                                                             operands[2]));
8432   DONE;
8433 })
8434
8435 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8436   [(set (mem:BLK (scratch))
8437         (unspec:BLK
8438           [(match_operand:SI 0 "register_operand" "r")
8439            (match_operand:V2DI 1 "s_register_operand" "w")
8440            (match_operand:V2DI 2 "s_register_operand" "w")]
8441           VSTRDSOQ))]
8442   "TARGET_HAVE_MVE"
8443   "vstrd.64\t%q2, [%0, %q1]"
8444   [(set_attr "length" "4")])
8445
8446 ;;
8447 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8448 ;;
8449 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8450   [(match_operand:V2DI 0 "mve_scatter_memory")
8451    (match_operand:V2DI 1 "s_register_operand")
8452    (match_operand:V2DI 2 "s_register_operand")
8453    (match_operand:HI 3 "vpr_register_operand")
8454    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8455   "TARGET_HAVE_MVE"
8456 {
8457   rtx ind = XEXP (operands[0], 0);
8458   gcc_assert (REG_P (ind));
8459   emit_insn (
8460     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8461                                                              operands[2],
8462                                                              operands[3]));
8463   DONE;
8464 })
8465
8466 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8467   [(set (mem:BLK (scratch))
8468         (unspec:BLK
8469           [(match_operand:SI 0 "register_operand" "r")
8470            (match_operand:V2DI 1 "s_register_operand" "w")
8471            (match_operand:V2DI 2 "s_register_operand" "w")
8472            (match_operand:HI 3 "vpr_register_operand" "Up")]
8473           VSTRDSSOQ))]
8474   "TARGET_HAVE_MVE"
8475   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8476   [(set_attr "length" "8")])
8477
8478 ;;
8479 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8480 ;;
8481 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8482   [(match_operand:V2DI 0 "mve_scatter_memory")
8483    (match_operand:V2DI 1 "s_register_operand")
8484    (match_operand:V2DI 2 "s_register_operand")
8485    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8486   "TARGET_HAVE_MVE"
8487 {
8488   rtx ind = XEXP (operands[0], 0);
8489   gcc_assert (REG_P (ind));
8490   emit_insn (
8491     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8492                                                            operands[2]));
8493   DONE;
8494 })
8495
8496 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8497   [(set (mem:BLK (scratch))
8498         (unspec:BLK
8499           [(match_operand:SI 0 "register_operand" "r")
8500            (match_operand:V2DI 1 "s_register_operand" "w")
8501            (match_operand:V2DI 2 "s_register_operand" "w")]
8502           VSTRDSSOQ))]
8503   "TARGET_HAVE_MVE"
8504   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8505   [(set_attr "length" "4")])
8506
8507 ;;
8508 ;; [vstrhq_scatter_offset_f]
8509 ;;
8510 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8511   [(match_operand:V8HI 0 "mve_scatter_memory")
8512    (match_operand:V8HI 1 "s_register_operand")
8513    (match_operand:V8HF 2 "s_register_operand")
8514    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8515   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8516 {
8517   rtx ind = XEXP (operands[0], 0);
8518   gcc_assert (REG_P (ind));
8519   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8520                                                        operands[2]));
8521   DONE;
8522 })
8523
8524 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8525   [(set (mem:BLK (scratch))
8526         (unspec:BLK
8527           [(match_operand:SI 0 "register_operand" "r")
8528            (match_operand:V8HI 1 "s_register_operand" "w")
8529            (match_operand:V8HF 2 "s_register_operand" "w")]
8530           VSTRHQSO_F))]
8531   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8532   "vstrh.16\t%q2, [%0, %q1]"
8533   [(set_attr "length" "4")])
8534
8535 ;;
8536 ;; [vstrhq_scatter_offset_p_f]
8537 ;;
8538 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8539   [(match_operand:V8HI 0 "mve_scatter_memory")
8540    (match_operand:V8HI 1 "s_register_operand")
8541    (match_operand:V8HF 2 "s_register_operand")
8542    (match_operand:HI 3 "vpr_register_operand")
8543    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8544   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8545 {
8546   rtx ind = XEXP (operands[0], 0);
8547   gcc_assert (REG_P (ind));
8548   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8549                                                          operands[2],
8550                                                          operands[3]));
8551   DONE;
8552 })
8553
8554 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8555   [(set (mem:BLK (scratch))
8556         (unspec:BLK
8557           [(match_operand:SI 0 "register_operand" "r")
8558            (match_operand:V8HI 1 "s_register_operand" "w")
8559            (match_operand:V8HF 2 "s_register_operand" "w")
8560            (match_operand:HI 3 "vpr_register_operand" "Up")]
8561           VSTRHQSO_F))]
8562   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8563   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8564   [(set_attr "length" "8")])
8565
8566 ;;
8567 ;; [vstrhq_scatter_shifted_offset_f]
8568 ;;
8569 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8570   [(match_operand:V8HI 0 "memory_operand" "=Us")
8571    (match_operand:V8HI 1 "s_register_operand" "w")
8572    (match_operand:V8HF 2 "s_register_operand" "w")
8573    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8574   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8575 {
8576   rtx ind = XEXP (operands[0], 0);
8577   gcc_assert (REG_P (ind));
8578   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8579                                                                operands[2]));
8580   DONE;
8581 })
8582
8583 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8584   [(set (mem:BLK (scratch))
8585         (unspec:BLK
8586           [(match_operand:SI 0 "register_operand" "r")
8587            (match_operand:V8HI 1 "s_register_operand" "w")
8588            (match_operand:V8HF 2 "s_register_operand" "w")]
8589           VSTRHQSSO_F))]
8590   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8591   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8592   [(set_attr "length" "4")])
8593
8594 ;;
8595 ;; [vstrhq_scatter_shifted_offset_p_f]
8596 ;;
8597 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8598   [(match_operand:V8HI 0 "memory_operand" "=Us")
8599    (match_operand:V8HI 1 "s_register_operand" "w")
8600    (match_operand:V8HF 2 "s_register_operand" "w")
8601    (match_operand:HI 3 "vpr_register_operand" "Up")
8602    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8603   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8604 {
8605   rtx ind = XEXP (operands[0], 0);
8606   gcc_assert (REG_P (ind));
8607   emit_insn (
8608     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8609                                                         operands[2],
8610                                                         operands[3]));
8611   DONE;
8612 })
8613
8614 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
8615   [(set (mem:BLK (scratch))
8616         (unspec:BLK
8617           [(match_operand:SI 0 "register_operand" "r")
8618            (match_operand:V8HI 1 "s_register_operand" "w")
8619            (match_operand:V8HF 2 "s_register_operand" "w")
8620            (match_operand:HI 3 "vpr_register_operand" "Up")]
8621           VSTRHQSSO_F))]
8622   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8623   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
8624   [(set_attr "length" "8")])
8625
8626 ;;
8627 ;; [vstrwq_scatter_base_f]
8628 ;;
8629 (define_insn "mve_vstrwq_scatter_base_fv4sf"
8630   [(set (mem:BLK (scratch))
8631         (unspec:BLK
8632                 [(match_operand:V4SI 0 "s_register_operand" "w")
8633                  (match_operand:SI 1 "immediate_operand" "i")
8634                  (match_operand:V4SF 2 "s_register_operand" "w")]
8635          VSTRWQSB_F))
8636   ]
8637   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8638 {
8639    rtx ops[3];
8640    ops[0] = operands[0];
8641    ops[1] = operands[1];
8642    ops[2] = operands[2];
8643    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
8644    return "";
8645 }
8646   [(set_attr "length" "4")])
8647
8648 ;;
8649 ;; [vstrwq_scatter_base_p_f]
8650 ;;
8651 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
8652   [(set (mem:BLK (scratch))
8653         (unspec:BLK
8654                 [(match_operand:V4SI 0 "s_register_operand" "w")
8655                  (match_operand:SI 1 "immediate_operand" "i")
8656                  (match_operand:V4SF 2 "s_register_operand" "w")
8657                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8658          VSTRWQSB_F))
8659   ]
8660   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8661 {
8662    rtx ops[3];
8663    ops[0] = operands[0];
8664    ops[1] = operands[1];
8665    ops[2] = operands[2];
8666    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8667    return "";
8668 }
8669   [(set_attr "length" "8")])
8670
8671 ;;
8672 ;; [vstrwq_scatter_offset_f]
8673 ;;
8674 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
8675   [(match_operand:V4SI 0 "mve_scatter_memory")
8676    (match_operand:V4SI 1 "s_register_operand")
8677    (match_operand:V4SF 2 "s_register_operand")
8678    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8679   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8680 {
8681   rtx ind = XEXP (operands[0], 0);
8682   gcc_assert (REG_P (ind));
8683   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
8684                                                        operands[2]));
8685   DONE;
8686 })
8687
8688 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
8689   [(set (mem:BLK (scratch))
8690         (unspec:BLK
8691           [(match_operand:SI 0 "register_operand" "r")
8692            (match_operand:V4SI 1 "s_register_operand" "w")
8693            (match_operand:V4SF 2 "s_register_operand" "w")]
8694           VSTRWQSO_F))]
8695   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8696   "vstrw.32\t%q2, [%0, %q1]"
8697   [(set_attr "length" "4")])
8698
8699 ;;
8700 ;; [vstrwq_scatter_offset_p_f]
8701 ;;
8702 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
8703   [(match_operand:V4SI 0 "mve_scatter_memory")
8704    (match_operand:V4SI 1 "s_register_operand")
8705    (match_operand:V4SF 2 "s_register_operand")
8706    (match_operand:HI 3 "vpr_register_operand")
8707    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8708   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8709 {
8710   rtx ind = XEXP (operands[0], 0);
8711   gcc_assert (REG_P (ind));
8712   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
8713                                                          operands[2],
8714                                                          operands[3]));
8715   DONE;
8716 })
8717
8718 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
8719   [(set (mem:BLK (scratch))
8720         (unspec:BLK
8721           [(match_operand:SI 0 "register_operand" "r")
8722            (match_operand:V4SI 1 "s_register_operand" "w")
8723            (match_operand:V4SF 2 "s_register_operand" "w")
8724            (match_operand:HI 3 "vpr_register_operand" "Up")]
8725           VSTRWQSO_F))]
8726   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8727   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8728   [(set_attr "length" "8")])
8729
8730 ;;
8731 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8732 ;;
8733 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
8734   [(match_operand:V4SI 0 "mve_scatter_memory")
8735    (match_operand:V4SI 1 "s_register_operand")
8736    (match_operand:V4SI 2 "s_register_operand")
8737    (match_operand:HI 3 "vpr_register_operand")
8738    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8739   "TARGET_HAVE_MVE"
8740 {
8741   rtx ind = XEXP (operands[0], 0);
8742   gcc_assert (REG_P (ind));
8743   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
8744                                                               operands[2],
8745                                                               operands[3]));
8746   DONE;
8747 })
8748
8749 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
8750   [(set (mem:BLK (scratch))
8751         (unspec:BLK
8752           [(match_operand:SI 0 "register_operand" "r")
8753            (match_operand:V4SI 1 "s_register_operand" "w")
8754            (match_operand:V4SI 2 "s_register_operand" "w")
8755            (match_operand:HI 3 "vpr_register_operand" "Up")]
8756           VSTRWSOQ))]
8757   "TARGET_HAVE_MVE"
8758   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8759   [(set_attr "length" "8")])
8760
8761 ;;
8762 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8763 ;;
8764 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
8765   [(match_operand:V4SI 0 "mve_scatter_memory")
8766    (match_operand:V4SI 1 "s_register_operand")
8767    (match_operand:V4SI 2 "s_register_operand")
8768    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8769   "TARGET_HAVE_MVE"
8770 {
8771   rtx ind = XEXP (operands[0], 0);
8772   gcc_assert (REG_P (ind));
8773   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
8774                                                             operands[2]));
8775   DONE;
8776 })
8777
8778 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
8779   [(set (mem:BLK (scratch))
8780         (unspec:BLK
8781           [(match_operand:SI 0 "register_operand" "r")
8782            (match_operand:V4SI 1 "s_register_operand" "w")
8783            (match_operand:V4SI 2 "s_register_operand" "w")]
8784           VSTRWSOQ))]
8785   "TARGET_HAVE_MVE"
8786   "vstrw.32\t%q2, [%0, %q1]"
8787   [(set_attr "length" "4")])
8788
8789 ;;
8790 ;; [vstrwq_scatter_shifted_offset_f]
8791 ;;
8792 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
8793   [(match_operand:V4SI 0 "mve_scatter_memory")
8794    (match_operand:V4SI 1 "s_register_operand")
8795    (match_operand:V4SF 2 "s_register_operand")
8796    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8797   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8798 {
8799   rtx ind = XEXP (operands[0], 0);
8800   gcc_assert (REG_P (ind));
8801   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
8802                                                                operands[2]));
8803   DONE;
8804 })
8805
8806 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
8807   [(set (mem:BLK (scratch))
8808         (unspec:BLK
8809           [(match_operand:SI 0 "register_operand" "r")
8810            (match_operand:V4SI 1 "s_register_operand" "w")
8811            (match_operand:V4SF 2 "s_register_operand" "w")]
8812          VSTRWQSSO_F))]
8813   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8814   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8815   [(set_attr "length" "8")])
8816
8817 ;;
8818 ;; [vstrwq_scatter_shifted_offset_p_f]
8819 ;;
8820 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
8821   [(match_operand:V4SI 0 "mve_scatter_memory")
8822    (match_operand:V4SI 1 "s_register_operand")
8823    (match_operand:V4SF 2 "s_register_operand")
8824    (match_operand:HI 3 "vpr_register_operand")
8825    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8826   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8827 {
8828   rtx ind = XEXP (operands[0], 0);
8829   gcc_assert (REG_P (ind));
8830   emit_insn (
8831     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
8832                                                         operands[2],
8833                                                         operands[3]));
8834   DONE;
8835 })
8836
8837 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
8838   [(set (mem:BLK (scratch))
8839         (unspec:BLK
8840           [(match_operand:SI 0 "register_operand" "r")
8841            (match_operand:V4SI 1 "s_register_operand" "w")
8842            (match_operand:V4SF 2 "s_register_operand" "w")
8843            (match_operand:HI 3 "vpr_register_operand" "Up")]
8844           VSTRWQSSO_F))]
8845   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8846   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8847   [(set_attr "length" "8")])
8848
8849 ;;
8850 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
8851 ;;
8852 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
8853   [(match_operand:V4SI 0 "mve_scatter_memory")
8854    (match_operand:V4SI 1 "s_register_operand")
8855    (match_operand:V4SI 2 "s_register_operand")
8856    (match_operand:HI 3 "vpr_register_operand")
8857    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8858   "TARGET_HAVE_MVE"
8859 {
8860   rtx ind = XEXP (operands[0], 0);
8861   gcc_assert (REG_P (ind));
8862   emit_insn (
8863     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
8864                                                              operands[2],
8865                                                              operands[3]));
8866   DONE;
8867 })
8868
8869 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
8870   [(set (mem:BLK (scratch))
8871         (unspec:BLK
8872           [(match_operand:SI 0 "register_operand" "r")
8873            (match_operand:V4SI 1 "s_register_operand" "w")
8874            (match_operand:V4SI 2 "s_register_operand" "w")
8875            (match_operand:HI 3 "vpr_register_operand" "Up")]
8876           VSTRWSSOQ))]
8877   "TARGET_HAVE_MVE"
8878   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8879   [(set_attr "length" "8")])
8880
8881 ;;
8882 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
8883 ;;
8884 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
8885   [(match_operand:V4SI 0 "mve_scatter_memory")
8886    (match_operand:V4SI 1 "s_register_operand")
8887    (match_operand:V4SI 2 "s_register_operand")
8888    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8889   "TARGET_HAVE_MVE"
8890 {
8891   rtx ind = XEXP (operands[0], 0);
8892   gcc_assert (REG_P (ind));
8893   emit_insn (
8894     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
8895                                                            operands[2]));
8896   DONE;
8897 })
8898
8899 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
8900   [(set (mem:BLK (scratch))
8901         (unspec:BLK
8902           [(match_operand:SI 0 "register_operand" "r")
8903            (match_operand:V4SI 1 "s_register_operand" "w")
8904            (match_operand:V4SI 2 "s_register_operand" "w")]
8905           VSTRWSSOQ))]
8906   "TARGET_HAVE_MVE"
8907   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8908   [(set_attr "length" "4")])
8909
8910 ;;
8911 ;; [vaddq_s, vaddq_u])
8912 ;;
8913 (define_insn "mve_vaddq<mode>"
8914   [
8915    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
8916         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
8917                     (match_operand:MVE_2 2 "s_register_operand" "w")))
8918   ]
8919   "TARGET_HAVE_MVE"
8920   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
8921   [(set_attr "type" "mve_move")
8922 ])
8923
8924 ;;
8925 ;; [vaddq_f])
8926 ;;
8927 (define_insn "mve_vaddq_f<mode>"
8928   [
8929    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8930         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
8931                     (match_operand:MVE_0 2 "s_register_operand" "w")))
8932   ]
8933   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8934   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
8935   [(set_attr "type" "mve_move")
8936 ])
8937
8938 ;;
8939 ;; [vidupq_n_u])
8940 ;;
8941 (define_expand "mve_vidupq_n_u<mode>"
8942  [(match_operand:MVE_2 0 "s_register_operand")
8943   (match_operand:SI 1 "s_register_operand")
8944   (match_operand:SI 2 "mve_imm_selective_upto_8")]
8945  "TARGET_HAVE_MVE"
8946 {
8947   rtx temp = gen_reg_rtx (SImode);
8948   emit_move_insn (temp, operands[1]);
8949   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
8950   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
8951                                           operands[2], inc));
8952   DONE;
8953 })
8954
8955 ;;
8956 ;; [vidupq_u_insn])
8957 ;;
8958 (define_insn "mve_vidupq_u<mode>_insn"
8959  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8960        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
8961                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
8962          VIDUPQ))
8963   (set (match_operand:SI 1 "s_register_operand" "=Te")
8964        (plus:SI (match_dup 2)
8965                 (match_operand:SI 4 "immediate_operand" "i")))]
8966  "TARGET_HAVE_MVE"
8967  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
8968
8969 ;;
8970 ;; [vidupq_m_n_u])
8971 ;;
8972 (define_expand "mve_vidupq_m_n_u<mode>"
8973   [(match_operand:MVE_2 0 "s_register_operand")
8974    (match_operand:MVE_2 1 "s_register_operand")
8975    (match_operand:SI 2 "s_register_operand")
8976    (match_operand:SI 3 "mve_imm_selective_upto_8")
8977    (match_operand:HI 4 "vpr_register_operand")]
8978   "TARGET_HAVE_MVE"
8979 {
8980   rtx temp = gen_reg_rtx (SImode);
8981   emit_move_insn (temp, operands[2]);
8982   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
8983   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
8984                                              operands[2], operands[3],
8985                                              operands[4], inc));
8986   DONE;
8987 })
8988
8989 ;;
8990 ;; [vidupq_m_wb_u_insn])
8991 ;;
8992 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
8993  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8994        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
8995                       (match_operand:SI 3 "s_register_operand" "2")
8996                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
8997                       (match_operand:HI 5 "vpr_register_operand" "Up")]
8998         VIDUPQ_M))
8999   (set (match_operand:SI 2 "s_register_operand" "=Te")
9000        (plus:SI (match_dup 3)
9001                 (match_operand:SI 6 "immediate_operand" "i")))]
9002  "TARGET_HAVE_MVE"
9003  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9004  [(set_attr "length""8")])
9005
9006 ;;
9007 ;; [vddupq_n_u])
9008 ;;
9009 (define_expand "mve_vddupq_n_u<mode>"
9010  [(match_operand:MVE_2 0 "s_register_operand")
9011   (match_operand:SI 1 "s_register_operand")
9012   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9013  "TARGET_HAVE_MVE"
9014 {
9015   rtx temp = gen_reg_rtx (SImode);
9016   emit_move_insn (temp, operands[1]);
9017   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9018   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9019                                           operands[2], inc));
9020   DONE;
9021 })
9022
9023 ;;
9024 ;; [vddupq_u_insn])
9025 ;;
9026 (define_insn "mve_vddupq_u<mode>_insn"
9027  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9028        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9029                       (match_operand:SI 3 "immediate_operand" "i")]
9030         VDDUPQ))
9031   (set (match_operand:SI 1 "s_register_operand" "=Te")
9032        (minus:SI (match_dup 2)
9033                  (match_operand:SI 4 "immediate_operand" "i")))]
9034  "TARGET_HAVE_MVE"
9035  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9036
9037 ;;
9038 ;; [vddupq_m_n_u])
9039 ;;
9040 (define_expand "mve_vddupq_m_n_u<mode>"
9041   [(match_operand:MVE_2 0 "s_register_operand")
9042    (match_operand:MVE_2 1 "s_register_operand")
9043    (match_operand:SI 2 "s_register_operand")
9044    (match_operand:SI 3 "mve_imm_selective_upto_8")
9045    (match_operand:HI 4 "vpr_register_operand")]
9046   "TARGET_HAVE_MVE"
9047 {
9048   rtx temp = gen_reg_rtx (SImode);
9049   emit_move_insn (temp, operands[2]);
9050   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9051   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9052                                              operands[2], operands[3],
9053                                              operands[4], inc));
9054   DONE;
9055 })
9056
9057 ;;
9058 ;; [vddupq_m_wb_u_insn])
9059 ;;
9060 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9061  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9062        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9063                       (match_operand:SI 3 "s_register_operand" "2")
9064                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9065                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9066         VDDUPQ_M))
9067   (set (match_operand:SI 2 "s_register_operand" "=Te")
9068        (minus:SI (match_dup 3)
9069                  (match_operand:SI 6 "immediate_operand" "i")))]
9070  "TARGET_HAVE_MVE"
9071  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9072  [(set_attr "length""8")])
9073
9074 ;;
9075 ;; [vdwdupq_n_u])
9076 ;;
9077 (define_expand "mve_vdwdupq_n_u<mode>"
9078  [(match_operand:MVE_2 0 "s_register_operand")
9079   (match_operand:SI 1 "s_register_operand")
9080   (match_operand:DI 2 "s_register_operand")
9081   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9082  "TARGET_HAVE_MVE"
9083 {
9084   rtx ignore_wb = gen_reg_rtx (SImode);
9085   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9086                                               operands[1], operands[2],
9087                                               operands[3]));
9088   DONE;
9089 })
9090
9091 ;;
9092 ;; [vdwdupq_wb_u])
9093 ;;
9094 (define_expand "mve_vdwdupq_wb_u<mode>"
9095  [(match_operand:SI 0 "s_register_operand")
9096   (match_operand:SI 1 "s_register_operand")
9097   (match_operand:DI 2 "s_register_operand")
9098   (match_operand:SI 3 "mve_imm_selective_upto_8")
9099   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9100  "TARGET_HAVE_MVE"
9101 {
9102   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9103   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9104                                               operands[1], operands[2],
9105                                               operands[3]));
9106   DONE;
9107 })
9108
9109 ;;
9110 ;; [vdwdupq_wb_u_insn])
9111 ;;
9112 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9113   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9114         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9115                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9116                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9117          VDWDUPQ))
9118    (set (match_operand:SI 1 "s_register_operand" "=Te")
9119         (unspec:SI [(match_dup 2)
9120                     (subreg:SI (match_dup 3) 4)
9121                     (match_dup 4)]
9122          VDWDUPQ))]
9123   "TARGET_HAVE_MVE"
9124   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9125 )
9126
9127 ;;
9128 ;; [vdwdupq_m_n_u])
9129 ;;
9130 (define_expand "mve_vdwdupq_m_n_u<mode>"
9131  [(match_operand:MVE_2 0 "s_register_operand")
9132   (match_operand:MVE_2 1 "s_register_operand")
9133   (match_operand:SI 2 "s_register_operand")
9134   (match_operand:DI 3 "s_register_operand")
9135   (match_operand:SI 4 "mve_imm_selective_upto_8")
9136   (match_operand:HI 5 "vpr_register_operand")]
9137  "TARGET_HAVE_MVE"
9138 {
9139   rtx ignore_wb = gen_reg_rtx (SImode);
9140   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9141                                                 operands[1], operands[2],
9142                                                 operands[3], operands[4],
9143                                                 operands[5]));
9144   DONE;
9145 })
9146
9147 ;;
9148 ;; [vdwdupq_m_wb_u])
9149 ;;
9150 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9151  [(match_operand:SI 0 "s_register_operand")
9152   (match_operand:MVE_2 1 "s_register_operand")
9153   (match_operand:SI 2 "s_register_operand")
9154   (match_operand:DI 3 "s_register_operand")
9155   (match_operand:SI 4 "mve_imm_selective_upto_8")
9156   (match_operand:HI 5 "vpr_register_operand")]
9157  "TARGET_HAVE_MVE"
9158 {
9159   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9160   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9161                                                 operands[1], operands[2],
9162                                                 operands[3], operands[4],
9163                                                 operands[5]));
9164   DONE;
9165 })
9166
9167 ;;
9168 ;; [vdwdupq_m_wb_u_insn])
9169 ;;
9170 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9171   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9172         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9173                        (match_operand:SI 3 "s_register_operand" "1")
9174                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9175                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9176                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9177          VDWDUPQ_M))
9178    (set (match_operand:SI 1 "s_register_operand" "=Te")
9179         (unspec:SI [(match_dup 2)
9180                     (match_dup 3)
9181                     (subreg:SI (match_dup 4) 4)
9182                     (match_dup 5)
9183                     (match_dup 6)]
9184          VDWDUPQ_M))
9185   ]
9186   "TARGET_HAVE_MVE"
9187   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9188   [(set_attr "type" "mve_move")
9189    (set_attr "length""8")])
9190
9191 ;;
9192 ;; [viwdupq_n_u])
9193 ;;
9194 (define_expand "mve_viwdupq_n_u<mode>"
9195  [(match_operand:MVE_2 0 "s_register_operand")
9196   (match_operand:SI 1 "s_register_operand")
9197   (match_operand:DI 2 "s_register_operand")
9198   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9199  "TARGET_HAVE_MVE"
9200 {
9201   rtx ignore_wb = gen_reg_rtx (SImode);
9202   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9203                                               operands[1], operands[2],
9204                                               operands[3]));
9205   DONE;
9206 })
9207
9208 ;;
9209 ;; [viwdupq_wb_u])
9210 ;;
9211 (define_expand "mve_viwdupq_wb_u<mode>"
9212  [(match_operand:SI 0 "s_register_operand")
9213   (match_operand:SI 1 "s_register_operand")
9214   (match_operand:DI 2 "s_register_operand")
9215   (match_operand:SI 3 "mve_imm_selective_upto_8")
9216   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9217  "TARGET_HAVE_MVE"
9218 {
9219   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9220   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9221                                               operands[1], operands[2],
9222                                               operands[3]));
9223   DONE;
9224 })
9225
9226 ;;
9227 ;; [viwdupq_wb_u_insn])
9228 ;;
9229 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9230   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9231         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9232                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9233                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9234          VIWDUPQ))
9235    (set (match_operand:SI 1 "s_register_operand" "=Te")
9236         (unspec:SI [(match_dup 2)
9237                     (subreg:SI (match_dup 3) 4)
9238                     (match_dup 4)]
9239          VIWDUPQ))]
9240   "TARGET_HAVE_MVE"
9241   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9242 )
9243
9244 ;;
9245 ;; [viwdupq_m_n_u])
9246 ;;
9247 (define_expand "mve_viwdupq_m_n_u<mode>"
9248  [(match_operand:MVE_2 0 "s_register_operand")
9249   (match_operand:MVE_2 1 "s_register_operand")
9250   (match_operand:SI 2 "s_register_operand")
9251   (match_operand:DI 3 "s_register_operand")
9252   (match_operand:SI 4 "mve_imm_selective_upto_8")
9253   (match_operand:HI 5 "vpr_register_operand")]
9254  "TARGET_HAVE_MVE"
9255 {
9256   rtx ignore_wb = gen_reg_rtx (SImode);
9257   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9258                                                 operands[1], operands[2],
9259                                                 operands[3], operands[4],
9260                                                 operands[5]));
9261   DONE;
9262 })
9263
9264 ;;
9265 ;; [viwdupq_m_wb_u])
9266 ;;
9267 (define_expand "mve_viwdupq_m_wb_u<mode>"
9268  [(match_operand:SI 0 "s_register_operand")
9269   (match_operand:MVE_2 1 "s_register_operand")
9270   (match_operand:SI 2 "s_register_operand")
9271   (match_operand:DI 3 "s_register_operand")
9272   (match_operand:SI 4 "mve_imm_selective_upto_8")
9273   (match_operand:HI 5 "vpr_register_operand")]
9274  "TARGET_HAVE_MVE"
9275 {
9276   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9277   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9278                                                 operands[1], operands[2],
9279                                                 operands[3], operands[4],
9280                                                 operands[5]));
9281   DONE;
9282 })
9283
9284 ;;
9285 ;; [viwdupq_m_wb_u_insn])
9286 ;;
9287 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9288   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9289         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9290                        (match_operand:SI 3 "s_register_operand" "1")
9291                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9292                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9293                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9294          VIWDUPQ_M))
9295    (set (match_operand:SI 1 "s_register_operand" "=Te")
9296         (unspec:SI [(match_dup 2)
9297                     (match_dup 3)
9298                     (subreg:SI (match_dup 4) 4)
9299                     (match_dup 5)
9300                     (match_dup 6)]
9301          VIWDUPQ_M))
9302   ]
9303   "TARGET_HAVE_MVE"
9304   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9305   [(set_attr "type" "mve_move")
9306    (set_attr "length""8")])
9307
9308 ;;
9309 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9310 ;;
9311 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9312   [(set (mem:BLK (scratch))
9313         (unspec:BLK
9314                 [(match_operand:V4SI 1 "s_register_operand" "0")
9315                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9316                  (match_operand:V4SI 3 "s_register_operand" "w")]
9317          VSTRWSBWBQ))
9318    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9319         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9320          VSTRWSBWBQ))
9321   ]
9322   "TARGET_HAVE_MVE"
9323 {
9324    rtx ops[3];
9325    ops[0] = operands[1];
9326    ops[1] = operands[2];
9327    ops[2] = operands[3];
9328    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9329    return "";
9330 }
9331   [(set_attr "length" "4")])
9332
9333 ;;
9334 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9335 ;;
9336 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9337  [(set (mem:BLK (scratch))
9338        (unspec:BLK
9339                 [(match_operand:V4SI 1 "s_register_operand" "0")
9340                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9341                  (match_operand:V4SI 3 "s_register_operand" "w")
9342                  (match_operand:HI 4 "vpr_register_operand")]
9343         VSTRWSBWBQ))
9344    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9345         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9346          VSTRWSBWBQ))
9347   ]
9348   "TARGET_HAVE_MVE"
9349 {
9350    rtx ops[3];
9351    ops[0] = operands[1];
9352    ops[1] = operands[2];
9353    ops[2] = operands[3];
9354    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9355    return "";
9356 }
9357   [(set_attr "length" "8")])
9358
9359 ;;
9360 ;; [vstrwq_scatter_base_wb_f]
9361 ;;
9362 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9363  [(set (mem:BLK (scratch))
9364        (unspec:BLK
9365                 [(match_operand:V4SI 1 "s_register_operand" "0")
9366                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9367                  (match_operand:V4SF 3 "s_register_operand" "w")]
9368          VSTRWQSBWB_F))
9369    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9370         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9371          VSTRWQSBWB_F))
9372   ]
9373   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9374 {
9375    rtx ops[3];
9376    ops[0] = operands[1];
9377    ops[1] = operands[2];
9378    ops[2] = operands[3];
9379    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9380    return "";
9381 }
9382   [(set_attr "length" "4")])
9383
9384 ;;
9385 ;; [vstrwq_scatter_base_wb_p_f]
9386 ;;
9387 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9388  [(set (mem:BLK (scratch))
9389        (unspec:BLK
9390                 [(match_operand:V4SI 1 "s_register_operand" "0")
9391                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9392                  (match_operand:V4SF 3 "s_register_operand" "w")
9393                  (match_operand:HI 4 "vpr_register_operand")]
9394         VSTRWQSBWB_F))
9395    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9396         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9397          VSTRWQSBWB_F))
9398   ]
9399   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9400 {
9401    rtx ops[3];
9402    ops[0] = operands[1];
9403    ops[1] = operands[2];
9404    ops[2] = operands[3];
9405    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9406    return "";
9407 }
9408   [(set_attr "length" "8")])
9409
9410 ;;
9411 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9412 ;;
9413 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9414   [(set (mem:BLK (scratch))
9415         (unspec:BLK
9416                 [(match_operand:V2DI 1 "s_register_operand" "0")
9417                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9418                  (match_operand:V2DI 3 "s_register_operand" "w")]
9419          VSTRDSBWBQ))
9420    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9421         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9422          VSTRDSBWBQ))
9423   ]
9424   "TARGET_HAVE_MVE"
9425 {
9426    rtx ops[3];
9427    ops[0] = operands[1];
9428    ops[1] = operands[2];
9429    ops[2] = operands[3];
9430    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9431    return "";
9432 }
9433   [(set_attr "length" "4")])
9434
9435 ;;
9436 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9437 ;;
9438 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9439   [(set (mem:BLK (scratch))
9440         (unspec:BLK
9441                 [(match_operand:V2DI 1 "s_register_operand" "0")
9442                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9443                  (match_operand:V2DI 3 "s_register_operand" "w")
9444                  (match_operand:HI 4 "vpr_register_operand")]
9445          VSTRDSBWBQ))
9446    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9447         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9448          VSTRDSBWBQ))
9449   ]
9450   "TARGET_HAVE_MVE"
9451 {
9452    rtx ops[3];
9453    ops[0] = operands[1];
9454    ops[1] = operands[2];
9455    ops[2] = operands[3];
9456    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9457    return "";
9458 }
9459   [(set_attr "length" "8")])
9460
9461 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9462   [(match_operand:V4SI 0 "s_register_operand")
9463    (match_operand:V4SI 1 "s_register_operand")
9464    (match_operand:SI 2 "mve_vldrd_immediate")
9465    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9466   "TARGET_HAVE_MVE"
9467 {
9468   rtx ignore_result = gen_reg_rtx (V4SImode);
9469   emit_insn (
9470   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9471                                                  operands[1], operands[2]));
9472   DONE;
9473 })
9474
9475 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9476   [(match_operand:V4SI 0 "s_register_operand")
9477    (match_operand:V4SI 1 "s_register_operand")
9478    (match_operand:SI 2 "mve_vldrd_immediate")
9479    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9480   "TARGET_HAVE_MVE"
9481 {
9482   rtx ignore_wb = gen_reg_rtx (V4SImode);
9483   emit_insn (
9484   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9485                                                  operands[1], operands[2]));
9486   DONE;
9487 })
9488
9489 ;;
9490 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9491 ;;
9492 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9493   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9494         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9495                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9496                       (mem:BLK (scratch))]
9497          VLDRWGBWBQ))
9498    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9499         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9500          VLDRWGBWBQ))
9501   ]
9502   "TARGET_HAVE_MVE"
9503 {
9504    rtx ops[3];
9505    ops[0] = operands[0];
9506    ops[1] = operands[2];
9507    ops[2] = operands[3];
9508    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9509    return "";
9510 }
9511   [(set_attr "length" "4")])
9512
9513 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9514   [(match_operand:V4SI 0 "s_register_operand")
9515    (match_operand:V4SI 1 "s_register_operand")
9516    (match_operand:SI 2 "mve_vldrd_immediate")
9517    (match_operand:HI 3 "vpr_register_operand")
9518    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9519   "TARGET_HAVE_MVE"
9520 {
9521   rtx ignore_result = gen_reg_rtx (V4SImode);
9522   emit_insn (
9523   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9524                                                    operands[1], operands[2],
9525                                                    operands[3]));
9526   DONE;
9527 })
9528 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9529   [(match_operand:V4SI 0 "s_register_operand")
9530    (match_operand:V4SI 1 "s_register_operand")
9531    (match_operand:SI 2 "mve_vldrd_immediate")
9532    (match_operand:HI 3 "vpr_register_operand")
9533    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9534   "TARGET_HAVE_MVE"
9535 {
9536   rtx ignore_wb = gen_reg_rtx (V4SImode);
9537   emit_insn (
9538   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9539                                                    operands[1], operands[2],
9540                                                    operands[3]));
9541   DONE;
9542 })
9543
9544 ;;
9545 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9546 ;;
9547 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9548   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9549         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9550                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9551                       (match_operand:HI 4 "vpr_register_operand" "Up")
9552                       (mem:BLK (scratch))]
9553          VLDRWGBWBQ))
9554    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9555         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9556          VLDRWGBWBQ))
9557   ]
9558   "TARGET_HAVE_MVE"
9559 {
9560    rtx ops[3];
9561    ops[0] = operands[0];
9562    ops[1] = operands[2];
9563    ops[2] = operands[3];
9564    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9565    return "";
9566 }
9567   [(set_attr "length" "8")])
9568
9569 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9570   [(match_operand:V4SI 0 "s_register_operand")
9571    (match_operand:V4SI 1 "s_register_operand")
9572    (match_operand:SI 2 "mve_vldrd_immediate")
9573    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9574   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9575 {
9576   rtx ignore_result = gen_reg_rtx (V4SFmode);
9577   emit_insn (
9578   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9579                                             operands[1], operands[2]));
9580   DONE;
9581 })
9582
9583 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9584   [(match_operand:V4SF 0 "s_register_operand")
9585    (match_operand:V4SI 1 "s_register_operand")
9586    (match_operand:SI 2 "mve_vldrd_immediate")
9587    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9588   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9589 {
9590   rtx ignore_wb = gen_reg_rtx (V4SImode);
9591   emit_insn (
9592   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9593                                             operands[1], operands[2]));
9594   DONE;
9595 })
9596
9597 ;;
9598 ;; [vldrwq_gather_base_wb_f]
9599 ;;
9600 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9601   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9602         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9603                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9604                       (mem:BLK (scratch))]
9605          VLDRWQGBWB_F))
9606    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9607         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9608          VLDRWQGBWB_F))
9609   ]
9610   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9611 {
9612    rtx ops[3];
9613    ops[0] = operands[0];
9614    ops[1] = operands[2];
9615    ops[2] = operands[3];
9616    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9617    return "";
9618 }
9619   [(set_attr "length" "4")])
9620
9621 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
9622   [(match_operand:V4SI 0 "s_register_operand")
9623    (match_operand:V4SI 1 "s_register_operand")
9624    (match_operand:SI 2 "mve_vldrd_immediate")
9625    (match_operand:HI 3 "vpr_register_operand")
9626    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9627   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9628 {
9629   rtx ignore_result = gen_reg_rtx (V4SFmode);
9630   emit_insn (
9631   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
9632                                               operands[1], operands[2],
9633                                               operands[3]));
9634   DONE;
9635 })
9636
9637 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
9638   [(match_operand:V4SF 0 "s_register_operand")
9639    (match_operand:V4SI 1 "s_register_operand")
9640    (match_operand:SI 2 "mve_vldrd_immediate")
9641    (match_operand:HI 3 "vpr_register_operand")
9642    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9643   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9644 {
9645   rtx ignore_wb = gen_reg_rtx (V4SImode);
9646   emit_insn (
9647   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
9648                                               operands[1], operands[2],
9649                                               operands[3]));
9650   DONE;
9651 })
9652
9653 ;;
9654 ;; [vldrwq_gather_base_wb_z_f]
9655 ;;
9656 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
9657   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9658         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9659                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9660                       (match_operand:HI 4 "vpr_register_operand" "Up")
9661                       (mem:BLK (scratch))]
9662          VLDRWQGBWB_F))
9663    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9664         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9665          VLDRWQGBWB_F))
9666   ]
9667   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9668 {
9669    rtx ops[3];
9670    ops[0] = operands[0];
9671    ops[1] = operands[2];
9672    ops[2] = operands[3];
9673    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9674    return "";
9675 }
9676   [(set_attr "length" "8")])
9677
9678 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
9679   [(match_operand:V2DI 0 "s_register_operand")
9680    (match_operand:V2DI 1 "s_register_operand")
9681    (match_operand:SI 2 "mve_vldrd_immediate")
9682    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9683   "TARGET_HAVE_MVE"
9684 {
9685   rtx ignore_result = gen_reg_rtx (V2DImode);
9686   emit_insn (
9687   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
9688                                                  operands[1], operands[2]));
9689   DONE;
9690 })
9691
9692 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
9693   [(match_operand:V2DI 0 "s_register_operand")
9694    (match_operand:V2DI 1 "s_register_operand")
9695    (match_operand:SI 2 "mve_vldrd_immediate")
9696    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9697   "TARGET_HAVE_MVE"
9698 {
9699   rtx ignore_wb = gen_reg_rtx (V2DImode);
9700   emit_insn (
9701   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
9702                                                  operands[1], operands[2]));
9703   DONE;
9704 })
9705
9706
9707 ;;
9708 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
9709 ;;
9710 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
9711   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9712         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9713                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9714                       (mem:BLK (scratch))]
9715          VLDRDGBWBQ))
9716    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9717         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9718          VLDRDGBWBQ))
9719   ]
9720   "TARGET_HAVE_MVE"
9721 {
9722    rtx ops[3];
9723    ops[0] = operands[0];
9724    ops[1] = operands[2];
9725    ops[2] = operands[3];
9726    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
9727    return "";
9728 }
9729   [(set_attr "length" "4")])
9730
9731 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
9732   [(match_operand:V2DI 0 "s_register_operand")
9733    (match_operand:V2DI 1 "s_register_operand")
9734    (match_operand:SI 2 "mve_vldrd_immediate")
9735    (match_operand:HI 3 "vpr_register_operand")
9736    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9737   "TARGET_HAVE_MVE"
9738 {
9739   rtx ignore_result = gen_reg_rtx (V2DImode);
9740   emit_insn (
9741   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
9742                                                    operands[1], operands[2],
9743                                                    operands[3]));
9744   DONE;
9745 })
9746
9747 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
9748   [(match_operand:V2DI 0 "s_register_operand")
9749    (match_operand:V2DI 1 "s_register_operand")
9750    (match_operand:SI 2 "mve_vldrd_immediate")
9751    (match_operand:HI 3 "vpr_register_operand")
9752    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9753   "TARGET_HAVE_MVE"
9754 {
9755   rtx ignore_wb = gen_reg_rtx (V2DImode);
9756   emit_insn (
9757   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
9758                                                    operands[1], operands[2],
9759                                                    operands[3]));
9760   DONE;
9761 })
9762
9763 (define_insn "get_fpscr_nzcvqc"
9764  [(set (match_operand:SI 0 "register_operand" "=r")
9765    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
9766  "TARGET_HAVE_MVE"
9767  "vmrs\\t%0, FPSCR_nzcvqc"
9768   [(set_attr "type" "mve_move")])
9769
9770 (define_insn "set_fpscr_nzcvqc"
9771  [(set (reg:SI VFPCC_REGNUM)
9772    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
9773     VUNSPEC_SET_FPSCR_NZCVQC))]
9774  "TARGET_HAVE_MVE"
9775  "vmsr\\tFPSCR_nzcvqc, %0"
9776   [(set_attr "type" "mve_move")])
9777
9778 ;;
9779 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
9780 ;;
9781 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
9782   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9783         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9784                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9785                       (match_operand:HI 4 "vpr_register_operand" "Up")
9786                       (mem:BLK (scratch))]
9787          VLDRDGBWBQ))
9788    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9789         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9790          VLDRDGBWBQ))
9791   ]
9792   "TARGET_HAVE_MVE"
9793 {
9794    rtx ops[3];
9795    ops[0] = operands[0];
9796    ops[1] = operands[2];
9797    ops[2] = operands[3];
9798    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
9799    return "";
9800 }
9801   [(set_attr "length" "8")])
9802 ;;
9803 ;; [vadciq_m_s, vadciq_m_u])
9804 ;;
9805 (define_insn "mve_vadciq_m_<supf>v4si"
9806   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9807         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9808                       (match_operand:V4SI 2 "s_register_operand" "w")
9809                       (match_operand:V4SI 3 "s_register_operand" "w")
9810                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9811          VADCIQ_M))
9812    (set (reg:SI VFPCC_REGNUM)
9813         (unspec:SI [(const_int 0)]
9814          VADCIQ_M))
9815   ]
9816   "TARGET_HAVE_MVE"
9817   "vpst\;vadcit.i32\t%q0, %q2, %q3"
9818   [(set_attr "type" "mve_move")
9819    (set_attr "length" "8")])
9820
9821 ;;
9822 ;; [vadciq_u, vadciq_s])
9823 ;;
9824 (define_insn "mve_vadciq_<supf>v4si"
9825   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9826         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9827                       (match_operand:V4SI 2 "s_register_operand" "w")]
9828          VADCIQ))
9829    (set (reg:SI VFPCC_REGNUM)
9830         (unspec:SI [(const_int 0)]
9831          VADCIQ))
9832   ]
9833   "TARGET_HAVE_MVE"
9834   "vadci.i32\t%q0, %q1, %q2"
9835   [(set_attr "type" "mve_move")
9836    (set_attr "length" "4")])
9837
9838 ;;
9839 ;; [vadcq_m_s, vadcq_m_u])
9840 ;;
9841 (define_insn "mve_vadcq_m_<supf>v4si"
9842   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9843         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9844                       (match_operand:V4SI 2 "s_register_operand" "w")
9845                       (match_operand:V4SI 3 "s_register_operand" "w")
9846                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9847          VADCQ_M))
9848    (set (reg:SI VFPCC_REGNUM)
9849         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9850          VADCQ_M))
9851   ]
9852   "TARGET_HAVE_MVE"
9853   "vpst\;vadct.i32\t%q0, %q2, %q3"
9854   [(set_attr "type" "mve_move")
9855    (set_attr "length" "8")])
9856
9857 ;;
9858 ;; [vadcq_u, vadcq_s])
9859 ;;
9860 (define_insn "mve_vadcq_<supf>v4si"
9861   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9862         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9863                        (match_operand:V4SI 2 "s_register_operand" "w")]
9864          VADCQ))
9865    (set (reg:SI VFPCC_REGNUM)
9866         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9867          VADCQ))
9868   ]
9869   "TARGET_HAVE_MVE"
9870   "vadc.i32\t%q0, %q1, %q2"
9871   [(set_attr "type" "mve_move")
9872    (set_attr "length" "4")
9873    (set_attr "conds" "set")])
9874
9875 ;;
9876 ;; [vsbciq_m_u, vsbciq_m_s])
9877 ;;
9878 (define_insn "mve_vsbciq_m_<supf>v4si"
9879   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9880         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9881                       (match_operand:V4SI 2 "s_register_operand" "w")
9882                       (match_operand:V4SI 3 "s_register_operand" "w")
9883                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9884          VSBCIQ_M))
9885    (set (reg:SI VFPCC_REGNUM)
9886         (unspec:SI [(const_int 0)]
9887          VSBCIQ_M))
9888   ]
9889   "TARGET_HAVE_MVE"
9890   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
9891   [(set_attr "type" "mve_move")
9892    (set_attr "length" "8")])
9893
9894 ;;
9895 ;; [vsbciq_s, vsbciq_u])
9896 ;;
9897 (define_insn "mve_vsbciq_<supf>v4si"
9898   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9899         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9900                       (match_operand:V4SI 2 "s_register_operand" "w")]
9901          VSBCIQ))
9902    (set (reg:SI VFPCC_REGNUM)
9903         (unspec:SI [(const_int 0)]
9904          VSBCIQ))
9905   ]
9906   "TARGET_HAVE_MVE"
9907   "vsbci.i32\t%q0, %q1, %q2"
9908   [(set_attr "type" "mve_move")
9909    (set_attr "length" "4")])
9910
9911 ;;
9912 ;; [vsbcq_m_u, vsbcq_m_s])
9913 ;;
9914 (define_insn "mve_vsbcq_m_<supf>v4si"
9915   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9916         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9917                       (match_operand:V4SI 2 "s_register_operand" "w")
9918                       (match_operand:V4SI 3 "s_register_operand" "w")
9919                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9920          VSBCQ_M))
9921    (set (reg:SI VFPCC_REGNUM)
9922         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9923          VSBCQ_M))
9924   ]
9925   "TARGET_HAVE_MVE"
9926   "vpst\;vsbct.i32\t%q0, %q2, %q3"
9927   [(set_attr "type" "mve_move")
9928    (set_attr "length" "8")])
9929
9930 ;;
9931 ;; [vsbcq_s, vsbcq_u])
9932 ;;
9933 (define_insn "mve_vsbcq_<supf>v4si"
9934   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9935         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9936                       (match_operand:V4SI 2 "s_register_operand" "w")]
9937          VSBCQ))
9938    (set (reg:SI VFPCC_REGNUM)
9939         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9940          VSBCQ))
9941   ]
9942   "TARGET_HAVE_MVE"
9943   "vsbc.i32\t%q0, %q1, %q2"
9944   [(set_attr "type" "mve_move")
9945    (set_attr "length" "4")])
9946
9947 ;;
9948 ;; [vst2q])
9949 ;;
9950 (define_insn "mve_vst2q<mode>"
9951   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
9952         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
9953                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9954          VST2Q))
9955   ]
9956   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9957    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9958 {
9959    rtx ops[4];
9960    int regno = REGNO (operands[1]);
9961    ops[0] = gen_rtx_REG (TImode, regno);
9962    ops[1] = gen_rtx_REG (TImode, regno + 4);
9963    rtx reg  = operands[0];
9964    while (reg && !REG_P (reg))
9965     reg = XEXP (reg, 0);
9966    gcc_assert (REG_P (reg));
9967    ops[2] = reg;
9968    ops[3] = operands[0];
9969    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
9970                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
9971    return "";
9972 }
9973   [(set_attr "length" "8")])
9974
9975 ;;
9976 ;; [vld2q])
9977 ;;
9978 (define_insn "mve_vld2q<mode>"
9979   [(set (match_operand:OI 0 "s_register_operand" "=w")
9980         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
9981                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9982          VLD2Q))
9983   ]
9984   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9985    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9986 {
9987    rtx ops[4];
9988    int regno = REGNO (operands[0]);
9989    ops[0] = gen_rtx_REG (TImode, regno);
9990    ops[1] = gen_rtx_REG (TImode, regno + 4);
9991    rtx reg  = operands[1];
9992    while (reg && !REG_P (reg))
9993     reg = XEXP (reg, 0);
9994    gcc_assert (REG_P (reg));
9995    ops[2] = reg;
9996    ops[3] = operands[1];
9997    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
9998                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
9999    return "";
10000 }
10001   [(set_attr "length" "8")])
10002
10003 ;;
10004 ;; [vld4q])
10005 ;;
10006 (define_insn "mve_vld4q<mode>"
10007   [(set (match_operand:XI 0 "s_register_operand" "=w")
10008         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10009                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10010          VLD4Q))
10011   ]
10012   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10013    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10014 {
10015    rtx ops[6];
10016    int regno = REGNO (operands[0]);
10017    ops[0] = gen_rtx_REG (TImode, regno);
10018    ops[1] = gen_rtx_REG (TImode, regno+4);
10019    ops[2] = gen_rtx_REG (TImode, regno+8);
10020    ops[3] = gen_rtx_REG (TImode, regno + 12);
10021    rtx reg  = operands[1];
10022    while (reg && !REG_P (reg))
10023     reg = XEXP (reg, 0);
10024    gcc_assert (REG_P (reg));
10025    ops[4] = reg;
10026    ops[5] = operands[1];
10027    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10028                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10029                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10030                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10031    return "";
10032 }
10033   [(set_attr "length" "16")])
10034 ;;
10035 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10036 ;;
10037 (define_insn "mve_vec_extract<mode><V_elem_l>"
10038  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10039    (vec_select:<V_elem>
10040     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10041     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10042   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10043    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10044 {
10045   if (BYTES_BIG_ENDIAN)
10046     {
10047       int elt = INTVAL (operands[2]);
10048       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10049       operands[2] = GEN_INT (elt);
10050     }
10051   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10052 }
10053  [(set_attr "type" "mve_move")])
10054
10055 (define_insn "mve_vec_extractv2didi"
10056  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10057    (vec_select:DI
10058     (match_operand:V2DI 1 "s_register_operand" "w")
10059     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10060   "TARGET_HAVE_MVE"
10061 {
10062   int elt = INTVAL (operands[2]);
10063   if (BYTES_BIG_ENDIAN)
10064     elt = 1 - elt;
10065
10066   if (elt == 0)
10067    return "vmov\t%Q0, %R0, %e1";
10068   else
10069    return "vmov\t%Q0, %R0, %f1";
10070 }
10071  [(set_attr "type" "mve_move")])
10072
10073 (define_insn "*mve_vec_extract_sext_internal<mode>"
10074  [(set (match_operand:SI 0 "s_register_operand" "=r")
10075    (sign_extend:SI
10076     (vec_select:<V_elem>
10077      (match_operand:MVE_2 1 "s_register_operand" "w")
10078      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10079   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10080    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10081 {
10082   if (BYTES_BIG_ENDIAN)
10083     {
10084       int elt = INTVAL (operands[2]);
10085       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10086       operands[2] = GEN_INT (elt);
10087     }
10088   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10089 }
10090  [(set_attr "type" "mve_move")])
10091
10092 (define_insn "*mve_vec_extract_zext_internal<mode>"
10093  [(set (match_operand:SI 0 "s_register_operand" "=r")
10094    (zero_extend:SI
10095     (vec_select:<V_elem>
10096      (match_operand:MVE_2 1 "s_register_operand" "w")
10097      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10098   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10099    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10100 {
10101   if (BYTES_BIG_ENDIAN)
10102     {
10103       int elt = INTVAL (operands[2]);
10104       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10105       operands[2] = GEN_INT (elt);
10106     }
10107   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10108 }
10109  [(set_attr "type" "mve_move")])
10110
10111 ;;
10112 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10113 ;;
10114 (define_insn "mve_vec_set<mode>_internal"
10115  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10116        (vec_merge:VQ2
10117         (vec_duplicate:VQ2
10118           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10119         (match_operand:VQ2 3 "s_register_operand" "0")
10120         (match_operand:SI 2 "immediate_operand" "i")))]
10121   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10122    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10123 {
10124   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10125   if (BYTES_BIG_ENDIAN)
10126     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10127   operands[2] = GEN_INT (elt);
10128
10129   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10130 }
10131  [(set_attr "type" "mve_move")])
10132
10133 (define_insn "mve_vec_setv2di_internal"
10134  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10135        (vec_merge:V2DI
10136         (vec_duplicate:V2DI
10137           (match_operand:DI 1 "nonimmediate_operand" "r"))
10138         (match_operand:V2DI 3 "s_register_operand" "0")
10139         (match_operand:SI 2 "immediate_operand" "i")))]
10140  "TARGET_HAVE_MVE"
10141 {
10142   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10143   if (BYTES_BIG_ENDIAN)
10144     elt = 1 - elt;
10145
10146   if (elt == 0)
10147    return "vmov\t%e0, %Q1, %R1";
10148   else
10149    return "vmov\t%f0, %J1, %K1";
10150 }
10151  [(set_attr "type" "mve_move")])
10152
10153 ;;
10154 ;; [uqrshll_di]
10155 ;;
10156 (define_insn "mve_uqrshll_sat<supf>_di"
10157   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10158         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10159                     (match_operand:SI 2 "register_operand" "r")]
10160          UQRSHLLQ))]
10161   "TARGET_HAVE_MVE"
10162   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10163   [(set_attr "predicable" "yes")])
10164
10165 ;;
10166 ;; [sqrshrl_di]
10167 ;;
10168 (define_insn "mve_sqrshrl_sat<supf>_di"
10169   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10170         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10171                     (match_operand:SI 2 "register_operand" "r")]
10172          SQRSHRLQ))]
10173   "TARGET_HAVE_MVE"
10174   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10175   [(set_attr "predicable" "yes")])
10176
10177 ;;
10178 ;; [uqrshl_si]
10179 ;;
10180 (define_insn "mve_uqrshl_si"
10181   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10182         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10183                     (match_operand:SI 2 "register_operand" "r")]
10184          UQRSHL))]
10185   "TARGET_HAVE_MVE"
10186   "uqrshl%?\\t%1, %2"
10187   [(set_attr "predicable" "yes")])
10188
10189 ;;
10190 ;; [sqrshr_si]
10191 ;;
10192 (define_insn "mve_sqrshr_si"
10193   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10194         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10195                     (match_operand:SI 2 "register_operand" "r")]
10196          SQRSHR))]
10197   "TARGET_HAVE_MVE"
10198   "sqrshr%?\\t%1, %2"
10199   [(set_attr "predicable" "yes")])
10200
10201 ;;
10202 ;; [uqshll_di]
10203 ;;
10204 (define_insn "mve_uqshll_di"
10205   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10206         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10207                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10208   "TARGET_HAVE_MVE"
10209   "uqshll%?\\t%Q1, %R1, %2"
10210   [(set_attr "predicable" "yes")])
10211
10212 ;;
10213 ;; [urshrl_di]
10214 ;;
10215 (define_insn "mve_urshrl_di"
10216   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10217         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10218                     (match_operand:SI 2 "immediate_operand" "Pg")]
10219          URSHRL))]
10220   "TARGET_HAVE_MVE"
10221   "urshrl%?\\t%Q1, %R1, %2"
10222   [(set_attr "predicable" "yes")])
10223
10224 ;;
10225 ;; [uqshl_si]
10226 ;;
10227 (define_insn "mve_uqshl_si"
10228   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10229         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10230                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10231   "TARGET_HAVE_MVE"
10232   "uqshl%?\\t%1, %2"
10233   [(set_attr "predicable" "yes")])
10234
10235 ;;
10236 ;; [urshr_si]
10237 ;;
10238 (define_insn "mve_urshr_si"
10239   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10240         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10241                     (match_operand:SI 2 "immediate_operand" "Pg")]
10242          URSHR))]
10243   "TARGET_HAVE_MVE"
10244   "urshr%?\\t%1, %2"
10245   [(set_attr "predicable" "yes")])
10246
10247 ;;
10248 ;; [sqshl_si]
10249 ;;
10250 (define_insn "mve_sqshl_si"
10251   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10252         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10253                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10254   "TARGET_HAVE_MVE"
10255   "sqshl%?\\t%1, %2"
10256   [(set_attr "predicable" "yes")])
10257
10258 ;;
10259 ;; [srshr_si]
10260 ;;
10261 (define_insn "mve_srshr_si"
10262   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10263         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10264                     (match_operand:SI 2 "immediate_operand" "Pg")]
10265          SRSHR))]
10266   "TARGET_HAVE_MVE"
10267   "srshr%?\\t%1, %2"
10268   [(set_attr "predicable" "yes")])
10269
10270 ;;
10271 ;; [srshrl_di]
10272 ;;
10273 (define_insn "mve_srshrl_di"
10274   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10275         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10276                     (match_operand:SI 2 "immediate_operand" "Pg")]
10277          SRSHRL))]
10278   "TARGET_HAVE_MVE"
10279   "srshrl%?\\t%Q1, %R1, %2"
10280   [(set_attr "predicable" "yes")])
10281
10282 ;;
10283 ;; [sqshll_di]
10284 ;;
10285 (define_insn "mve_sqshll_di"
10286   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10287         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10288                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10289   "TARGET_HAVE_MVE"
10290   "sqshll%?\\t%Q1, %R1, %2"
10291   [(set_attr "predicable" "yes")])
10292
10293 ;;
10294 ;; [vshlcq_m_u vshlcq_m_s]
10295 ;;
10296 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10297  [(match_operand:MVE_2 0 "s_register_operand")
10298   (match_operand:MVE_2 1 "s_register_operand")
10299   (match_operand:SI 2 "s_register_operand")
10300   (match_operand:SI 3 "mve_imm_32")
10301   (match_operand:HI 4 "vpr_register_operand")
10302   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10303  "TARGET_HAVE_MVE"
10304 {
10305   rtx ignore_wb = gen_reg_rtx (SImode);
10306   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10307                                             operands[2], operands[3],
10308                                             operands[4]));
10309   DONE;
10310 })
10311
10312 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10313  [(match_operand:SI 0 "s_register_operand")
10314   (match_operand:MVE_2 1 "s_register_operand")
10315   (match_operand:SI 2 "s_register_operand")
10316   (match_operand:SI 3 "mve_imm_32")
10317   (match_operand:HI 4 "vpr_register_operand")
10318   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10319  "TARGET_HAVE_MVE"
10320 {
10321   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10322   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10323                                             operands[1], operands[2],
10324                                             operands[3], operands[4]));
10325   DONE;
10326 })
10327
10328 (define_insn "mve_vshlcq_m_<supf><mode>"
10329  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10330        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10331                       (match_operand:SI 3 "s_register_operand" "1")
10332                       (match_operand:SI 4 "mve_imm_32" "Rf")
10333                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10334         VSHLCQ_M))
10335   (set (match_operand:SI  1 "s_register_operand" "=r")
10336        (unspec:SI [(match_dup 2)
10337                    (match_dup 3)
10338                    (match_dup 4)
10339                    (match_dup 5)]
10340         VSHLCQ_M))
10341  ]
10342  "TARGET_HAVE_MVE"
10343  "vpst\;vshlct\t%q0, %1, %4"
10344  [(set_attr "type" "mve_move")
10345   (set_attr "length" "8")])
10346
10347 ;; CDE instructions on MVE registers.
10348
10349 (define_insn "arm_vcx1qv16qi"
10350   [(set (match_operand:V16QI 0 "register_operand" "=t")
10351         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10352                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10353          UNSPEC_VCDE))]
10354   "TARGET_CDE && TARGET_HAVE_MVE"
10355   "vcx1\\tp%c1, %q0, #%c2"
10356   [(set_attr "type" "coproc")]
10357 )
10358
10359 (define_insn "arm_vcx1qav16qi"
10360   [(set (match_operand:V16QI 0 "register_operand" "=t")
10361         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10362                             (match_operand:V16QI 2 "register_operand" "0")
10363                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10364          UNSPEC_VCDEA))]
10365   "TARGET_CDE && TARGET_HAVE_MVE"
10366   "vcx1a\\tp%c1, %q0, #%c3"
10367   [(set_attr "type" "coproc")]
10368 )
10369
10370 (define_insn "arm_vcx2qv16qi"
10371   [(set (match_operand:V16QI 0 "register_operand" "=t")
10372         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10373                           (match_operand:V16QI 2 "register_operand" "t")
10374                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10375          UNSPEC_VCDE))]
10376   "TARGET_CDE && TARGET_HAVE_MVE"
10377   "vcx2\\tp%c1, %q0, %q2, #%c3"
10378   [(set_attr "type" "coproc")]
10379 )
10380
10381 (define_insn "arm_vcx2qav16qi"
10382   [(set (match_operand:V16QI 0 "register_operand" "=t")
10383         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10384                           (match_operand:V16QI 2 "register_operand" "0")
10385                           (match_operand:V16QI 3 "register_operand" "t")
10386                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10387          UNSPEC_VCDEA))]
10388   "TARGET_CDE && TARGET_HAVE_MVE"
10389   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10390   [(set_attr "type" "coproc")]
10391 )
10392
10393 (define_insn "arm_vcx3qv16qi"
10394   [(set (match_operand:V16QI 0 "register_operand" "=t")
10395         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10396                           (match_operand:V16QI 2 "register_operand" "t")
10397                           (match_operand:V16QI 3 "register_operand" "t")
10398                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10399          UNSPEC_VCDE))]
10400   "TARGET_CDE && TARGET_HAVE_MVE"
10401   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10402   [(set_attr "type" "coproc")]
10403 )
10404
10405 (define_insn "arm_vcx3qav16qi"
10406   [(set (match_operand:V16QI 0 "register_operand" "=t")
10407         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10408                           (match_operand:V16QI 2 "register_operand" "0")
10409                           (match_operand:V16QI 3 "register_operand" "t")
10410                           (match_operand:V16QI 4 "register_operand" "t")
10411                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10412          UNSPEC_VCDEA))]
10413   "TARGET_CDE && TARGET_HAVE_MVE"
10414   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10415   [(set_attr "type" "coproc")]
10416 )
10417
10418 (define_insn "arm_vcx1q<a>_p_v16qi"
10419   [(set (match_operand:V16QI 0 "register_operand" "=t")
10420         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10421                            (match_operand:V16QI 2 "register_operand" "0")
10422                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10423                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10424          CDE_VCX))]
10425   "TARGET_CDE && TARGET_HAVE_MVE"
10426   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10427   [(set_attr "type" "coproc")
10428    (set_attr "length" "8")]
10429 )
10430
10431 (define_insn "arm_vcx2q<a>_p_v16qi"
10432   [(set (match_operand:V16QI 0 "register_operand" "=t")
10433         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10434                           (match_operand:V16QI 2 "register_operand" "0")
10435                           (match_operand:V16QI 3 "register_operand" "t")
10436                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10437                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10438          CDE_VCX))]
10439   "TARGET_CDE && TARGET_HAVE_MVE"
10440   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10441   [(set_attr "type" "coproc")
10442    (set_attr "length" "8")]
10443 )
10444
10445 (define_insn "arm_vcx3q<a>_p_v16qi"
10446   [(set (match_operand:V16QI 0 "register_operand" "=t")
10447         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10448                           (match_operand:V16QI 2 "register_operand" "0")
10449                           (match_operand:V16QI 3 "register_operand" "t")
10450                           (match_operand:V16QI 4 "register_operand" "t")
10451                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10452                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10453          CDE_VCX))]
10454   "TARGET_CDE && TARGET_HAVE_MVE"
10455   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10456   [(set_attr "type" "coproc")
10457    (set_attr "length" "8")]
10458 )
10459
10460 (define_insn "*movmisalign<mode>_mve_store"
10461   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Ux")
10462         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10463          UNSPEC_MISALIGNED_ACCESS))]
10464   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10465     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10466    && !BYTES_BIG_ENDIAN && unaligned_access"
10467   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10468   [(set_attr "type" "mve_store")]
10469 )
10470
10471
10472 (define_insn "*movmisalign<mode>_mve_load"
10473   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10474         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Ux")]
10475          UNSPEC_MISALIGNED_ACCESS))]
10476   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10477     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10478    && !BYTES_BIG_ENDIAN && unaligned_access"
10479   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10480   [(set_attr "type" "mve_load")]
10481 )