gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2021 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,UxUi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode || <MODE>mode == TImode)
  45         {
  46           if (which_alternative == 7)
  47             output_asm_insn ("vstrw.32\t%q1, %E0", operands);
  48           else
  49             output_asm_insn ("vldrw.u32\t%q0, %E1",operands);
  50         }
  51       else
  52         {
  53           if (which_alternative == 7)
  54             output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0", operands);
  55           else
  56             output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1", operands);
  57         }
  58       return "";
  59     }
  60   switch (which_alternative)
  61     {
  62     case 0:
  63       return "vmov\t%q0, %q1";
  64     case 1:
  65       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  66     case 2:
  67       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  68     case 5:
  69       return output_move_quad (operands);
  70     case 8:
  71         return output_move_neon (operands);
  72     default:
  73       gcc_unreachable ();
  74       return "";
  75     }
  76 }
  77   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
  78    (set_attr "length" "4,8,8,4,8,8,4,4,4")
  79    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
  80    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
  81
  82 (define_insn "*mve_vdup<mode>"
  83   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  84         (vec_duplicate:MVE_vecs
  85           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  86   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  87   "vdup.<V_sz_elem>\t%q0, %1"
  88   [(set_attr "length" "4")
  89    (set_attr "type" "mve_move")])
  90
  91 ;;
  92 ;; [vst4q])
  93 ;;
  94 (define_insn "mve_vst4q<mode>"
  95   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
  96         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
  97                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
  98          VST4Q))
  99   ]
 100   "TARGET_HAVE_MVE"
 101 {
 102    rtx ops[6];
 103    int regno = REGNO (operands[1]);
 104    ops[0] = gen_rtx_REG (TImode, regno);
 105    ops[1] = gen_rtx_REG (TImode, regno+4);
 106    ops[2] = gen_rtx_REG (TImode, regno+8);
 107    ops[3] = gen_rtx_REG (TImode, regno+12);
 108    rtx reg  = operands[0];
 109    while (reg && !REG_P (reg))
 110     reg = XEXP (reg, 0);
 111    gcc_assert (REG_P (reg));
 112    ops[4] = reg;
 113    ops[5] = operands[0];
 114    /* Here in first three instructions data is stored to ops[4]'s location but
 115       in the fourth instruction data is stored to operands[0], this is to
 116       support the writeback.  */
 117    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 118                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 119                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 120                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 121    return "";
 122 }
 123   [(set_attr "length" "16")])
 124
 125 ;;
 126 ;; [vrndq_m_f])
 127 ;;
 128 (define_insn "mve_vrndq_m_f<mode>"
 129   [
 130    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 131         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 132                        (match_operand:MVE_0 2 "s_register_operand" "w")
 133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 134          VRNDQ_M_F))
 135   ]
 136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 137   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 138   [(set_attr "type" "mve_move")
 139    (set_attr "length""8")])
 140
 141 ;;
 142 ;; [vrndxq_f])
 143 ;;
 144 (define_insn "mve_vrndxq_f<mode>"
 145   [
 146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 148          VRNDXQ_F))
 149   ]
 150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 151   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 152   [(set_attr "type" "mve_move")
 153 ])
 154
 155 ;;
 156 ;; [vrndq_f])
 157 ;;
 158 (define_insn "mve_vrndq_f<mode>"
 159   [
 160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 162          VRNDQ_F))
 163   ]
 164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 165   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 166   [(set_attr "type" "mve_move")
 167 ])
 168
 169 ;;
 170 ;; [vrndpq_f])
 171 ;;
 172 (define_insn "mve_vrndpq_f<mode>"
 173   [
 174    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 175         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 176          VRNDPQ_F))
 177   ]
 178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 179   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 180   [(set_attr "type" "mve_move")
 181 ])
 182
 183 ;;
 184 ;; [vrndnq_f])
 185 ;;
 186 (define_insn "mve_vrndnq_f<mode>"
 187   [
 188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 190          VRNDNQ_F))
 191   ]
 192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 193   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 194   [(set_attr "type" "mve_move")
 195 ])
 196
 197 ;;
 198 ;; [vrndmq_f])
 199 ;;
 200 (define_insn "mve_vrndmq_f<mode>"
 201   [
 202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 204          VRNDMQ_F))
 205   ]
 206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 207   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 208   [(set_attr "type" "mve_move")
 209 ])
 210
 211 ;;
 212 ;; [vrndaq_f])
 213 ;;
 214 (define_insn "mve_vrndaq_f<mode>"
 215   [
 216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 217         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 218          VRNDAQ_F))
 219   ]
 220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 221   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 222   [(set_attr "type" "mve_move")
 223 ])
 224
 225 ;;
 226 ;; [vrev64q_f])
 227 ;;
 228 (define_insn "mve_vrev64q_f<mode>"
 229   [
 230    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 232          VREV64Q_F))
 233   ]
 234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 235   "vrev64.%#<V_sz_elem> %q0, %q1"
 236   [(set_attr "type" "mve_move")
 237 ])
 238
 239 ;;
 240 ;; [vnegq_f])
 241 ;;
 242 (define_insn "mve_vnegq_f<mode>"
 243   [
 244    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 245         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 246   ]
 247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 248   "vneg.f%#<V_sz_elem>  %q0, %q1"
 249   [(set_attr "type" "mve_move")
 250 ])
 251
 252 ;;
 253 ;; [vdupq_n_f])
 254 ;;
 255 (define_insn "mve_vdupq_n_f<mode>"
 256   [
 257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 258         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 259          VDUPQ_N_F))
 260   ]
 261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 262   "vdup.%#<V_sz_elem>   %q0, %1"
 263   [(set_attr "type" "mve_move")
 264 ])
 265
 266 ;;
 267 ;; [vabsq_f])
 268 ;;
 269 (define_insn "mve_vabsq_f<mode>"
 270   [
 271    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 272         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 273          VABSQ_F))
 274   ]
 275   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 276   "vabs.f%#<V_sz_elem>  %q0, %q1"
 277   [(set_attr "type" "mve_move")
 278 ])
 279
 280 ;;
 281 ;; [vrev32q_f])
 282 ;;
 283 (define_insn "mve_vrev32q_fv8hf"
 284   [
 285    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 286         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 287          VREV32Q_F))
 288   ]
 289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 290   "vrev32.16 %q0, %q1"
 291   [(set_attr "type" "mve_move")
 292 ])
 293 ;;
 294 ;; [vcvttq_f32_f16])
 295 ;;
 296 (define_insn "mve_vcvttq_f32_f16v4sf"
 297   [
 298    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 299         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 300          VCVTTQ_F32_F16))
 301   ]
 302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 303   "vcvtt.f32.f16 %q0, %q1"
 304   [(set_attr "type" "mve_move")
 305 ])
 306
 307 ;;
 308 ;; [vcvtbq_f32_f16])
 309 ;;
 310 (define_insn "mve_vcvtbq_f32_f16v4sf"
 311   [
 312    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 313         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 314          VCVTBQ_F32_F16))
 315   ]
 316   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 317   "vcvtb.f32.f16 %q0, %q1"
 318   [(set_attr "type" "mve_move")
 319 ])
 320
 321 ;;
 322 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 323 ;;
 324 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 325   [
 326    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 327         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 328          VCVTQ_TO_F))
 329   ]
 330   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 331   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 332   [(set_attr "type" "mve_move")
 333 ])
 334
 335 ;;
 336 ;; [vrev64q_u, vrev64q_s])
 337 ;;
 338 (define_insn "mve_vrev64q_<supf><mode>"
 339   [
 340    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 341         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 342          VREV64Q))
 343   ]
 344   "TARGET_HAVE_MVE"
 345   "vrev64.%#<V_sz_elem> %q0, %q1"
 346   [(set_attr "type" "mve_move")
 347 ])
 348
 349 ;;
 350 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 351 ;;
 352 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 353   [
 354    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 355         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 356          VCVTQ_FROM_F))
 357   ]
 358   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 359   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 360   [(set_attr "type" "mve_move")
 361 ])
 362 ;; [vqnegq_s])
 363 ;;
 364 (define_insn "mve_vqnegq_s<mode>"
 365   [
 366    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 367         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 368          VQNEGQ_S))
 369   ]
 370   "TARGET_HAVE_MVE"
 371   "vqneg.s%#<V_sz_elem> %q0, %q1"
 372   [(set_attr "type" "mve_move")
 373 ])
 374
 375 ;;
 376 ;; [vqabsq_s])
 377 ;;
 378 (define_insn "mve_vqabsq_s<mode>"
 379   [
 380    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 381         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 382          VQABSQ_S))
 383   ]
 384   "TARGET_HAVE_MVE"
 385   "vqabs.s%#<V_sz_elem> %q0, %q1"
 386   [(set_attr "type" "mve_move")
 387 ])
 388
 389 ;;
 390 ;; [vnegq_s])
 391 ;;
 392 (define_insn "mve_vnegq_s<mode>"
 393   [
 394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 395         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 396   ]
 397   "TARGET_HAVE_MVE"
 398   "vneg.s%#<V_sz_elem>  %q0, %q1"
 399   [(set_attr "type" "mve_move")
 400 ])
 401
 402 ;;
 403 ;; [vmvnq_u, vmvnq_s])
 404 ;;
 405 (define_insn "mve_vmvnq_u<mode>"
 406   [
 407    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 408         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 409   ]
 410   "TARGET_HAVE_MVE"
 411   "vmvn\t%q0, %q1"
 412   [(set_attr "type" "mve_move")
 413 ])
 414 (define_expand "mve_vmvnq_s<mode>"
 415   [
 416    (set (match_operand:MVE_2 0 "s_register_operand")
 417         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 418   ]
 419   "TARGET_HAVE_MVE"
 420 )
 421
 422 ;;
 423 ;; [vdupq_n_u, vdupq_n_s])
 424 ;;
 425 (define_insn "mve_vdupq_n_<supf><mode>"
 426   [
 427    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 428         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 429          VDUPQ_N))
 430   ]
 431   "TARGET_HAVE_MVE"
 432   "vdup.%#<V_sz_elem>   %q0, %1"
 433   [(set_attr "type" "mve_move")
 434 ])
 435
 436 ;;
 437 ;; [vclzq_u, vclzq_s])
 438 ;;
 439 (define_insn "mve_vclzq_<supf><mode>"
 440   [
 441    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 442         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 443          VCLZQ))
 444   ]
 445   "TARGET_HAVE_MVE"
 446   "vclz.i%#<V_sz_elem>  %q0, %q1"
 447   [(set_attr "type" "mve_move")
 448 ])
 449
 450 ;;
 451 ;; [vclsq_s])
 452 ;;
 453 (define_insn "mve_vclsq_s<mode>"
 454   [
 455    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 456         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 457          VCLSQ_S))
 458   ]
 459   "TARGET_HAVE_MVE"
 460   "vcls.s%#<V_sz_elem>  %q0, %q1"
 461   [(set_attr "type" "mve_move")
 462 ])
 463
 464 ;;
 465 ;; [vaddvq_s, vaddvq_u])
 466 ;;
 467 (define_insn "mve_vaddvq_<supf><mode>"
 468   [
 469    (set (match_operand:SI 0 "s_register_operand" "=Te")
 470         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 471          VADDVQ))
 472   ]
 473   "TARGET_HAVE_MVE"
 474   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 475   [(set_attr "type" "mve_move")
 476 ])
 477
 478 ;;
 479 ;; [vabsq_s])
 480 ;;
 481 (define_insn "mve_vabsq_s<mode>"
 482   [
 483    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 484         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 485          VABSQ_S))
 486   ]
 487   "TARGET_HAVE_MVE"
 488   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 489   [(set_attr "type" "mve_move")
 490 ])
 491
 492 ;;
 493 ;; [vrev32q_u, vrev32q_s])
 494 ;;
 495 (define_insn "mve_vrev32q_<supf><mode>"
 496   [
 497    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 498         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 499          VREV32Q))
 500   ]
 501   "TARGET_HAVE_MVE"
 502   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 503   [(set_attr "type" "mve_move")
 504 ])
 505
 506 ;;
 507 ;; [vmovltq_u, vmovltq_s])
 508 ;;
 509 (define_insn "mve_vmovltq_<supf><mode>"
 510   [
 511    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 512         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 513          VMOVLTQ))
 514   ]
 515   "TARGET_HAVE_MVE"
 516   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 517   [(set_attr "type" "mve_move")
 518 ])
 519
 520 ;;
 521 ;; [vmovlbq_s, vmovlbq_u])
 522 ;;
 523 (define_insn "mve_vmovlbq_<supf><mode>"
 524   [
 525    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 526         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 527          VMOVLBQ))
 528   ]
 529   "TARGET_HAVE_MVE"
 530   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 531   [(set_attr "type" "mve_move")
 532 ])
 533
 534 ;;
 535 ;; [vcvtpq_s, vcvtpq_u])
 536 ;;
 537 (define_insn "mve_vcvtpq_<supf><mode>"
 538   [
 539    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 540         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 541          VCVTPQ))
 542   ]
 543   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 544   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 545   [(set_attr "type" "mve_move")
 546 ])
 547
 548 ;;
 549 ;; [vcvtnq_s, vcvtnq_u])
 550 ;;
 551 (define_insn "mve_vcvtnq_<supf><mode>"
 552   [
 553    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 554         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 555          VCVTNQ))
 556   ]
 557   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 558   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 559   [(set_attr "type" "mve_move")
 560 ])
 561
 562 ;;
 563 ;; [vcvtmq_s, vcvtmq_u])
 564 ;;
 565 (define_insn "mve_vcvtmq_<supf><mode>"
 566   [
 567    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 568         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 569          VCVTMQ))
 570   ]
 571   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 572   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 573   [(set_attr "type" "mve_move")
 574 ])
 575
 576 ;;
 577 ;; [vcvtaq_u, vcvtaq_s])
 578 ;;
 579 (define_insn "mve_vcvtaq_<supf><mode>"
 580   [
 581    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 582         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 583          VCVTAQ))
 584   ]
 585   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 586   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 587   [(set_attr "type" "mve_move")
 588 ])
 589
 590 ;;
 591 ;; [vmvnq_n_u, vmvnq_n_s])
 592 ;;
 593 (define_insn "mve_vmvnq_n_<supf><mode>"
 594   [
 595    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 596         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 597          VMVNQ_N))
 598   ]
 599   "TARGET_HAVE_MVE"
 600   "vmvn.i%#<V_sz_elem>  %q0, %1"
 601   [(set_attr "type" "mve_move")
 602 ])
 603
 604 ;;
 605 ;; [vrev16q_u, vrev16q_s])
 606 ;;
 607 (define_insn "mve_vrev16q_<supf>v16qi"
 608   [
 609    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 610         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 611          VREV16Q))
 612   ]
 613   "TARGET_HAVE_MVE"
 614   "vrev16.8 %q0, %q1"
 615   [(set_attr "type" "mve_move")
 616 ])
 617
 618 ;;
 619 ;; [vaddlvq_s vaddlvq_u])
 620 ;;
 621 (define_insn "mve_vaddlvq_<supf>v4si"
 622   [
 623    (set (match_operand:DI 0 "s_register_operand" "=r")
 624         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 625          VADDLVQ))
 626   ]
 627   "TARGET_HAVE_MVE"
 628   "vaddlv.<supf>32 %Q0, %R0, %q1"
 629   [(set_attr "type" "mve_move")
 630 ])
 631
 632 ;;
 633 ;; [vctp8q vctp16q vctp32q vctp64q])
 634 ;;
 635 (define_insn "mve_vctp<mode1>qhi"
 636   [
 637    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 638         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 639         VCTPQ))
 640   ]
 641   "TARGET_HAVE_MVE"
 642   "vctp.<mode1> %1"
 643   [(set_attr "type" "mve_move")
 644 ])
 645
 646 ;;
 647 ;; [vpnot])
 648 ;;
 649 (define_insn "mve_vpnothi"
 650   [
 651    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 652         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 653          VPNOT))
 654   ]
 655   "TARGET_HAVE_MVE"
 656   "vpnot"
 657   [(set_attr "type" "mve_move")
 658 ])
 659
 660 ;;
 661 ;; [vsubq_n_f])
 662 ;;
 663 (define_insn "mve_vsubq_n_f<mode>"
 664   [
 665    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 666         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 667                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 668          VSUBQ_N_F))
 669   ]
 670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 671   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 672   [(set_attr "type" "mve_move")
 673 ])
 674
 675 ;;
 676 ;; [vbrsrq_n_f])
 677 ;;
 678 (define_insn "mve_vbrsrq_n_f<mode>"
 679   [
 680    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 681         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 682                        (match_operand:SI 2 "s_register_operand" "r")]
 683          VBRSRQ_N_F))
 684   ]
 685   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 686   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 687   [(set_attr "type" "mve_move")
 688 ])
 689
 690 ;;
 691 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 692 ;;
 693 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 694   [
 695    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 696         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 697                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 698          VCVTQ_N_TO_F))
 699   ]
 700   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 701   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 702   [(set_attr "type" "mve_move")
 703 ])
 704
 705 ;; [vcreateq_f])
 706 ;;
 707 (define_insn "mve_vcreateq_f<mode>"
 708   [
 709    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 710         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 711                        (match_operand:DI 2 "s_register_operand" "r")]
 712          VCREATEQ_F))
 713   ]
 714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 715   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 716   [(set_attr "type" "mve_move")
 717    (set_attr "length""8")])
 718
 719 ;;
 720 ;; [vcreateq_u, vcreateq_s])
 721 ;;
 722 (define_insn "mve_vcreateq_<supf><mode>"
 723   [
 724    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 725         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 726                        (match_operand:DI 2 "s_register_operand" "r")]
 727          VCREATEQ))
 728   ]
 729   "TARGET_HAVE_MVE"
 730   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 731   [(set_attr "type" "mve_move")
 732    (set_attr "length""8")])
 733
 734 ;;
 735 ;; [vshrq_n_s, vshrq_n_u])
 736 ;;
 737 ;; Version that takes an immediate as operand 2.
 738 (define_insn "mve_vshrq_n_<supf><mode>"
 739   [
 740    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 741         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 742                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 743          VSHRQ_N))
 744   ]
 745   "TARGET_HAVE_MVE"
 746   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 747   [(set_attr "type" "mve_move")
 748 ])
 749
 750 ;; Versions that take constant vectors as operand 2 (with all elements
 751 ;; equal).
 752 (define_insn "mve_vshrq_n_s<mode>_imm"
 753   [
 754    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 755         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 756                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 757   ]
 758   "TARGET_HAVE_MVE"
 759   {
 760     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 761                                         <MODE>mode,
 762                                         VALID_NEON_QREG_MODE (<MODE>mode),
 763                                         true);
 764   }
 765   [(set_attr "type" "mve_move")
 766 ])
 767 (define_insn "mve_vshrq_n_u<mode>_imm"
 768   [
 769    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 770         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 771                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 772   ]
 773   "TARGET_HAVE_MVE"
 774   {
 775     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 776                                         <MODE>mode,
 777                                         VALID_NEON_QREG_MODE (<MODE>mode),
 778                                         true);
 779   }
 780   [(set_attr "type" "mve_move")
 781 ])
 782
 783 ;;
 784 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 785 ;;
 786 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 787   [
 788    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 789         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 790                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 791          VCVTQ_N_FROM_F))
 792   ]
 793   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 794   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 795   [(set_attr "type" "mve_move")
 796 ])
 797
 798 ;;
 799 ;; [vaddlvq_p_s])
 800 ;;
 801 (define_insn "mve_vaddlvq_p_<supf>v4si"
 802   [
 803    (set (match_operand:DI 0 "s_register_operand" "=r")
 804         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 805                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 806          VADDLVQ_P))
 807   ]
 808   "TARGET_HAVE_MVE"
 809   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 810   [(set_attr "type" "mve_move")
 811    (set_attr "length""8")])
 812
 813 ;;
 814 ;; [vcmpneq_u, vcmpneq_s])
 815 ;;
 816 (define_insn "mve_vcmpneq_<supf><mode>"
 817   [
 818    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 819         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 820                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 821          VCMPNEQ))
 822   ]
 823   "TARGET_HAVE_MVE"
 824   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
 825   [(set_attr "type" "mve_move")
 826 ])
 827
 828 ;;
 829 ;; [vshlq_s, vshlq_u])
 830 ;; See vec-common.md
 831
 832 ;;
 833 ;; [vabdq_s, vabdq_u])
 834 ;;
 835 (define_insn "mve_vabdq_<supf><mode>"
 836   [
 837    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 838         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 839                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 840          VABDQ))
 841   ]
 842   "TARGET_HAVE_MVE"
 843   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 844   [(set_attr "type" "mve_move")
 845 ])
 846
 847 ;;
 848 ;; [vaddq_n_s, vaddq_n_u])
 849 ;;
 850 (define_insn "mve_vaddq_n_<supf><mode>"
 851   [
 852    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 853         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 854                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 855          VADDQ_N))
 856   ]
 857   "TARGET_HAVE_MVE"
 858   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 859   [(set_attr "type" "mve_move")
 860 ])
 861
 862 ;;
 863 ;; [vaddvaq_s, vaddvaq_u])
 864 ;;
 865 (define_insn "mve_vaddvaq_<supf><mode>"
 866   [
 867    (set (match_operand:SI 0 "s_register_operand" "=Te")
 868         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 869                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 870          VADDVAQ))
 871   ]
 872   "TARGET_HAVE_MVE"
 873   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 874   [(set_attr "type" "mve_move")
 875 ])
 876
 877 ;;
 878 ;; [vaddvq_p_u, vaddvq_p_s])
 879 ;;
 880 (define_insn "mve_vaddvq_p_<supf><mode>"
 881   [
 882    (set (match_operand:SI 0 "s_register_operand" "=Te")
 883         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 884                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 885          VADDVQ_P))
 886   ]
 887   "TARGET_HAVE_MVE"
 888   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 889   [(set_attr "type" "mve_move")
 890    (set_attr "length""8")])
 891
 892 ;;
 893 ;; [vandq_u, vandq_s])
 894 ;;
 895 ;; signed and unsigned versions are the same: define the unsigned
 896 ;; insn, and use an expander for the signed one as we still reference
 897 ;; both names from arm_mve.h.
 898 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 899 (define_insn "mve_vandq_u<mode>"
 900   [
 901    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 902         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 903                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 904   ]
 905   "TARGET_HAVE_MVE"
 906   "@
 907    vand\t%q0, %q1, %q2
 908    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 909   [(set_attr "type" "mve_move")
 910 ])
 911 (define_expand "mve_vandq_s<mode>"
 912   [
 913    (set (match_operand:MVE_2 0 "s_register_operand")
 914         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 915                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 916   ]
 917   "TARGET_HAVE_MVE"
 918 )
 919
 920 ;;
 921 ;; [vbicq_s, vbicq_u])
 922 ;;
 923 (define_insn "mve_vbicq_u<mode>"
 924   [
 925    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 926         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 927                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 928   ]
 929   "TARGET_HAVE_MVE"
 930   "vbic\t%q0, %q1, %q2"
 931   [(set_attr "type" "mve_move")
 932 ])
 933
 934 (define_expand "mve_vbicq_s<mode>"
 935   [
 936    (set (match_operand:MVE_2 0 "s_register_operand")
 937         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 938                    (match_operand:MVE_2 1 "s_register_operand")))
 939   ]
 940   "TARGET_HAVE_MVE"
 941 )
 942
 943 ;;
 944 ;; [vbrsrq_n_u, vbrsrq_n_s])
 945 ;;
 946 (define_insn "mve_vbrsrq_n_<supf><mode>"
 947   [
 948    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 949         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 950                        (match_operand:SI 2 "s_register_operand" "r")]
 951          VBRSRQ_N))
 952   ]
 953   "TARGET_HAVE_MVE"
 954   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 955   [(set_attr "type" "mve_move")
 956 ])
 957
 958 ;;
 959 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 960 ;;
 961 (define_insn "mve_vcaddq<mve_rot><mode>"
 962   [
 963    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 964         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 965                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 966          VCADD))
 967   ]
 968   "TARGET_HAVE_MVE"
 969   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 970   [(set_attr "type" "mve_move")
 971 ])
 972
 973 ;; Auto vectorizer pattern for int vcadd
 974 (define_expand "cadd<rot><mode>3"
 975   [(set (match_operand:MVE_2 0 "register_operand")
 976         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
 977                        (match_operand:MVE_2 2 "register_operand")]
 978           VCADD))]
 979   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
 980 )
 981
 982 ;;
 983 ;; [vcmpcsq_n_u])
 984 ;;
 985 (define_insn "mve_vcmpcsq_n_u<mode>"
 986   [
 987    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 988         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 989                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
 990          VCMPCSQ_N_U))
 991   ]
 992   "TARGET_HAVE_MVE"
 993   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
 994   [(set_attr "type" "mve_move")
 995 ])
 996
 997 ;;
 998 ;; [vcmpcsq_u])
 999 ;;
1000 (define_insn "mve_vcmpcsq_u<mode>"
1001   [
1002    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1003         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1004                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1005          VCMPCSQ_U))
1006   ]
1007   "TARGET_HAVE_MVE"
1008   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1009   [(set_attr "type" "mve_move")
1010 ])
1011
1012 ;;
1013 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1014 ;;
1015 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1016   [
1017    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1018         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1019                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1020          VCMPEQQ_N))
1021   ]
1022   "TARGET_HAVE_MVE"
1023   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1024   [(set_attr "type" "mve_move")
1025 ])
1026
1027 ;;
1028 ;; [vcmpeqq_u, vcmpeqq_s])
1029 ;;
1030 (define_insn "mve_vcmpeqq_<supf><mode>"
1031   [
1032    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1033         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1034                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1035          VCMPEQQ))
1036   ]
1037   "TARGET_HAVE_MVE"
1038   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1039   [(set_attr "type" "mve_move")
1040 ])
1041
1042 ;;
1043 ;; [vcmpgeq_n_s])
1044 ;;
1045 (define_insn "mve_vcmpgeq_n_s<mode>"
1046   [
1047    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1048         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1049                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1050          VCMPGEQ_N_S))
1051   ]
1052   "TARGET_HAVE_MVE"
1053   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1054   [(set_attr "type" "mve_move")
1055 ])
1056
1057 ;;
1058 ;; [vcmpgeq_s])
1059 ;;
1060 (define_insn "mve_vcmpgeq_s<mode>"
1061   [
1062    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1063         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1064                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1065          VCMPGEQ_S))
1066   ]
1067   "TARGET_HAVE_MVE"
1068   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1069   [(set_attr "type" "mve_move")
1070 ])
1071
1072 ;;
1073 ;; [vcmpgtq_n_s])
1074 ;;
1075 (define_insn "mve_vcmpgtq_n_s<mode>"
1076   [
1077    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1078         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1079                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1080          VCMPGTQ_N_S))
1081   ]
1082   "TARGET_HAVE_MVE"
1083   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1084   [(set_attr "type" "mve_move")
1085 ])
1086
1087 ;;
1088 ;; [vcmpgtq_s])
1089 ;;
1090 (define_insn "mve_vcmpgtq_s<mode>"
1091   [
1092    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1093         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1094                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1095          VCMPGTQ_S))
1096   ]
1097   "TARGET_HAVE_MVE"
1098   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1099   [(set_attr "type" "mve_move")
1100 ])
1101
1102 ;;
1103 ;; [vcmphiq_n_u])
1104 ;;
1105 (define_insn "mve_vcmphiq_n_u<mode>"
1106   [
1107    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1108         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1109                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1110          VCMPHIQ_N_U))
1111   ]
1112   "TARGET_HAVE_MVE"
1113   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1114   [(set_attr "type" "mve_move")
1115 ])
1116
1117 ;;
1118 ;; [vcmphiq_u])
1119 ;;
1120 (define_insn "mve_vcmphiq_u<mode>"
1121   [
1122    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1123         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1124                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1125          VCMPHIQ_U))
1126   ]
1127   "TARGET_HAVE_MVE"
1128   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1129   [(set_attr "type" "mve_move")
1130 ])
1131
1132 ;;
1133 ;; [vcmpleq_n_s])
1134 ;;
1135 (define_insn "mve_vcmpleq_n_s<mode>"
1136   [
1137    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1138         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1139                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1140          VCMPLEQ_N_S))
1141   ]
1142   "TARGET_HAVE_MVE"
1143   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1144   [(set_attr "type" "mve_move")
1145 ])
1146
1147 ;;
1148 ;; [vcmpleq_s])
1149 ;;
1150 (define_insn "mve_vcmpleq_s<mode>"
1151   [
1152    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1153         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1154                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1155          VCMPLEQ_S))
1156   ]
1157   "TARGET_HAVE_MVE"
1158   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1159   [(set_attr "type" "mve_move")
1160 ])
1161
1162 ;;
1163 ;; [vcmpltq_n_s])
1164 ;;
1165 (define_insn "mve_vcmpltq_n_s<mode>"
1166   [
1167    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1168         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1169                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1170          VCMPLTQ_N_S))
1171   ]
1172   "TARGET_HAVE_MVE"
1173   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1174   [(set_attr "type" "mve_move")
1175 ])
1176
1177 ;;
1178 ;; [vcmpltq_s])
1179 ;;
1180 (define_insn "mve_vcmpltq_s<mode>"
1181   [
1182    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1183         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1184                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1185          VCMPLTQ_S))
1186   ]
1187   "TARGET_HAVE_MVE"
1188   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1189   [(set_attr "type" "mve_move")
1190 ])
1191
1192 ;;
1193 ;; [vcmpneq_n_u, vcmpneq_n_s])
1194 ;;
1195 (define_insn "mve_vcmpneq_n_<supf><mode>"
1196   [
1197    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1198         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1199                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1200          VCMPNEQ_N))
1201   ]
1202   "TARGET_HAVE_MVE"
1203   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1204   [(set_attr "type" "mve_move")
1205 ])
1206
1207 ;;
1208 ;; [veorq_u, veorq_s])
1209 ;;
1210 (define_insn "mve_veorq_u<mode>"
1211   [
1212    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1213         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1214                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1215   ]
1216   "TARGET_HAVE_MVE"
1217   "veor\t%q0, %q1, %q2"
1218   [(set_attr "type" "mve_move")
1219 ])
1220 (define_expand "mve_veorq_s<mode>"
1221   [
1222    (set (match_operand:MVE_2 0 "s_register_operand")
1223         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1224                    (match_operand:MVE_2 2 "s_register_operand")))
1225   ]
1226   "TARGET_HAVE_MVE"
1227 )
1228
1229 ;;
1230 ;; [vhaddq_n_u, vhaddq_n_s])
1231 ;;
1232 (define_insn "mve_vhaddq_n_<supf><mode>"
1233   [
1234    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1235         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1236                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1237          VHADDQ_N))
1238   ]
1239   "TARGET_HAVE_MVE"
1240   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1241   [(set_attr "type" "mve_move")
1242 ])
1243
1244 ;;
1245 ;; [vhaddq_s, vhaddq_u])
1246 ;;
1247 (define_insn "mve_vhaddq_<supf><mode>"
1248   [
1249    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1250         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1251                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1252          VHADDQ))
1253   ]
1254   "TARGET_HAVE_MVE"
1255   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1256   [(set_attr "type" "mve_move")
1257 ])
1258
1259 ;;
1260 ;; [vhcaddq_rot270_s])
1261 ;;
1262 (define_insn "mve_vhcaddq_rot270_s<mode>"
1263   [
1264    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1265         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1266                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1267          VHCADDQ_ROT270_S))
1268   ]
1269   "TARGET_HAVE_MVE"
1270   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1271   [(set_attr "type" "mve_move")
1272 ])
1273
1274 ;;
1275 ;; [vhcaddq_rot90_s])
1276 ;;
1277 (define_insn "mve_vhcaddq_rot90_s<mode>"
1278   [
1279    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1280         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1281                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1282          VHCADDQ_ROT90_S))
1283   ]
1284   "TARGET_HAVE_MVE"
1285   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1286   [(set_attr "type" "mve_move")
1287 ])
1288
1289 ;;
1290 ;; [vhsubq_n_u, vhsubq_n_s])
1291 ;;
1292 (define_insn "mve_vhsubq_n_<supf><mode>"
1293   [
1294    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1295         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1296                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1297          VHSUBQ_N))
1298   ]
1299   "TARGET_HAVE_MVE"
1300   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1301   [(set_attr "type" "mve_move")
1302 ])
1303
1304 ;;
1305 ;; [vhsubq_s, vhsubq_u])
1306 ;;
1307 (define_insn "mve_vhsubq_<supf><mode>"
1308   [
1309    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1310         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1311                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1312          VHSUBQ))
1313   ]
1314   "TARGET_HAVE_MVE"
1315   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1316   [(set_attr "type" "mve_move")
1317 ])
1318
1319 ;;
1320 ;; [vmaxaq_s])
1321 ;;
1322 (define_insn "mve_vmaxaq_s<mode>"
1323   [
1324    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1325         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1326                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1327          VMAXAQ_S))
1328   ]
1329   "TARGET_HAVE_MVE"
1330   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1331   [(set_attr "type" "mve_move")
1332 ])
1333
1334 ;;
1335 ;; [vmaxavq_s])
1336 ;;
1337 (define_insn "mve_vmaxavq_s<mode>"
1338   [
1339    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1340         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1341                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1342          VMAXAVQ_S))
1343   ]
1344   "TARGET_HAVE_MVE"
1345   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1346   [(set_attr "type" "mve_move")
1347 ])
1348
1349 ;;
1350 ;; [vmaxq_u, vmaxq_s])
1351 ;;
1352 (define_insn "mve_vmaxq_s<mode>"
1353   [
1354    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1355         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1356                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1357   ]
1358   "TARGET_HAVE_MVE"
1359   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1360   [(set_attr "type" "mve_move")
1361 ])
1362
1363 (define_insn "mve_vmaxq_u<mode>"
1364   [
1365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1366         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1367                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1368   ]
1369   "TARGET_HAVE_MVE"
1370   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1371   [(set_attr "type" "mve_move")
1372 ])
1373
1374 ;;
1375 ;; [vmaxvq_u, vmaxvq_s])
1376 ;;
1377 (define_insn "mve_vmaxvq_<supf><mode>"
1378   [
1379    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1380         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1381                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1382          VMAXVQ))
1383   ]
1384   "TARGET_HAVE_MVE"
1385   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1386   [(set_attr "type" "mve_move")
1387 ])
1388
1389 ;;
1390 ;; [vminaq_s])
1391 ;;
1392 (define_insn "mve_vminaq_s<mode>"
1393   [
1394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1395         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1396                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1397          VMINAQ_S))
1398   ]
1399   "TARGET_HAVE_MVE"
1400   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1401   [(set_attr "type" "mve_move")
1402 ])
1403
1404 ;;
1405 ;; [vminavq_s])
1406 ;;
1407 (define_insn "mve_vminavq_s<mode>"
1408   [
1409    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1410         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1411                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1412          VMINAVQ_S))
1413   ]
1414   "TARGET_HAVE_MVE"
1415   "vminav.s%#<V_sz_elem>\t%0, %q2"
1416   [(set_attr "type" "mve_move")
1417 ])
1418
1419 ;;
1420 ;; [vminq_s, vminq_u])
1421 ;;
1422 (define_insn "mve_vminq_s<mode>"
1423   [
1424    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1425         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1426                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1427   ]
1428   "TARGET_HAVE_MVE"
1429   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1430   [(set_attr "type" "mve_move")
1431 ])
1432
1433 (define_insn "mve_vminq_u<mode>"
1434   [
1435    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1436         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1437                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1438   ]
1439   "TARGET_HAVE_MVE"
1440   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1441   [(set_attr "type" "mve_move")
1442 ])
1443
1444 ;;
1445 ;; [vminvq_u, vminvq_s])
1446 ;;
1447 (define_insn "mve_vminvq_<supf><mode>"
1448   [
1449    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1450         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1451                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1452          VMINVQ))
1453   ]
1454   "TARGET_HAVE_MVE"
1455   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1456   [(set_attr "type" "mve_move")
1457 ])
1458
1459 ;;
1460 ;; [vmladavq_u, vmladavq_s])
1461 ;;
1462 (define_insn "mve_vmladavq_<supf><mode>"
1463   [
1464    (set (match_operand:SI 0 "s_register_operand" "=Te")
1465         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1466                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1467          VMLADAVQ))
1468   ]
1469   "TARGET_HAVE_MVE"
1470   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1471   [(set_attr "type" "mve_move")
1472 ])
1473
1474 ;;
1475 ;; [vmladavxq_s])
1476 ;;
1477 (define_insn "mve_vmladavxq_s<mode>"
1478   [
1479    (set (match_operand:SI 0 "s_register_operand" "=Te")
1480         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1481                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1482          VMLADAVXQ_S))
1483   ]
1484   "TARGET_HAVE_MVE"
1485   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1486   [(set_attr "type" "mve_move")
1487 ])
1488
1489 ;;
1490 ;; [vmlsdavq_s])
1491 ;;
1492 (define_insn "mve_vmlsdavq_s<mode>"
1493   [
1494    (set (match_operand:SI 0 "s_register_operand" "=Te")
1495         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1496                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1497          VMLSDAVQ_S))
1498   ]
1499   "TARGET_HAVE_MVE"
1500   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1501   [(set_attr "type" "mve_move")
1502 ])
1503
1504 ;;
1505 ;; [vmlsdavxq_s])
1506 ;;
1507 (define_insn "mve_vmlsdavxq_s<mode>"
1508   [
1509    (set (match_operand:SI 0 "s_register_operand" "=Te")
1510         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1511                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1512          VMLSDAVXQ_S))
1513   ]
1514   "TARGET_HAVE_MVE"
1515   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1516   [(set_attr "type" "mve_move")
1517 ])
1518
1519 ;;
1520 ;; [vmulhq_s, vmulhq_u])
1521 ;;
1522 (define_insn "mve_vmulhq_<supf><mode>"
1523   [
1524    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1525         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1526                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1527          VMULHQ))
1528   ]
1529   "TARGET_HAVE_MVE"
1530   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1531   [(set_attr "type" "mve_move")
1532 ])
1533
1534 ;;
1535 ;; [vmullbq_int_u, vmullbq_int_s])
1536 ;;
1537 (define_insn "mve_vmullbq_int_<supf><mode>"
1538   [
1539    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1540         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1541                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1542          VMULLBQ_INT))
1543   ]
1544   "TARGET_HAVE_MVE"
1545   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1546   [(set_attr "type" "mve_move")
1547 ])
1548
1549 ;;
1550 ;; [vmulltq_int_u, vmulltq_int_s])
1551 ;;
1552 (define_insn "mve_vmulltq_int_<supf><mode>"
1553   [
1554    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1555         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1556                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1557          VMULLTQ_INT))
1558   ]
1559   "TARGET_HAVE_MVE"
1560   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1561   [(set_attr "type" "mve_move")
1562 ])
1563
1564 ;;
1565 ;; [vmulq_n_u, vmulq_n_s])
1566 ;;
1567 (define_insn "mve_vmulq_n_<supf><mode>"
1568   [
1569    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1570         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1571                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1572          VMULQ_N))
1573   ]
1574   "TARGET_HAVE_MVE"
1575   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1576   [(set_attr "type" "mve_move")
1577 ])
1578
1579 ;;
1580 ;; [vmulq_u, vmulq_s])
1581 ;;
1582 (define_insn "mve_vmulq_<supf><mode>"
1583   [
1584    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1585         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1586                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1587          VMULQ))
1588   ]
1589   "TARGET_HAVE_MVE"
1590   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1591   [(set_attr "type" "mve_move")
1592 ])
1593
1594 (define_insn "mve_vmulq<mode>"
1595   [
1596    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1597         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1598                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1599   ]
1600   "TARGET_HAVE_MVE"
1601   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1602   [(set_attr "type" "mve_move")
1603 ])
1604
1605 ;;
1606 ;; [vornq_u, vornq_s])
1607 ;;
1608 (define_insn "mve_vornq_s<mode>"
1609   [
1610    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1611         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1612                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1613   ]
1614   "TARGET_HAVE_MVE"
1615    "vorn\t%q0, %q1, %q2"
1616   [(set_attr "type" "mve_move")
1617 ])
1618
1619 (define_expand "mve_vornq_u<mode>"
1620   [
1621    (set (match_operand:MVE_2 0 "s_register_operand")
1622         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1623                    (match_operand:MVE_2 1 "s_register_operand")))
1624   ]
1625   "TARGET_HAVE_MVE"
1626 )
1627
1628 ;;
1629 ;; [vorrq_s, vorrq_u])
1630 ;;
1631 ;; signed and unsigned versions are the same: define the unsigned
1632 ;; insn, and use an expander for the signed one as we still reference
1633 ;; both names from arm_mve.h.
1634 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1635 (define_insn "mve_vorrq_s<mode>"
1636   [
1637    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1638         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1639                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1640   ]
1641   "TARGET_HAVE_MVE"
1642   "@
1643    vorr\t%q0, %q1, %q2
1644    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1645   [(set_attr "type" "mve_move")
1646 ])
1647 (define_expand "mve_vorrq_u<mode>"
1648   [
1649    (set (match_operand:MVE_2 0 "s_register_operand")
1650         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1651                    (match_operand:MVE_2 2 "neon_logic_op2")))
1652   ]
1653   "TARGET_HAVE_MVE"
1654 )
1655
1656 ;;
1657 ;; [vqaddq_n_s, vqaddq_n_u])
1658 ;;
1659 (define_insn "mve_vqaddq_n_<supf><mode>"
1660   [
1661    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1662         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1663                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1664          VQADDQ_N))
1665   ]
1666   "TARGET_HAVE_MVE"
1667   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1668   [(set_attr "type" "mve_move")
1669 ])
1670
1671 ;;
1672 ;; [vqaddq_u, vqaddq_s])
1673 ;;
1674 (define_insn "mve_vqaddq_<supf><mode>"
1675   [
1676    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1677         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1678                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1679          VQADDQ))
1680   ]
1681   "TARGET_HAVE_MVE"
1682   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1683   [(set_attr "type" "mve_move")
1684 ])
1685
1686 ;;
1687 ;; [vqdmulhq_n_s])
1688 ;;
1689 (define_insn "mve_vqdmulhq_n_s<mode>"
1690   [
1691    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1692         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1693                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1694          VQDMULHQ_N_S))
1695   ]
1696   "TARGET_HAVE_MVE"
1697   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1698   [(set_attr "type" "mve_move")
1699 ])
1700
1701 ;;
1702 ;; [vqdmulhq_s])
1703 ;;
1704 (define_insn "mve_vqdmulhq_s<mode>"
1705   [
1706    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1707         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1708                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1709          VQDMULHQ_S))
1710   ]
1711   "TARGET_HAVE_MVE"
1712   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1713   [(set_attr "type" "mve_move")
1714 ])
1715
1716 ;;
1717 ;; [vqrdmulhq_n_s])
1718 ;;
1719 (define_insn "mve_vqrdmulhq_n_s<mode>"
1720   [
1721    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1722         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1723                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1724          VQRDMULHQ_N_S))
1725   ]
1726   "TARGET_HAVE_MVE"
1727   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1728   [(set_attr "type" "mve_move")
1729 ])
1730
1731 ;;
1732 ;; [vqrdmulhq_s])
1733 ;;
1734 (define_insn "mve_vqrdmulhq_s<mode>"
1735   [
1736    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1737         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1738                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1739          VQRDMULHQ_S))
1740   ]
1741   "TARGET_HAVE_MVE"
1742   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1743   [(set_attr "type" "mve_move")
1744 ])
1745
1746 ;;
1747 ;; [vqrshlq_n_s, vqrshlq_n_u])
1748 ;;
1749 (define_insn "mve_vqrshlq_n_<supf><mode>"
1750   [
1751    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1752         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1753                        (match_operand:SI 2 "s_register_operand" "r")]
1754          VQRSHLQ_N))
1755   ]
1756   "TARGET_HAVE_MVE"
1757   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1758   [(set_attr "type" "mve_move")
1759 ])
1760
1761 ;;
1762 ;; [vqrshlq_s, vqrshlq_u])
1763 ;;
1764 (define_insn "mve_vqrshlq_<supf><mode>"
1765   [
1766    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1767         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1768                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1769          VQRSHLQ))
1770   ]
1771   "TARGET_HAVE_MVE"
1772   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1773   [(set_attr "type" "mve_move")
1774 ])
1775
1776 ;;
1777 ;; [vqshlq_n_s, vqshlq_n_u])
1778 ;;
1779 (define_insn "mve_vqshlq_n_<supf><mode>"
1780   [
1781    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1782         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1783                        (match_operand:SI 2 "immediate_operand" "i")]
1784          VQSHLQ_N))
1785   ]
1786   "TARGET_HAVE_MVE"
1787   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1788   [(set_attr "type" "mve_move")
1789 ])
1790
1791 ;;
1792 ;; [vqshlq_r_u, vqshlq_r_s])
1793 ;;
1794 (define_insn "mve_vqshlq_r_<supf><mode>"
1795   [
1796    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1797         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1798                        (match_operand:SI 2 "s_register_operand" "r")]
1799          VQSHLQ_R))
1800   ]
1801   "TARGET_HAVE_MVE"
1802   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1803   [(set_attr "type" "mve_move")
1804 ])
1805
1806 ;;
1807 ;; [vqshlq_s, vqshlq_u])
1808 ;;
1809 (define_insn "mve_vqshlq_<supf><mode>"
1810   [
1811    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1812         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1813                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1814          VQSHLQ))
1815   ]
1816   "TARGET_HAVE_MVE"
1817   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1818   [(set_attr "type" "mve_move")
1819 ])
1820
1821 ;;
1822 ;; [vqshluq_n_s])
1823 ;;
1824 (define_insn "mve_vqshluq_n_s<mode>"
1825   [
1826    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1827         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1828                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1829          VQSHLUQ_N_S))
1830   ]
1831   "TARGET_HAVE_MVE"
1832   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1833   [(set_attr "type" "mve_move")
1834 ])
1835
1836 ;;
1837 ;; [vqsubq_n_s, vqsubq_n_u])
1838 ;;
1839 (define_insn "mve_vqsubq_n_<supf><mode>"
1840   [
1841    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1842         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1843                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1844          VQSUBQ_N))
1845   ]
1846   "TARGET_HAVE_MVE"
1847   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1848   [(set_attr "type" "mve_move")
1849 ])
1850
1851 ;;
1852 ;; [vqsubq_u, vqsubq_s])
1853 ;;
1854 (define_insn "mve_vqsubq_<supf><mode>"
1855   [
1856    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1857         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1858                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1859          VQSUBQ))
1860   ]
1861   "TARGET_HAVE_MVE"
1862   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1863   [(set_attr "type" "mve_move")
1864 ])
1865
1866 ;;
1867 ;; [vrhaddq_s, vrhaddq_u])
1868 ;;
1869 (define_insn "mve_vrhaddq_<supf><mode>"
1870   [
1871    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1872         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1873                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1874          VRHADDQ))
1875   ]
1876   "TARGET_HAVE_MVE"
1877   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1878   [(set_attr "type" "mve_move")
1879 ])
1880
1881 ;;
1882 ;; [vrmulhq_s, vrmulhq_u])
1883 ;;
1884 (define_insn "mve_vrmulhq_<supf><mode>"
1885   [
1886    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1887         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1888                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1889          VRMULHQ))
1890   ]
1891   "TARGET_HAVE_MVE"
1892   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1893   [(set_attr "type" "mve_move")
1894 ])
1895
1896 ;;
1897 ;; [vrshlq_n_u, vrshlq_n_s])
1898 ;;
1899 (define_insn "mve_vrshlq_n_<supf><mode>"
1900   [
1901    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1902         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1903                        (match_operand:SI 2 "s_register_operand" "r")]
1904          VRSHLQ_N))
1905   ]
1906   "TARGET_HAVE_MVE"
1907   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1908   [(set_attr "type" "mve_move")
1909 ])
1910
1911 ;;
1912 ;; [vrshlq_s, vrshlq_u])
1913 ;;
1914 (define_insn "mve_vrshlq_<supf><mode>"
1915   [
1916    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1917         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1918                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1919          VRSHLQ))
1920   ]
1921   "TARGET_HAVE_MVE"
1922   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1923   [(set_attr "type" "mve_move")
1924 ])
1925
1926 ;;
1927 ;; [vrshrq_n_s, vrshrq_n_u])
1928 ;;
1929 (define_insn "mve_vrshrq_n_<supf><mode>"
1930   [
1931    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1932         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1933                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1934          VRSHRQ_N))
1935   ]
1936   "TARGET_HAVE_MVE"
1937   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1938   [(set_attr "type" "mve_move")
1939 ])
1940
1941 ;;
1942 ;; [vshlq_n_u, vshlq_n_s])
1943 ;;
1944 (define_insn "mve_vshlq_n_<supf><mode>"
1945   [
1946    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1947         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1948                        (match_operand:SI 2 "immediate_operand" "i")]
1949          VSHLQ_N))
1950   ]
1951   "TARGET_HAVE_MVE"
1952   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1953   [(set_attr "type" "mve_move")
1954 ])
1955
1956 ;;
1957 ;; [vshlq_r_s, vshlq_r_u])
1958 ;;
1959 (define_insn "mve_vshlq_r_<supf><mode>"
1960   [
1961    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1962         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1963                        (match_operand:SI 2 "s_register_operand" "r")]
1964          VSHLQ_R))
1965   ]
1966   "TARGET_HAVE_MVE"
1967   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1968   [(set_attr "type" "mve_move")
1969 ])
1970
1971 ;;
1972 ;; [vsubq_n_s, vsubq_n_u])
1973 ;;
1974 (define_insn "mve_vsubq_n_<supf><mode>"
1975   [
1976    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1977         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1978                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1979          VSUBQ_N))
1980   ]
1981   "TARGET_HAVE_MVE"
1982   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1983   [(set_attr "type" "mve_move")
1984 ])
1985
1986 ;;
1987 ;; [vsubq_s, vsubq_u])
1988 ;;
1989 (define_insn "mve_vsubq_<supf><mode>"
1990   [
1991    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1992         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1993                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1994          VSUBQ))
1995   ]
1996   "TARGET_HAVE_MVE"
1997   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1998   [(set_attr "type" "mve_move")
1999 ])
2000
2001 (define_insn "mve_vsubq<mode>"
2002   [
2003    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2004         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
2005                      (match_operand:MVE_2 2 "s_register_operand" "w")))
2006   ]
2007   "TARGET_HAVE_MVE"
2008   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2009   [(set_attr "type" "mve_move")
2010 ])
2011
2012 ;;
2013 ;; [vabdq_f])
2014 ;;
2015 (define_insn "mve_vabdq_f<mode>"
2016   [
2017    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2018         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2019                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2020          VABDQ_F))
2021   ]
2022   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2023   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2024   [(set_attr "type" "mve_move")
2025 ])
2026
2027 ;;
2028 ;; [vaddlvaq_s vaddlvaq_u])
2029 ;;
2030 (define_insn "mve_vaddlvaq_<supf>v4si"
2031   [
2032    (set (match_operand:DI 0 "s_register_operand" "=r")
2033         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2034                     (match_operand:V4SI 2 "s_register_operand" "w")]
2035          VADDLVAQ))
2036   ]
2037   "TARGET_HAVE_MVE"
2038   "vaddlva.<supf>32 %Q0, %R0, %q2"
2039   [(set_attr "type" "mve_move")
2040 ])
2041
2042 ;;
2043 ;; [vaddq_n_f])
2044 ;;
2045 (define_insn "mve_vaddq_n_f<mode>"
2046   [
2047    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2048         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2049                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2050          VADDQ_N_F))
2051   ]
2052   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2053   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2054   [(set_attr "type" "mve_move")
2055 ])
2056
2057 ;;
2058 ;; [vandq_f])
2059 ;;
2060 (define_insn "mve_vandq_f<mode>"
2061   [
2062    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2063         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2064                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2065   ]
2066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2067   "vand %q0, %q1, %q2"
2068   [(set_attr "type" "mve_move")
2069 ])
2070
2071 ;;
2072 ;; [vbicq_f])
2073 ;;
2074 (define_insn "mve_vbicq_f<mode>"
2075   [
2076    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2077         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
2078                               (match_operand:MVE_0 2 "s_register_operand" "w")))
2079   ]
2080   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2081   "vbic %q0, %q1, %q2"
2082   [(set_attr "type" "mve_move")
2083 ])
2084
2085 ;;
2086 ;; [vbicq_n_s, vbicq_n_u])
2087 ;;
2088 (define_insn "mve_vbicq_n_<supf><mode>"
2089   [
2090    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2091         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2092                        (match_operand:SI 2 "immediate_operand" "i")]
2093          VBICQ_N))
2094   ]
2095   "TARGET_HAVE_MVE"
2096   "vbic.i%#<V_sz_elem>  %q0, %2"
2097   [(set_attr "type" "mve_move")
2098 ])
2099
2100 ;;
2101 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
2102 ;;
2103 (define_insn "mve_vcaddq<mve_rot><mode>"
2104   [
2105    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2106         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2107                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2108          VCADD))
2109   ]
2110   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2111   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
2112   [(set_attr "type" "mve_move")
2113 ])
2114
2115 ;;
2116 ;; [vcmpeqq_f])
2117 ;;
2118 (define_insn "mve_vcmpeqq_f<mode>"
2119   [
2120    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2121         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2122                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2123          VCMPEQQ_F))
2124   ]
2125   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2126   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2127   [(set_attr "type" "mve_move")
2128 ])
2129
2130 ;;
2131 ;; [vcmpeqq_n_f])
2132 ;;
2133 (define_insn "mve_vcmpeqq_n_f<mode>"
2134   [
2135    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2136         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2137                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2138          VCMPEQQ_N_F))
2139   ]
2140   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2141   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2142   [(set_attr "type" "mve_move")
2143 ])
2144
2145 ;;
2146 ;; [vcmpgeq_f])
2147 ;;
2148 (define_insn "mve_vcmpgeq_f<mode>"
2149   [
2150    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2151         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2152                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2153          VCMPGEQ_F))
2154   ]
2155   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2156   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2157   [(set_attr "type" "mve_move")
2158 ])
2159
2160 ;;
2161 ;; [vcmpgeq_n_f])
2162 ;;
2163 (define_insn "mve_vcmpgeq_n_f<mode>"
2164   [
2165    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2166         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2167                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2168          VCMPGEQ_N_F))
2169   ]
2170   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2171   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2172   [(set_attr "type" "mve_move")
2173 ])
2174
2175 ;;
2176 ;; [vcmpgtq_f])
2177 ;;
2178 (define_insn "mve_vcmpgtq_f<mode>"
2179   [
2180    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2181         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2182                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2183          VCMPGTQ_F))
2184   ]
2185   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2186   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2187   [(set_attr "type" "mve_move")
2188 ])
2189
2190 ;;
2191 ;; [vcmpgtq_n_f])
2192 ;;
2193 (define_insn "mve_vcmpgtq_n_f<mode>"
2194   [
2195    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2196         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2197                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2198          VCMPGTQ_N_F))
2199   ]
2200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2201   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2202   [(set_attr "type" "mve_move")
2203 ])
2204
2205 ;;
2206 ;; [vcmpleq_f])
2207 ;;
2208 (define_insn "mve_vcmpleq_f<mode>"
2209   [
2210    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2211         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2212                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2213          VCMPLEQ_F))
2214   ]
2215   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2216   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2217   [(set_attr "type" "mve_move")
2218 ])
2219
2220 ;;
2221 ;; [vcmpleq_n_f])
2222 ;;
2223 (define_insn "mve_vcmpleq_n_f<mode>"
2224   [
2225    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2226         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2227                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2228          VCMPLEQ_N_F))
2229   ]
2230   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2231   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2232   [(set_attr "type" "mve_move")
2233 ])
2234
2235 ;;
2236 ;; [vcmpltq_f])
2237 ;;
2238 (define_insn "mve_vcmpltq_f<mode>"
2239   [
2240    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2241         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2242                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2243          VCMPLTQ_F))
2244   ]
2245   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2246   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2247   [(set_attr "type" "mve_move")
2248 ])
2249
2250 ;;
2251 ;; [vcmpltq_n_f])
2252 ;;
2253 (define_insn "mve_vcmpltq_n_f<mode>"
2254   [
2255    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2256         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2257                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2258          VCMPLTQ_N_F))
2259   ]
2260   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2261   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2262   [(set_attr "type" "mve_move")
2263 ])
2264
2265 ;;
2266 ;; [vcmpneq_f])
2267 ;;
2268 (define_insn "mve_vcmpneq_f<mode>"
2269   [
2270    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2271         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2272                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2273          VCMPNEQ_F))
2274   ]
2275   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2276   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2277   [(set_attr "type" "mve_move")
2278 ])
2279
2280 ;;
2281 ;; [vcmpneq_n_f])
2282 ;;
2283 (define_insn "mve_vcmpneq_n_f<mode>"
2284   [
2285    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2286         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2287                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2288          VCMPNEQ_N_F))
2289   ]
2290   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2291   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2292   [(set_attr "type" "mve_move")
2293 ])
2294
2295 ;;
2296 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
2297 ;;
2298 (define_insn "mve_vcmulq<mve_rot><mode>"
2299   [
2300    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2301         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2302                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2303          VCMUL))
2304   ]
2305   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2306   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
2307   [(set_attr "type" "mve_move")
2308 ])
2309
2310 ;;
2311 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2312 ;;
2313 (define_insn "mve_vctp<mode1>q_mhi"
2314   [
2315    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2316         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2317                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2318          VCTPQ_M))
2319   ]
2320   "TARGET_HAVE_MVE"
2321   "vpst\;vctpt.<mode1> %1"
2322   [(set_attr "type" "mve_move")
2323    (set_attr "length""8")])
2324
2325 ;;
2326 ;; [vcvtbq_f16_f32])
2327 ;;
2328 (define_insn "mve_vcvtbq_f16_f32v8hf"
2329   [
2330    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2331         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2332                       (match_operand:V4SF 2 "s_register_operand" "w")]
2333          VCVTBQ_F16_F32))
2334   ]
2335   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2336   "vcvtb.f16.f32 %q0, %q2"
2337   [(set_attr "type" "mve_move")
2338 ])
2339
2340 ;;
2341 ;; [vcvttq_f16_f32])
2342 ;;
2343 (define_insn "mve_vcvttq_f16_f32v8hf"
2344   [
2345    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2346         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2347                       (match_operand:V4SF 2 "s_register_operand" "w")]
2348          VCVTTQ_F16_F32))
2349   ]
2350   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2351   "vcvtt.f16.f32 %q0, %q2"
2352   [(set_attr "type" "mve_move")
2353 ])
2354
2355 ;;
2356 ;; [veorq_f])
2357 ;;
2358 (define_insn "mve_veorq_f<mode>"
2359   [
2360    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2361         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2362                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2363   ]
2364   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2365   "veor %q0, %q1, %q2"
2366   [(set_attr "type" "mve_move")
2367 ])
2368
2369 ;;
2370 ;; [vmaxnmaq_f])
2371 ;;
2372 (define_insn "mve_vmaxnmaq_f<mode>"
2373   [
2374    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2375         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2376                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2377          VMAXNMAQ_F))
2378   ]
2379   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2380   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2381   [(set_attr "type" "mve_move")
2382 ])
2383
2384 ;;
2385 ;; [vmaxnmavq_f])
2386 ;;
2387 (define_insn "mve_vmaxnmavq_f<mode>"
2388   [
2389    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2390         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2391                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2392          VMAXNMAVQ_F))
2393   ]
2394   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2395   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2396   [(set_attr "type" "mve_move")
2397 ])
2398
2399 ;;
2400 ;; [vmaxnmq_f])
2401 ;;
2402 (define_insn "mve_vmaxnmq_f<mode>"
2403   [
2404    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2405         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2406                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2407   ]
2408   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2409   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2410   [(set_attr "type" "mve_move")
2411 ])
2412
2413 ;;
2414 ;; [vmaxnmvq_f])
2415 ;;
2416 (define_insn "mve_vmaxnmvq_f<mode>"
2417   [
2418    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2419         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2420                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2421          VMAXNMVQ_F))
2422   ]
2423   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2424   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2425   [(set_attr "type" "mve_move")
2426 ])
2427
2428 ;;
2429 ;; [vminnmaq_f])
2430 ;;
2431 (define_insn "mve_vminnmaq_f<mode>"
2432   [
2433    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2434         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2435                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2436          VMINNMAQ_F))
2437   ]
2438   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2439   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2440   [(set_attr "type" "mve_move")
2441 ])
2442
2443 ;;
2444 ;; [vminnmavq_f])
2445 ;;
2446 (define_insn "mve_vminnmavq_f<mode>"
2447   [
2448    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2449         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2450                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2451          VMINNMAVQ_F))
2452   ]
2453   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2454   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2455   [(set_attr "type" "mve_move")
2456 ])
2457
2458 ;;
2459 ;; [vminnmq_f])
2460 ;;
2461 (define_insn "mve_vminnmq_f<mode>"
2462   [
2463    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2464         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2465                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2466   ]
2467   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2468   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2469   [(set_attr "type" "mve_move")
2470 ])
2471
2472 ;;
2473 ;; [vminnmvq_f])
2474 ;;
2475 (define_insn "mve_vminnmvq_f<mode>"
2476   [
2477    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2478         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2479                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2480          VMINNMVQ_F))
2481   ]
2482   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2483   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2484   [(set_attr "type" "mve_move")
2485 ])
2486
2487 ;;
2488 ;; [vmlaldavq_u, vmlaldavq_s])
2489 ;;
2490 (define_insn "mve_vmlaldavq_<supf><mode>"
2491   [
2492    (set (match_operand:DI 0 "s_register_operand" "=r")
2493         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2494                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2495          VMLALDAVQ))
2496   ]
2497   "TARGET_HAVE_MVE"
2498   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2499   [(set_attr "type" "mve_move")
2500 ])
2501
2502 ;;
2503 ;; [vmlaldavxq_s])
2504 ;;
2505 (define_insn "mve_vmlaldavxq_s<mode>"
2506   [
2507    (set (match_operand:DI 0 "s_register_operand" "=r")
2508         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2509                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2510          VMLALDAVXQ_S))
2511   ]
2512   "TARGET_HAVE_MVE"
2513   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2514   [(set_attr "type" "mve_move")
2515 ])
2516
2517 ;;
2518 ;; [vmlsldavq_s])
2519 ;;
2520 (define_insn "mve_vmlsldavq_s<mode>"
2521   [
2522    (set (match_operand:DI 0 "s_register_operand" "=r")
2523         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2524                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2525          VMLSLDAVQ_S))
2526   ]
2527   "TARGET_HAVE_MVE"
2528   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2529   [(set_attr "type" "mve_move")
2530 ])
2531
2532 ;;
2533 ;; [vmlsldavxq_s])
2534 ;;
2535 (define_insn "mve_vmlsldavxq_s<mode>"
2536   [
2537    (set (match_operand:DI 0 "s_register_operand" "=r")
2538         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2539                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2540          VMLSLDAVXQ_S))
2541   ]
2542   "TARGET_HAVE_MVE"
2543   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2544   [(set_attr "type" "mve_move")
2545 ])
2546
2547 ;;
2548 ;; [vmovnbq_u, vmovnbq_s])
2549 ;;
2550 (define_insn "mve_vmovnbq_<supf><mode>"
2551   [
2552    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2553         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2554                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2555          VMOVNBQ))
2556   ]
2557   "TARGET_HAVE_MVE"
2558   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2559   [(set_attr "type" "mve_move")
2560 ])
2561
2562 ;;
2563 ;; [vmovntq_s, vmovntq_u])
2564 ;;
2565 (define_insn "mve_vmovntq_<supf><mode>"
2566   [
2567    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2568         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2569                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2570          VMOVNTQ))
2571   ]
2572   "TARGET_HAVE_MVE"
2573   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2574   [(set_attr "type" "mve_move")
2575 ])
2576
2577 ;;
2578 ;; [vmulq_f])
2579 ;;
2580 (define_insn "mve_vmulq_f<mode>"
2581   [
2582    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2583         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2584                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2585   ]
2586   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2587   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2588   [(set_attr "type" "mve_move")
2589 ])
2590
2591 ;;
2592 ;; [vmulq_n_f])
2593 ;;
2594 (define_insn "mve_vmulq_n_f<mode>"
2595   [
2596    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2597         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2598                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2599          VMULQ_N_F))
2600   ]
2601   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2602   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2603   [(set_attr "type" "mve_move")
2604 ])
2605
2606 ;;
2607 ;; [vornq_f])
2608 ;;
2609 (define_insn "mve_vornq_f<mode>"
2610   [
2611    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2612         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
2613                    (match_operand:MVE_0 1 "s_register_operand" "w")))
2614   ]
2615   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2616   "vorn %q0, %q1, %q2"
2617   [(set_attr "type" "mve_move")
2618 ])
2619
2620 ;;
2621 ;; [vorrq_f])
2622 ;;
2623 (define_insn "mve_vorrq_f<mode>"
2624   [
2625    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2626         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2627                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2628   ]
2629   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2630   "vorr %q0, %q1, %q2"
2631   [(set_attr "type" "mve_move")
2632 ])
2633
2634 ;;
2635 ;; [vorrq_n_u, vorrq_n_s])
2636 ;;
2637 (define_insn "mve_vorrq_n_<supf><mode>"
2638   [
2639    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2640         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2641                        (match_operand:SI 2 "immediate_operand" "i")]
2642          VORRQ_N))
2643   ]
2644   "TARGET_HAVE_MVE"
2645   "vorr.i%#<V_sz_elem>  %q0, %2"
2646   [(set_attr "type" "mve_move")
2647 ])
2648
2649 ;;
2650 ;; [vqdmullbq_n_s])
2651 ;;
2652 (define_insn "mve_vqdmullbq_n_s<mode>"
2653   [
2654    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2655         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2656                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2657          VQDMULLBQ_N_S))
2658   ]
2659   "TARGET_HAVE_MVE"
2660   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2661   [(set_attr "type" "mve_move")
2662 ])
2663
2664 ;;
2665 ;; [vqdmullbq_s])
2666 ;;
2667 (define_insn "mve_vqdmullbq_s<mode>"
2668   [
2669    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2670         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2671                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2672          VQDMULLBQ_S))
2673   ]
2674   "TARGET_HAVE_MVE"
2675   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2676   [(set_attr "type" "mve_move")
2677 ])
2678
2679 ;;
2680 ;; [vqdmulltq_n_s])
2681 ;;
2682 (define_insn "mve_vqdmulltq_n_s<mode>"
2683   [
2684    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2685         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2686                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2687          VQDMULLTQ_N_S))
2688   ]
2689   "TARGET_HAVE_MVE"
2690   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2691   [(set_attr "type" "mve_move")
2692 ])
2693
2694 ;;
2695 ;; [vqdmulltq_s])
2696 ;;
2697 (define_insn "mve_vqdmulltq_s<mode>"
2698   [
2699    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2700         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2701                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2702          VQDMULLTQ_S))
2703   ]
2704   "TARGET_HAVE_MVE"
2705   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2706   [(set_attr "type" "mve_move")
2707 ])
2708
2709 ;;
2710 ;; [vqmovnbq_u, vqmovnbq_s])
2711 ;;
2712 (define_insn "mve_vqmovnbq_<supf><mode>"
2713   [
2714    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2715         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2716                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2717          VQMOVNBQ))
2718   ]
2719   "TARGET_HAVE_MVE"
2720   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2721   [(set_attr "type" "mve_move")
2722 ])
2723
2724 ;;
2725 ;; [vqmovntq_u, vqmovntq_s])
2726 ;;
2727 (define_insn "mve_vqmovntq_<supf><mode>"
2728   [
2729    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2730         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2731                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2732          VQMOVNTQ))
2733   ]
2734   "TARGET_HAVE_MVE"
2735   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2736   [(set_attr "type" "mve_move")
2737 ])
2738
2739 ;;
2740 ;; [vqmovunbq_s])
2741 ;;
2742 (define_insn "mve_vqmovunbq_s<mode>"
2743   [
2744    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2745         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2746                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2747          VQMOVUNBQ_S))
2748   ]
2749   "TARGET_HAVE_MVE"
2750   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2751   [(set_attr "type" "mve_move")
2752 ])
2753
2754 ;;
2755 ;; [vqmovuntq_s])
2756 ;;
2757 (define_insn "mve_vqmovuntq_s<mode>"
2758   [
2759    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2760         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2761                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2762          VQMOVUNTQ_S))
2763   ]
2764   "TARGET_HAVE_MVE"
2765   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2766   [(set_attr "type" "mve_move")
2767 ])
2768
2769 ;;
2770 ;; [vrmlaldavhxq_s])
2771 ;;
2772 (define_insn "mve_vrmlaldavhxq_sv4si"
2773   [
2774    (set (match_operand:DI 0 "s_register_operand" "=r")
2775         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2776                     (match_operand:V4SI 2 "s_register_operand" "w")]
2777          VRMLALDAVHXQ_S))
2778   ]
2779   "TARGET_HAVE_MVE"
2780   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2781   [(set_attr "type" "mve_move")
2782 ])
2783
2784 ;;
2785 ;; [vrmlsldavhq_s])
2786 ;;
2787 (define_insn "mve_vrmlsldavhq_sv4si"
2788   [
2789    (set (match_operand:DI 0 "s_register_operand" "=r")
2790         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2791                     (match_operand:V4SI 2 "s_register_operand" "w")]
2792          VRMLSLDAVHQ_S))
2793   ]
2794   "TARGET_HAVE_MVE"
2795   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2796   [(set_attr "type" "mve_move")
2797 ])
2798
2799 ;;
2800 ;; [vrmlsldavhxq_s])
2801 ;;
2802 (define_insn "mve_vrmlsldavhxq_sv4si"
2803   [
2804    (set (match_operand:DI 0 "s_register_operand" "=r")
2805         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2806                     (match_operand:V4SI 2 "s_register_operand" "w")]
2807          VRMLSLDAVHXQ_S))
2808   ]
2809   "TARGET_HAVE_MVE"
2810   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2811   [(set_attr "type" "mve_move")
2812 ])
2813
2814 ;;
2815 ;; [vshllbq_n_s, vshllbq_n_u])
2816 ;;
2817 (define_insn "mve_vshllbq_n_<supf><mode>"
2818   [
2819    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2820         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2821                                   (match_operand:SI 2 "immediate_operand" "i")]
2822          VSHLLBQ_N))
2823   ]
2824   "TARGET_HAVE_MVE"
2825   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2826   [(set_attr "type" "mve_move")
2827 ])
2828
2829 ;;
2830 ;; [vshlltq_n_u, vshlltq_n_s])
2831 ;;
2832 (define_insn "mve_vshlltq_n_<supf><mode>"
2833   [
2834    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2835         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2836                                   (match_operand:SI 2 "immediate_operand" "i")]
2837          VSHLLTQ_N))
2838   ]
2839   "TARGET_HAVE_MVE"
2840   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2841   [(set_attr "type" "mve_move")
2842 ])
2843
2844 ;;
2845 ;; [vsubq_f])
2846 ;;
2847 (define_insn "mve_vsubq_f<mode>"
2848   [
2849    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2850         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2851                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2852   ]
2853   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2854   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2855   [(set_attr "type" "mve_move")
2856 ])
2857
2858 ;;
2859 ;; [vmulltq_poly_p])
2860 ;;
2861 (define_insn "mve_vmulltq_poly_p<mode>"
2862   [
2863    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2864         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2865                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2866          VMULLTQ_POLY_P))
2867   ]
2868   "TARGET_HAVE_MVE"
2869   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2870   [(set_attr "type" "mve_move")
2871 ])
2872
2873 ;;
2874 ;; [vmullbq_poly_p])
2875 ;;
2876 (define_insn "mve_vmullbq_poly_p<mode>"
2877   [
2878    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2879         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2880                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2881          VMULLBQ_POLY_P))
2882   ]
2883   "TARGET_HAVE_MVE"
2884   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2885   [(set_attr "type" "mve_move")
2886 ])
2887
2888 ;;
2889 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2890 ;;
2891 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2892   [
2893    (set (match_operand:DI 0 "s_register_operand" "=r")
2894         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2895                     (match_operand:V4SI 2 "s_register_operand" "w")]
2896          VRMLALDAVHQ))
2897   ]
2898   "TARGET_HAVE_MVE"
2899   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2900   [(set_attr "type" "mve_move")
2901 ])
2902
2903 ;;
2904 ;; [vbicq_m_n_s, vbicq_m_n_u])
2905 ;;
2906 (define_insn "mve_vbicq_m_n_<supf><mode>"
2907   [
2908    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2909         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2910                        (match_operand:SI 2 "immediate_operand" "i")
2911                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2912          VBICQ_M_N))
2913   ]
2914   "TARGET_HAVE_MVE"
2915   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2916   [(set_attr "type" "mve_move")
2917    (set_attr "length""8")])
2918 ;;
2919 ;; [vcmpeqq_m_f])
2920 ;;
2921 (define_insn "mve_vcmpeqq_m_f<mode>"
2922   [
2923    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2924         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2925                     (match_operand:MVE_0 2 "s_register_operand" "w")
2926                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2927          VCMPEQQ_M_F))
2928   ]
2929   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2930   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2931   [(set_attr "type" "mve_move")
2932    (set_attr "length""8")])
2933 ;;
2934 ;; [vcvtaq_m_u, vcvtaq_m_s])
2935 ;;
2936 (define_insn "mve_vcvtaq_m_<supf><mode>"
2937   [
2938    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2939         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2940                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2941                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2942          VCVTAQ_M))
2943   ]
2944   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2945   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2946   [(set_attr "type" "mve_move")
2947    (set_attr "length""8")])
2948 ;;
2949 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2950 ;;
2951 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2952   [
2953    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2954         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2955                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2956                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2957          VCVTQ_M_TO_F))
2958   ]
2959   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2960   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2961   [(set_attr "type" "mve_move")
2962    (set_attr "length""8")])
2963 ;;
2964 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2965 ;;
2966 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2967   [
2968    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2969         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2970                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2971                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2972          VQRSHRNBQ_N))
2973   ]
2974   "TARGET_HAVE_MVE"
2975   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2976   [(set_attr "type" "mve_move")
2977 ])
2978 ;;
2979 ;; [vqrshrunbq_n_s])
2980 ;;
2981 (define_insn "mve_vqrshrunbq_n_s<mode>"
2982   [
2983    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2984         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2985                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2986                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2987          VQRSHRUNBQ_N_S))
2988   ]
2989   "TARGET_HAVE_MVE"
2990   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
2991   [(set_attr "type" "mve_move")
2992 ])
2993 ;;
2994 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
2995 ;;
2996 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
2997   [
2998    (set (match_operand:DI 0 "s_register_operand" "=r")
2999         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3000                     (match_operand:V4SI 2 "s_register_operand" "w")
3001                     (match_operand:V4SI 3 "s_register_operand" "w")]
3002          VRMLALDAVHAQ))
3003   ]
3004   "TARGET_HAVE_MVE"
3005   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3006   [(set_attr "type" "mve_move")
3007 ])
3008
3009 ;;
3010 ;; [vabavq_s, vabavq_u])
3011 ;;
3012 (define_insn "mve_vabavq_<supf><mode>"
3013   [
3014    (set (match_operand:SI 0 "s_register_operand" "=r")
3015         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3016                     (match_operand:MVE_2 2 "s_register_operand" "w")
3017                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3018          VABAVQ))
3019   ]
3020   "TARGET_HAVE_MVE"
3021   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3022   [(set_attr "type" "mve_move")
3023 ])
3024
3025 ;;
3026 ;; [vshlcq_u vshlcq_s]
3027 ;;
3028 (define_expand "mve_vshlcq_vec_<supf><mode>"
3029  [(match_operand:MVE_2 0 "s_register_operand")
3030   (match_operand:MVE_2 1 "s_register_operand")
3031   (match_operand:SI 2 "s_register_operand")
3032   (match_operand:SI 3 "mve_imm_32")
3033   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3034  "TARGET_HAVE_MVE"
3035 {
3036   rtx ignore_wb = gen_reg_rtx (SImode);
3037   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3038                                       operands[2], operands[3]));
3039   DONE;
3040 })
3041
3042 (define_expand "mve_vshlcq_carry_<supf><mode>"
3043  [(match_operand:SI 0 "s_register_operand")
3044   (match_operand:MVE_2 1 "s_register_operand")
3045   (match_operand:SI 2 "s_register_operand")
3046   (match_operand:SI 3 "mve_imm_32")
3047   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3048  "TARGET_HAVE_MVE"
3049 {
3050   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3051   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3052                                       operands[2], operands[3]));
3053   DONE;
3054 })
3055
3056 (define_insn "mve_vshlcq_<supf><mode>"
3057  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3058        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3059                       (match_operand:SI 3 "s_register_operand" "1")
3060                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3061         VSHLCQ))
3062   (set (match_operand:SI  1 "s_register_operand" "=r")
3063        (unspec:SI [(match_dup 2)
3064                    (match_dup 3)
3065                    (match_dup 4)]
3066         VSHLCQ))]
3067  "TARGET_HAVE_MVE"
3068  "vshlc %q0, %1, %4")
3069
3070 ;;
3071 ;; [vabsq_m_s])
3072 ;;
3073 (define_insn "mve_vabsq_m_s<mode>"
3074   [
3075    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3076         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3077                        (match_operand:MVE_2 2 "s_register_operand" "w")
3078                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3079          VABSQ_M_S))
3080   ]
3081   "TARGET_HAVE_MVE"
3082   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3083   [(set_attr "type" "mve_move")
3084    (set_attr "length""8")])
3085
3086 ;;
3087 ;; [vaddvaq_p_u, vaddvaq_p_s])
3088 ;;
3089 (define_insn "mve_vaddvaq_p_<supf><mode>"
3090   [
3091    (set (match_operand:SI 0 "s_register_operand" "=Te")
3092         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3093                        (match_operand:MVE_2 2 "s_register_operand" "w")
3094                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3095          VADDVAQ_P))
3096   ]
3097   "TARGET_HAVE_MVE"
3098   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3099   [(set_attr "type" "mve_move")
3100    (set_attr "length""8")])
3101
3102 ;;
3103 ;; [vclsq_m_s])
3104 ;;
3105 (define_insn "mve_vclsq_m_s<mode>"
3106   [
3107    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3108         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3109                        (match_operand:MVE_2 2 "s_register_operand" "w")
3110                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3111          VCLSQ_M_S))
3112   ]
3113   "TARGET_HAVE_MVE"
3114   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3115   [(set_attr "type" "mve_move")
3116    (set_attr "length""8")])
3117
3118 ;;
3119 ;; [vclzq_m_s, vclzq_m_u])
3120 ;;
3121 (define_insn "mve_vclzq_m_<supf><mode>"
3122   [
3123    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3124         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3125                        (match_operand:MVE_2 2 "s_register_operand" "w")
3126                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3127          VCLZQ_M))
3128   ]
3129   "TARGET_HAVE_MVE"
3130   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3131   [(set_attr "type" "mve_move")
3132    (set_attr "length""8")])
3133
3134 ;;
3135 ;; [vcmpcsq_m_n_u])
3136 ;;
3137 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3138   [
3139    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3140         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3141                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3142                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3143          VCMPCSQ_M_N_U))
3144   ]
3145   "TARGET_HAVE_MVE"
3146   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3147   [(set_attr "type" "mve_move")
3148    (set_attr "length""8")])
3149
3150 ;;
3151 ;; [vcmpcsq_m_u])
3152 ;;
3153 (define_insn "mve_vcmpcsq_m_u<mode>"
3154   [
3155    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3156         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3157                        (match_operand:MVE_2 2 "s_register_operand" "w")
3158                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3159          VCMPCSQ_M_U))
3160   ]
3161   "TARGET_HAVE_MVE"
3162   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3163   [(set_attr "type" "mve_move")
3164    (set_attr "length""8")])
3165
3166 ;;
3167 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3168 ;;
3169 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3170   [
3171    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3172         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3173                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3174                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3175          VCMPEQQ_M_N))
3176   ]
3177   "TARGET_HAVE_MVE"
3178   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3179   [(set_attr "type" "mve_move")
3180    (set_attr "length""8")])
3181
3182 ;;
3183 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3184 ;;
3185 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3186   [
3187    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3188         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3189                        (match_operand:MVE_2 2 "s_register_operand" "w")
3190                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3191          VCMPEQQ_M))
3192   ]
3193   "TARGET_HAVE_MVE"
3194   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3195   [(set_attr "type" "mve_move")
3196    (set_attr "length""8")])
3197
3198 ;;
3199 ;; [vcmpgeq_m_n_s])
3200 ;;
3201 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3202   [
3203    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3204         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3205                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3206                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3207          VCMPGEQ_M_N_S))
3208   ]
3209   "TARGET_HAVE_MVE"
3210   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3211   [(set_attr "type" "mve_move")
3212    (set_attr "length""8")])
3213
3214 ;;
3215 ;; [vcmpgeq_m_s])
3216 ;;
3217 (define_insn "mve_vcmpgeq_m_s<mode>"
3218   [
3219    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3220         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3221                        (match_operand:MVE_2 2 "s_register_operand" "w")
3222                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3223          VCMPGEQ_M_S))
3224   ]
3225   "TARGET_HAVE_MVE"
3226   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3227   [(set_attr "type" "mve_move")
3228    (set_attr "length""8")])
3229
3230 ;;
3231 ;; [vcmpgtq_m_n_s])
3232 ;;
3233 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3234   [
3235    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3236         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3237                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3238                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3239          VCMPGTQ_M_N_S))
3240   ]
3241   "TARGET_HAVE_MVE"
3242   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3243   [(set_attr "type" "mve_move")
3244    (set_attr "length""8")])
3245
3246 ;;
3247 ;; [vcmpgtq_m_s])
3248 ;;
3249 (define_insn "mve_vcmpgtq_m_s<mode>"
3250   [
3251    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3252         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3253                        (match_operand:MVE_2 2 "s_register_operand" "w")
3254                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3255          VCMPGTQ_M_S))
3256   ]
3257   "TARGET_HAVE_MVE"
3258   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3259   [(set_attr "type" "mve_move")
3260    (set_attr "length""8")])
3261
3262 ;;
3263 ;; [vcmphiq_m_n_u])
3264 ;;
3265 (define_insn "mve_vcmphiq_m_n_u<mode>"
3266   [
3267    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3268         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3269                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3270                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3271          VCMPHIQ_M_N_U))
3272   ]
3273   "TARGET_HAVE_MVE"
3274   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3275   [(set_attr "type" "mve_move")
3276    (set_attr "length""8")])
3277
3278 ;;
3279 ;; [vcmphiq_m_u])
3280 ;;
3281 (define_insn "mve_vcmphiq_m_u<mode>"
3282   [
3283    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3284         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3285                        (match_operand:MVE_2 2 "s_register_operand" "w")
3286                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3287          VCMPHIQ_M_U))
3288   ]
3289   "TARGET_HAVE_MVE"
3290   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3291   [(set_attr "type" "mve_move")
3292    (set_attr "length""8")])
3293
3294 ;;
3295 ;; [vcmpleq_m_n_s])
3296 ;;
3297 (define_insn "mve_vcmpleq_m_n_s<mode>"
3298   [
3299    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3300         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3301                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3302                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3303          VCMPLEQ_M_N_S))
3304   ]
3305   "TARGET_HAVE_MVE"
3306   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3307   [(set_attr "type" "mve_move")
3308    (set_attr "length""8")])
3309
3310 ;;
3311 ;; [vcmpleq_m_s])
3312 ;;
3313 (define_insn "mve_vcmpleq_m_s<mode>"
3314   [
3315    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3316         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3317                        (match_operand:MVE_2 2 "s_register_operand" "w")
3318                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3319          VCMPLEQ_M_S))
3320   ]
3321   "TARGET_HAVE_MVE"
3322   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3323   [(set_attr "type" "mve_move")
3324    (set_attr "length""8")])
3325
3326 ;;
3327 ;; [vcmpltq_m_n_s])
3328 ;;
3329 (define_insn "mve_vcmpltq_m_n_s<mode>"
3330   [
3331    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3332         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3333                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3334                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3335          VCMPLTQ_M_N_S))
3336   ]
3337   "TARGET_HAVE_MVE"
3338   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3339   [(set_attr "type" "mve_move")
3340    (set_attr "length""8")])
3341
3342 ;;
3343 ;; [vcmpltq_m_s])
3344 ;;
3345 (define_insn "mve_vcmpltq_m_s<mode>"
3346   [
3347    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3348         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3349                        (match_operand:MVE_2 2 "s_register_operand" "w")
3350                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3351          VCMPLTQ_M_S))
3352   ]
3353   "TARGET_HAVE_MVE"
3354   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3355   [(set_attr "type" "mve_move")
3356    (set_attr "length""8")])
3357
3358 ;;
3359 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3360 ;;
3361 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3362   [
3363    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3364         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3365                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3366                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3367          VCMPNEQ_M_N))
3368   ]
3369   "TARGET_HAVE_MVE"
3370   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3371   [(set_attr "type" "mve_move")
3372    (set_attr "length""8")])
3373
3374 ;;
3375 ;; [vcmpneq_m_s, vcmpneq_m_u])
3376 ;;
3377 (define_insn "mve_vcmpneq_m_<supf><mode>"
3378   [
3379    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3380         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3381                        (match_operand:MVE_2 2 "s_register_operand" "w")
3382                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3383          VCMPNEQ_M))
3384   ]
3385   "TARGET_HAVE_MVE"
3386   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3387   [(set_attr "type" "mve_move")
3388    (set_attr "length""8")])
3389
3390 ;;
3391 ;; [vdupq_m_n_s, vdupq_m_n_u])
3392 ;;
3393 (define_insn "mve_vdupq_m_n_<supf><mode>"
3394   [
3395    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3396         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3397                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3398                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3399          VDUPQ_M_N))
3400   ]
3401   "TARGET_HAVE_MVE"
3402   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3403   [(set_attr "type" "mve_move")
3404    (set_attr "length""8")])
3405
3406 ;;
3407 ;; [vmaxaq_m_s])
3408 ;;
3409 (define_insn "mve_vmaxaq_m_s<mode>"
3410   [
3411    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3412         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3413                        (match_operand:MVE_2 2 "s_register_operand" "w")
3414                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3415          VMAXAQ_M_S))
3416   ]
3417   "TARGET_HAVE_MVE"
3418   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3419   [(set_attr "type" "mve_move")
3420    (set_attr "length""8")])
3421
3422 ;;
3423 ;; [vmaxavq_p_s])
3424 ;;
3425 (define_insn "mve_vmaxavq_p_s<mode>"
3426   [
3427    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3428         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3429                        (match_operand:MVE_2 2 "s_register_operand" "w")
3430                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3431          VMAXAVQ_P_S))
3432   ]
3433   "TARGET_HAVE_MVE"
3434   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3435   [(set_attr "type" "mve_move")
3436    (set_attr "length""8")])
3437
3438 ;;
3439 ;; [vmaxvq_p_u, vmaxvq_p_s])
3440 ;;
3441 (define_insn "mve_vmaxvq_p_<supf><mode>"
3442   [
3443    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3444         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3445                        (match_operand:MVE_2 2 "s_register_operand" "w")
3446                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3447          VMAXVQ_P))
3448   ]
3449   "TARGET_HAVE_MVE"
3450   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3451   [(set_attr "type" "mve_move")
3452    (set_attr "length""8")])
3453
3454 ;;
3455 ;; [vminaq_m_s])
3456 ;;
3457 (define_insn "mve_vminaq_m_s<mode>"
3458   [
3459    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3460         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3461                        (match_operand:MVE_2 2 "s_register_operand" "w")
3462                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3463          VMINAQ_M_S))
3464   ]
3465   "TARGET_HAVE_MVE"
3466   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3467   [(set_attr "type" "mve_move")
3468    (set_attr "length""8")])
3469
3470 ;;
3471 ;; [vminavq_p_s])
3472 ;;
3473 (define_insn "mve_vminavq_p_s<mode>"
3474   [
3475    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3476         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3477                        (match_operand:MVE_2 2 "s_register_operand" "w")
3478                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3479          VMINAVQ_P_S))
3480   ]
3481   "TARGET_HAVE_MVE"
3482   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3483   [(set_attr "type" "mve_move")
3484    (set_attr "length""8")])
3485
3486 ;;
3487 ;; [vminvq_p_s, vminvq_p_u])
3488 ;;
3489 (define_insn "mve_vminvq_p_<supf><mode>"
3490   [
3491    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3492         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3493                        (match_operand:MVE_2 2 "s_register_operand" "w")
3494                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3495          VMINVQ_P))
3496   ]
3497   "TARGET_HAVE_MVE"
3498   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3499   [(set_attr "type" "mve_move")
3500    (set_attr "length""8")])
3501
3502 ;;
3503 ;; [vmladavaq_u, vmladavaq_s])
3504 ;;
3505 (define_insn "mve_vmladavaq_<supf><mode>"
3506   [
3507    (set (match_operand:SI 0 "s_register_operand" "=Te")
3508         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3509                        (match_operand:MVE_2 2 "s_register_operand" "w")
3510                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3511          VMLADAVAQ))
3512   ]
3513   "TARGET_HAVE_MVE"
3514   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3515   [(set_attr "type" "mve_move")
3516 ])
3517
3518 ;;
3519 ;; [vmladavq_p_u, vmladavq_p_s])
3520 ;;
3521 (define_insn "mve_vmladavq_p_<supf><mode>"
3522   [
3523    (set (match_operand:SI 0 "s_register_operand" "=Te")
3524         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3525                        (match_operand:MVE_2 2 "s_register_operand" "w")
3526                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3527          VMLADAVQ_P))
3528   ]
3529   "TARGET_HAVE_MVE"
3530   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3531   [(set_attr "type" "mve_move")
3532    (set_attr "length""8")])
3533
3534 ;;
3535 ;; [vmladavxq_p_s])
3536 ;;
3537 (define_insn "mve_vmladavxq_p_s<mode>"
3538   [
3539    (set (match_operand:SI 0 "s_register_operand" "=Te")
3540         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3541                        (match_operand:MVE_2 2 "s_register_operand" "w")
3542                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3543          VMLADAVXQ_P_S))
3544   ]
3545   "TARGET_HAVE_MVE"
3546   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3547   [(set_attr "type" "mve_move")
3548    (set_attr "length""8")])
3549
3550 ;;
3551 ;; [vmlaq_n_u, vmlaq_n_s])
3552 ;;
3553 (define_insn "mve_vmlaq_n_<supf><mode>"
3554   [
3555    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3556         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3557                        (match_operand:MVE_2 2 "s_register_operand" "w")
3558                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3559          VMLAQ_N))
3560   ]
3561   "TARGET_HAVE_MVE"
3562   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3563   [(set_attr "type" "mve_move")
3564 ])
3565
3566 ;;
3567 ;; [vmlasq_n_u, vmlasq_n_s])
3568 ;;
3569 (define_insn "mve_vmlasq_n_<supf><mode>"
3570   [
3571    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3572         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3573                        (match_operand:MVE_2 2 "s_register_operand" "w")
3574                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3575          VMLASQ_N))
3576   ]
3577   "TARGET_HAVE_MVE"
3578   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3579   [(set_attr "type" "mve_move")
3580 ])
3581
3582 ;;
3583 ;; [vmlsdavq_p_s])
3584 ;;
3585 (define_insn "mve_vmlsdavq_p_s<mode>"
3586   [
3587    (set (match_operand:SI 0 "s_register_operand" "=Te")
3588         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3589                        (match_operand:MVE_2 2 "s_register_operand" "w")
3590                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3591          VMLSDAVQ_P_S))
3592   ]
3593   "TARGET_HAVE_MVE"
3594   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3595   [(set_attr "type" "mve_move")
3596    (set_attr "length""8")])
3597
3598 ;;
3599 ;; [vmlsdavxq_p_s])
3600 ;;
3601 (define_insn "mve_vmlsdavxq_p_s<mode>"
3602   [
3603    (set (match_operand:SI 0 "s_register_operand" "=Te")
3604         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3605                        (match_operand:MVE_2 2 "s_register_operand" "w")
3606                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3607          VMLSDAVXQ_P_S))
3608   ]
3609   "TARGET_HAVE_MVE"
3610   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3611   [(set_attr "type" "mve_move")
3612    (set_attr "length""8")])
3613
3614 ;;
3615 ;; [vmvnq_m_s, vmvnq_m_u])
3616 ;;
3617 (define_insn "mve_vmvnq_m_<supf><mode>"
3618   [
3619    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3620         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3621                        (match_operand:MVE_2 2 "s_register_operand" "w")
3622                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3623          VMVNQ_M))
3624   ]
3625   "TARGET_HAVE_MVE"
3626   "vpst\;vmvnt %q0, %q2"
3627   [(set_attr "type" "mve_move")
3628    (set_attr "length""8")])
3629
3630 ;;
3631 ;; [vnegq_m_s])
3632 ;;
3633 (define_insn "mve_vnegq_m_s<mode>"
3634   [
3635    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3636         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3637                        (match_operand:MVE_2 2 "s_register_operand" "w")
3638                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3639          VNEGQ_M_S))
3640   ]
3641   "TARGET_HAVE_MVE"
3642   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3643   [(set_attr "type" "mve_move")
3644    (set_attr "length""8")])
3645
3646 ;;
3647 ;; [vpselq_u, vpselq_s])
3648 ;;
3649 (define_insn "mve_vpselq_<supf><mode>"
3650   [
3651    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3652         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3653                        (match_operand:MVE_1 2 "s_register_operand" "w")
3654                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3655          VPSELQ))
3656   ]
3657   "TARGET_HAVE_MVE"
3658   "vpsel %q0, %q1, %q2"
3659   [(set_attr "type" "mve_move")
3660 ])
3661
3662 ;;
3663 ;; [vqabsq_m_s])
3664 ;;
3665 (define_insn "mve_vqabsq_m_s<mode>"
3666   [
3667    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3668         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3669                        (match_operand:MVE_2 2 "s_register_operand" "w")
3670                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3671          VQABSQ_M_S))
3672   ]
3673   "TARGET_HAVE_MVE"
3674   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3675   [(set_attr "type" "mve_move")
3676    (set_attr "length""8")])
3677
3678 ;;
3679 ;; [vqdmlahq_n_s])
3680 ;;
3681 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3682   [
3683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3685                        (match_operand:MVE_2 2 "s_register_operand" "w")
3686                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3687          VQDMLAHQ_N))
3688   ]
3689   "TARGET_HAVE_MVE"
3690   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3691   [(set_attr "type" "mve_move")
3692 ])
3693
3694 ;;
3695 ;; [vqdmlashq_n_s])
3696 ;;
3697 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3698   [
3699    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3700         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3701                        (match_operand:MVE_2 2 "s_register_operand" "w")
3702                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3703          VQDMLASHQ_N))
3704   ]
3705   "TARGET_HAVE_MVE"
3706   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3707   [(set_attr "type" "mve_move")
3708 ])
3709
3710 ;;
3711 ;; [vqnegq_m_s])
3712 ;;
3713 (define_insn "mve_vqnegq_m_s<mode>"
3714   [
3715    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3716         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3717                        (match_operand:MVE_2 2 "s_register_operand" "w")
3718                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3719          VQNEGQ_M_S))
3720   ]
3721   "TARGET_HAVE_MVE"
3722   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3723   [(set_attr "type" "mve_move")
3724    (set_attr "length""8")])
3725
3726 ;;
3727 ;; [vqrdmladhq_s])
3728 ;;
3729 (define_insn "mve_vqrdmladhq_s<mode>"
3730   [
3731    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3732         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3733                        (match_operand:MVE_2 2 "s_register_operand" "w")
3734                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3735          VQRDMLADHQ_S))
3736   ]
3737   "TARGET_HAVE_MVE"
3738   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3739   [(set_attr "type" "mve_move")
3740 ])
3741
3742 ;;
3743 ;; [vqrdmladhxq_s])
3744 ;;
3745 (define_insn "mve_vqrdmladhxq_s<mode>"
3746   [
3747    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3748         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3749                        (match_operand:MVE_2 2 "s_register_operand" "w")
3750                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3751          VQRDMLADHXQ_S))
3752   ]
3753   "TARGET_HAVE_MVE"
3754   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3755   [(set_attr "type" "mve_move")
3756 ])
3757
3758 ;;
3759 ;; [vqrdmlahq_n_s])
3760 ;;
3761 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3762   [
3763    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3764         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3765                        (match_operand:MVE_2 2 "s_register_operand" "w")
3766                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3767          VQRDMLAHQ_N))
3768   ]
3769   "TARGET_HAVE_MVE"
3770   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3771   [(set_attr "type" "mve_move")
3772 ])
3773
3774 ;;
3775 ;; [vqrdmlashq_n_s])
3776 ;;
3777 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3778   [
3779    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3780         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3781                        (match_operand:MVE_2 2 "s_register_operand" "w")
3782                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3783          VQRDMLASHQ_N))
3784   ]
3785   "TARGET_HAVE_MVE"
3786   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3787   [(set_attr "type" "mve_move")
3788 ])
3789
3790 ;;
3791 ;; [vqrdmlsdhq_s])
3792 ;;
3793 (define_insn "mve_vqrdmlsdhq_s<mode>"
3794   [
3795    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3796         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3797                        (match_operand:MVE_2 2 "s_register_operand" "w")
3798                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3799          VQRDMLSDHQ_S))
3800   ]
3801   "TARGET_HAVE_MVE"
3802   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3803   [(set_attr "type" "mve_move")
3804 ])
3805
3806 ;;
3807 ;; [vqrdmlsdhxq_s])
3808 ;;
3809 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3810   [
3811    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3812         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3813                        (match_operand:MVE_2 2 "s_register_operand" "w")
3814                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3815          VQRDMLSDHXQ_S))
3816   ]
3817   "TARGET_HAVE_MVE"
3818   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3819   [(set_attr "type" "mve_move")
3820 ])
3821
3822 ;;
3823 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3824 ;;
3825 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3826   [
3827    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3828         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3829                        (match_operand:SI 2 "s_register_operand" "r")
3830                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3831          VQRSHLQ_M_N))
3832   ]
3833   "TARGET_HAVE_MVE"
3834   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3835   [(set_attr "type" "mve_move")
3836    (set_attr "length""8")])
3837
3838 ;;
3839 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3840 ;;
3841 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3842   [
3843    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3844         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3845                        (match_operand:SI 2 "s_register_operand" "r")
3846                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3847          VQSHLQ_M_R))
3848   ]
3849   "TARGET_HAVE_MVE"
3850   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3851   [(set_attr "type" "mve_move")
3852    (set_attr "length""8")])
3853
3854 ;;
3855 ;; [vrev64q_m_u, vrev64q_m_s])
3856 ;;
3857 (define_insn "mve_vrev64q_m_<supf><mode>"
3858   [
3859    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3860         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3861                        (match_operand:MVE_2 2 "s_register_operand" "w")
3862                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3863          VREV64Q_M))
3864   ]
3865   "TARGET_HAVE_MVE"
3866   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3867   [(set_attr "type" "mve_move")
3868    (set_attr "length""8")])
3869
3870 ;;
3871 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3872 ;;
3873 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3874   [
3875    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3876         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3877                        (match_operand:SI 2 "s_register_operand" "r")
3878                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3879          VRSHLQ_M_N))
3880   ]
3881   "TARGET_HAVE_MVE"
3882   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3883   [(set_attr "type" "mve_move")
3884    (set_attr "length""8")])
3885
3886 ;;
3887 ;; [vshlq_m_r_u, vshlq_m_r_s])
3888 ;;
3889 (define_insn "mve_vshlq_m_r_<supf><mode>"
3890   [
3891    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3892         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3893                        (match_operand:SI 2 "s_register_operand" "r")
3894                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3895          VSHLQ_M_R))
3896   ]
3897   "TARGET_HAVE_MVE"
3898   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3899   [(set_attr "type" "mve_move")
3900    (set_attr "length""8")])
3901
3902 ;;
3903 ;; [vsliq_n_u, vsliq_n_s])
3904 ;;
3905 (define_insn "mve_vsliq_n_<supf><mode>"
3906   [
3907    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3908         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3909                        (match_operand:MVE_2 2 "s_register_operand" "w")
3910                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3911          VSLIQ_N))
3912   ]
3913   "TARGET_HAVE_MVE"
3914   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3915   [(set_attr "type" "mve_move")
3916 ])
3917
3918 ;;
3919 ;; [vsriq_n_u, vsriq_n_s])
3920 ;;
3921 (define_insn "mve_vsriq_n_<supf><mode>"
3922   [
3923    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3924         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3925                        (match_operand:MVE_2 2 "s_register_operand" "w")
3926                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3927          VSRIQ_N))
3928   ]
3929   "TARGET_HAVE_MVE"
3930   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3931   [(set_attr "type" "mve_move")
3932 ])
3933
3934 ;;
3935 ;; [vqdmlsdhxq_s])
3936 ;;
3937 (define_insn "mve_vqdmlsdhxq_s<mode>"
3938   [
3939    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3940         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3941                        (match_operand:MVE_2 2 "s_register_operand" "w")
3942                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3943          VQDMLSDHXQ_S))
3944   ]
3945   "TARGET_HAVE_MVE"
3946   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3947   [(set_attr "type" "mve_move")
3948 ])
3949
3950 ;;
3951 ;; [vqdmlsdhq_s])
3952 ;;
3953 (define_insn "mve_vqdmlsdhq_s<mode>"
3954   [
3955    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3956         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3957                        (match_operand:MVE_2 2 "s_register_operand" "w")
3958                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3959          VQDMLSDHQ_S))
3960   ]
3961   "TARGET_HAVE_MVE"
3962   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3963   [(set_attr "type" "mve_move")
3964 ])
3965
3966 ;;
3967 ;; [vqdmladhxq_s])
3968 ;;
3969 (define_insn "mve_vqdmladhxq_s<mode>"
3970   [
3971    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3972         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3973                        (match_operand:MVE_2 2 "s_register_operand" "w")
3974                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3975          VQDMLADHXQ_S))
3976   ]
3977   "TARGET_HAVE_MVE"
3978   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3979   [(set_attr "type" "mve_move")
3980 ])
3981
3982 ;;
3983 ;; [vqdmladhq_s])
3984 ;;
3985 (define_insn "mve_vqdmladhq_s<mode>"
3986   [
3987    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3988         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3989                        (match_operand:MVE_2 2 "s_register_operand" "w")
3990                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3991          VQDMLADHQ_S))
3992   ]
3993   "TARGET_HAVE_MVE"
3994   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3995   [(set_attr "type" "mve_move")
3996 ])
3997
3998 ;;
3999 ;; [vmlsdavaxq_s])
4000 ;;
4001 (define_insn "mve_vmlsdavaxq_s<mode>"
4002   [
4003    (set (match_operand:SI 0 "s_register_operand" "=Te")
4004         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4005                     (match_operand:MVE_2 2 "s_register_operand" "w")
4006                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4007          VMLSDAVAXQ_S))
4008   ]
4009   "TARGET_HAVE_MVE"
4010   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4011   [(set_attr "type" "mve_move")
4012 ])
4013
4014 ;;
4015 ;; [vmlsdavaq_s])
4016 ;;
4017 (define_insn "mve_vmlsdavaq_s<mode>"
4018   [
4019    (set (match_operand:SI 0 "s_register_operand" "=Te")
4020         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4021                     (match_operand:MVE_2 2 "s_register_operand" "w")
4022                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4023          VMLSDAVAQ_S))
4024   ]
4025   "TARGET_HAVE_MVE"
4026   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4027   [(set_attr "type" "mve_move")
4028 ])
4029
4030 ;;
4031 ;; [vmladavaxq_s])
4032 ;;
4033 (define_insn "mve_vmladavaxq_s<mode>"
4034   [
4035    (set (match_operand:SI 0 "s_register_operand" "=Te")
4036         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4037                     (match_operand:MVE_2 2 "s_register_operand" "w")
4038                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4039          VMLADAVAXQ_S))
4040   ]
4041   "TARGET_HAVE_MVE"
4042   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4043   [(set_attr "type" "mve_move")
4044 ])
4045 ;;
4046 ;; [vabsq_m_f])
4047 ;;
4048 (define_insn "mve_vabsq_m_f<mode>"
4049   [
4050    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4051         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4052                        (match_operand:MVE_0 2 "s_register_operand" "w")
4053                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4054          VABSQ_M_F))
4055   ]
4056   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4057   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4058   [(set_attr "type" "mve_move")
4059    (set_attr "length""8")])
4060
4061 ;;
4062 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4063 ;;
4064 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4065   [
4066    (set (match_operand:DI 0 "s_register_operand" "=r")
4067         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4068                        (match_operand:V4SI 2 "s_register_operand" "w")
4069                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4070          VADDLVAQ_P))
4071   ]
4072   "TARGET_HAVE_MVE"
4073   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4074   [(set_attr "type" "mve_move")
4075    (set_attr "length""8")])
4076 ;;
4077 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
4078 ;;
4079 (define_insn "mve_vcmlaq<mve_rot><mode>"
4080   [
4081    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
4082         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
4083                     (unspec:MVE_0
4084                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
4085                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
4086                      VCMLA)))
4087   ]
4088   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4089   "@
4090    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
4091    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
4092   [(set_attr "type" "mve_move")
4093 ])
4094
4095 ;;
4096 ;; [vcmpeqq_m_n_f])
4097 ;;
4098 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4099   [
4100    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4101         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4102                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4103                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4104          VCMPEQQ_M_N_F))
4105   ]
4106   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4107   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4108   [(set_attr "type" "mve_move")
4109    (set_attr "length""8")])
4110
4111 ;;
4112 ;; [vcmpgeq_m_f])
4113 ;;
4114 (define_insn "mve_vcmpgeq_m_f<mode>"
4115   [
4116    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4117         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4118                        (match_operand:MVE_0 2 "s_register_operand" "w")
4119                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4120          VCMPGEQ_M_F))
4121   ]
4122   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4123   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4124   [(set_attr "type" "mve_move")
4125    (set_attr "length""8")])
4126
4127 ;;
4128 ;; [vcmpgeq_m_n_f])
4129 ;;
4130 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4131   [
4132    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4133         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4134                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4135                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4136          VCMPGEQ_M_N_F))
4137   ]
4138   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4139   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4140   [(set_attr "type" "mve_move")
4141    (set_attr "length""8")])
4142
4143 ;;
4144 ;; [vcmpgtq_m_f])
4145 ;;
4146 (define_insn "mve_vcmpgtq_m_f<mode>"
4147   [
4148    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4149         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4150                        (match_operand:MVE_0 2 "s_register_operand" "w")
4151                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4152          VCMPGTQ_M_F))
4153   ]
4154   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4155   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4156   [(set_attr "type" "mve_move")
4157    (set_attr "length""8")])
4158
4159 ;;
4160 ;; [vcmpgtq_m_n_f])
4161 ;;
4162 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4163   [
4164    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4165         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4166                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4167                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4168          VCMPGTQ_M_N_F))
4169   ]
4170   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4171   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4172   [(set_attr "type" "mve_move")
4173    (set_attr "length""8")])
4174
4175 ;;
4176 ;; [vcmpleq_m_f])
4177 ;;
4178 (define_insn "mve_vcmpleq_m_f<mode>"
4179   [
4180    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4181         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4182                        (match_operand:MVE_0 2 "s_register_operand" "w")
4183                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4184          VCMPLEQ_M_F))
4185   ]
4186   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4187   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4188   [(set_attr "type" "mve_move")
4189    (set_attr "length""8")])
4190
4191 ;;
4192 ;; [vcmpleq_m_n_f])
4193 ;;
4194 (define_insn "mve_vcmpleq_m_n_f<mode>"
4195   [
4196    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4197         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4198                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4199                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4200          VCMPLEQ_M_N_F))
4201   ]
4202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4203   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4204   [(set_attr "type" "mve_move")
4205    (set_attr "length""8")])
4206
4207 ;;
4208 ;; [vcmpltq_m_f])
4209 ;;
4210 (define_insn "mve_vcmpltq_m_f<mode>"
4211   [
4212    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4213         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4214                        (match_operand:MVE_0 2 "s_register_operand" "w")
4215                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4216          VCMPLTQ_M_F))
4217   ]
4218   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4219   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4220   [(set_attr "type" "mve_move")
4221    (set_attr "length""8")])
4222
4223 ;;
4224 ;; [vcmpltq_m_n_f])
4225 ;;
4226 (define_insn "mve_vcmpltq_m_n_f<mode>"
4227   [
4228    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4229         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4230                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4231                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4232          VCMPLTQ_M_N_F))
4233   ]
4234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4235   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4236   [(set_attr "type" "mve_move")
4237    (set_attr "length""8")])
4238
4239 ;;
4240 ;; [vcmpneq_m_f])
4241 ;;
4242 (define_insn "mve_vcmpneq_m_f<mode>"
4243   [
4244    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4245         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4246                        (match_operand:MVE_0 2 "s_register_operand" "w")
4247                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4248          VCMPNEQ_M_F))
4249   ]
4250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4251   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4252   [(set_attr "type" "mve_move")
4253    (set_attr "length""8")])
4254
4255 ;;
4256 ;; [vcmpneq_m_n_f])
4257 ;;
4258 (define_insn "mve_vcmpneq_m_n_f<mode>"
4259   [
4260    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4261         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4262                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4263                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4264          VCMPNEQ_M_N_F))
4265   ]
4266   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4267   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4268   [(set_attr "type" "mve_move")
4269    (set_attr "length""8")])
4270
4271 ;;
4272 ;; [vcvtbq_m_f16_f32])
4273 ;;
4274 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4275   [
4276    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4277         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4278                        (match_operand:V4SF 2 "s_register_operand" "w")
4279                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4280          VCVTBQ_M_F16_F32))
4281   ]
4282   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4283   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4284   [(set_attr "type" "mve_move")
4285    (set_attr "length""8")])
4286
4287 ;;
4288 ;; [vcvtbq_m_f32_f16])
4289 ;;
4290 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4291   [
4292    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4293         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4294                        (match_operand:V8HF 2 "s_register_operand" "w")
4295                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4296          VCVTBQ_M_F32_F16))
4297   ]
4298   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4299   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4300   [(set_attr "type" "mve_move")
4301    (set_attr "length""8")])
4302
4303 ;;
4304 ;; [vcvttq_m_f16_f32])
4305 ;;
4306 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4307   [
4308    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4309         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4310                        (match_operand:V4SF 2 "s_register_operand" "w")
4311                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4312          VCVTTQ_M_F16_F32))
4313   ]
4314   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4315   "vpst\;vcvttt.f16.f32 %q0, %q2"
4316   [(set_attr "type" "mve_move")
4317    (set_attr "length""8")])
4318
4319 ;;
4320 ;; [vcvttq_m_f32_f16])
4321 ;;
4322 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4323   [
4324    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4325         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4326                        (match_operand:V8HF 2 "s_register_operand" "w")
4327                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4328          VCVTTQ_M_F32_F16))
4329   ]
4330   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4331   "vpst\;vcvttt.f32.f16 %q0, %q2"
4332   [(set_attr "type" "mve_move")
4333    (set_attr "length""8")])
4334
4335 ;;
4336 ;; [vdupq_m_n_f])
4337 ;;
4338 (define_insn "mve_vdupq_m_n_f<mode>"
4339   [
4340    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4341         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4342                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4343                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4344          VDUPQ_M_N_F))
4345   ]
4346   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4347   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4348   [(set_attr "type" "mve_move")
4349    (set_attr "length""8")])
4350
4351 ;;
4352 ;; [vfmaq_f])
4353 ;;
4354 (define_insn "mve_vfmaq_f<mode>"
4355   [
4356    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4357         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4358                        (match_operand:MVE_0 2 "s_register_operand" "w")
4359                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4360          VFMAQ_F))
4361   ]
4362   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4363   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4364   [(set_attr "type" "mve_move")
4365 ])
4366
4367 ;;
4368 ;; [vfmaq_n_f])
4369 ;;
4370 (define_insn "mve_vfmaq_n_f<mode>"
4371   [
4372    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4373         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4374                        (match_operand:MVE_0 2 "s_register_operand" "w")
4375                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4376          VFMAQ_N_F))
4377   ]
4378   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4379   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4380   [(set_attr "type" "mve_move")
4381 ])
4382
4383 ;;
4384 ;; [vfmasq_n_f])
4385 ;;
4386 (define_insn "mve_vfmasq_n_f<mode>"
4387   [
4388    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4389         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4390                        (match_operand:MVE_0 2 "s_register_operand" "w")
4391                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4392          VFMASQ_N_F))
4393   ]
4394   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4395   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4396   [(set_attr "type" "mve_move")
4397 ])
4398 ;;
4399 ;; [vfmsq_f])
4400 ;;
4401 (define_insn "mve_vfmsq_f<mode>"
4402   [
4403    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4404         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4405                        (match_operand:MVE_0 2 "s_register_operand" "w")
4406                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4407          VFMSQ_F))
4408   ]
4409   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4410   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4411   [(set_attr "type" "mve_move")
4412 ])
4413
4414 ;;
4415 ;; [vmaxnmaq_m_f])
4416 ;;
4417 (define_insn "mve_vmaxnmaq_m_f<mode>"
4418   [
4419    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4420         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4421                        (match_operand:MVE_0 2 "s_register_operand" "w")
4422                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4423          VMAXNMAQ_M_F))
4424   ]
4425   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4426   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4427   [(set_attr "type" "mve_move")
4428    (set_attr "length""8")])
4429 ;;
4430 ;; [vmaxnmavq_p_f])
4431 ;;
4432 (define_insn "mve_vmaxnmavq_p_f<mode>"
4433   [
4434    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4435         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4436                        (match_operand:MVE_0 2 "s_register_operand" "w")
4437                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4438          VMAXNMAVQ_P_F))
4439   ]
4440   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4441   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4442   [(set_attr "type" "mve_move")
4443    (set_attr "length""8")])
4444
4445 ;;
4446 ;; [vmaxnmvq_p_f])
4447 ;;
4448 (define_insn "mve_vmaxnmvq_p_f<mode>"
4449   [
4450    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4451         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4452                        (match_operand:MVE_0 2 "s_register_operand" "w")
4453                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4454          VMAXNMVQ_P_F))
4455   ]
4456   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4457   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4458   [(set_attr "type" "mve_move")
4459    (set_attr "length""8")])
4460 ;;
4461 ;; [vminnmaq_m_f])
4462 ;;
4463 (define_insn "mve_vminnmaq_m_f<mode>"
4464   [
4465    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4466         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4467                        (match_operand:MVE_0 2 "s_register_operand" "w")
4468                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4469          VMINNMAQ_M_F))
4470   ]
4471   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4472   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4473   [(set_attr "type" "mve_move")
4474    (set_attr "length""8")])
4475
4476 ;;
4477 ;; [vminnmavq_p_f])
4478 ;;
4479 (define_insn "mve_vminnmavq_p_f<mode>"
4480   [
4481    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4482         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4483                        (match_operand:MVE_0 2 "s_register_operand" "w")
4484                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4485          VMINNMAVQ_P_F))
4486   ]
4487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4488   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4489   [(set_attr "type" "mve_move")
4490    (set_attr "length""8")])
4491 ;;
4492 ;; [vminnmvq_p_f])
4493 ;;
4494 (define_insn "mve_vminnmvq_p_f<mode>"
4495   [
4496    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4497         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4498                        (match_operand:MVE_0 2 "s_register_operand" "w")
4499                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4500          VMINNMVQ_P_F))
4501   ]
4502   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4503   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4504   [(set_attr "type" "mve_move")
4505    (set_attr "length""8")])
4506
4507 ;;
4508 ;; [vmlaldavaq_s, vmlaldavaq_u])
4509 ;;
4510 (define_insn "mve_vmlaldavaq_<supf><mode>"
4511   [
4512    (set (match_operand:DI 0 "s_register_operand" "=r")
4513         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4514                        (match_operand:MVE_5 2 "s_register_operand" "w")
4515                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4516          VMLALDAVAQ))
4517   ]
4518   "TARGET_HAVE_MVE"
4519   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4520   [(set_attr "type" "mve_move")
4521 ])
4522
4523 ;;
4524 ;; [vmlaldavaxq_s])
4525 ;;
4526 (define_insn "mve_vmlaldavaxq_s<mode>"
4527   [
4528    (set (match_operand:DI 0 "s_register_operand" "=r")
4529         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4530                        (match_operand:MVE_5 2 "s_register_operand" "w")
4531                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4532          VMLALDAVAXQ_S))
4533   ]
4534   "TARGET_HAVE_MVE"
4535   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4536   [(set_attr "type" "mve_move")
4537 ])
4538
4539 ;;
4540 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4541 ;;
4542 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4543   [
4544    (set (match_operand:DI 0 "s_register_operand" "=r")
4545         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4546                        (match_operand:MVE_5 2 "s_register_operand" "w")
4547                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4548          VMLALDAVQ_P))
4549   ]
4550   "TARGET_HAVE_MVE"
4551   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4552   [(set_attr "type" "mve_move")
4553    (set_attr "length""8")])
4554
4555 ;;
4556 ;; [vmlaldavxq_p_s])
4557 ;;
4558 (define_insn "mve_vmlaldavxq_p_s<mode>"
4559   [
4560    (set (match_operand:DI 0 "s_register_operand" "=r")
4561         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4562                        (match_operand:MVE_5 2 "s_register_operand" "w")
4563                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4564          VMLALDAVXQ_P_S))
4565   ]
4566   "TARGET_HAVE_MVE"
4567   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4568   [(set_attr "type" "mve_move")
4569    (set_attr "length""8")])
4570 ;;
4571 ;; [vmlsldavaq_s])
4572 ;;
4573 (define_insn "mve_vmlsldavaq_s<mode>"
4574   [
4575    (set (match_operand:DI 0 "s_register_operand" "=r")
4576         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4577                        (match_operand:MVE_5 2 "s_register_operand" "w")
4578                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4579          VMLSLDAVAQ_S))
4580   ]
4581   "TARGET_HAVE_MVE"
4582   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4583   [(set_attr "type" "mve_move")
4584 ])
4585
4586 ;;
4587 ;; [vmlsldavaxq_s])
4588 ;;
4589 (define_insn "mve_vmlsldavaxq_s<mode>"
4590   [
4591    (set (match_operand:DI 0 "s_register_operand" "=r")
4592         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4593                        (match_operand:MVE_5 2 "s_register_operand" "w")
4594                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4595          VMLSLDAVAXQ_S))
4596   ]
4597   "TARGET_HAVE_MVE"
4598   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4599   [(set_attr "type" "mve_move")
4600 ])
4601
4602 ;;
4603 ;; [vmlsldavq_p_s])
4604 ;;
4605 (define_insn "mve_vmlsldavq_p_s<mode>"
4606   [
4607    (set (match_operand:DI 0 "s_register_operand" "=r")
4608         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4609                        (match_operand:MVE_5 2 "s_register_operand" "w")
4610                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4611          VMLSLDAVQ_P_S))
4612   ]
4613   "TARGET_HAVE_MVE"
4614   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4615   [(set_attr "type" "mve_move")
4616    (set_attr "length""8")])
4617
4618 ;;
4619 ;; [vmlsldavxq_p_s])
4620 ;;
4621 (define_insn "mve_vmlsldavxq_p_s<mode>"
4622   [
4623    (set (match_operand:DI 0 "s_register_operand" "=r")
4624         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4625                        (match_operand:MVE_5 2 "s_register_operand" "w")
4626                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4627          VMLSLDAVXQ_P_S))
4628   ]
4629   "TARGET_HAVE_MVE"
4630   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4631   [(set_attr "type" "mve_move")
4632    (set_attr "length""8")])
4633 ;;
4634 ;; [vmovlbq_m_u, vmovlbq_m_s])
4635 ;;
4636 (define_insn "mve_vmovlbq_m_<supf><mode>"
4637   [
4638    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4639         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4640                        (match_operand:MVE_3 2 "s_register_operand" "w")
4641                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4642          VMOVLBQ_M))
4643   ]
4644   "TARGET_HAVE_MVE"
4645   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4646   [(set_attr "type" "mve_move")
4647    (set_attr "length""8")])
4648 ;;
4649 ;; [vmovltq_m_u, vmovltq_m_s])
4650 ;;
4651 (define_insn "mve_vmovltq_m_<supf><mode>"
4652   [
4653    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4654         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4655                        (match_operand:MVE_3 2 "s_register_operand" "w")
4656                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4657          VMOVLTQ_M))
4658   ]
4659   "TARGET_HAVE_MVE"
4660   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4661   [(set_attr "type" "mve_move")
4662    (set_attr "length""8")])
4663 ;;
4664 ;; [vmovnbq_m_u, vmovnbq_m_s])
4665 ;;
4666 (define_insn "mve_vmovnbq_m_<supf><mode>"
4667   [
4668    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4669         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4670                        (match_operand:MVE_5 2 "s_register_operand" "w")
4671                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4672          VMOVNBQ_M))
4673   ]
4674   "TARGET_HAVE_MVE"
4675   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4676   [(set_attr "type" "mve_move")
4677    (set_attr "length""8")])
4678
4679 ;;
4680 ;; [vmovntq_m_u, vmovntq_m_s])
4681 ;;
4682 (define_insn "mve_vmovntq_m_<supf><mode>"
4683   [
4684    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4685         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4686                        (match_operand:MVE_5 2 "s_register_operand" "w")
4687                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4688          VMOVNTQ_M))
4689   ]
4690   "TARGET_HAVE_MVE"
4691   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4692   [(set_attr "type" "mve_move")
4693    (set_attr "length""8")])
4694
4695 ;;
4696 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4697 ;;
4698 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4699   [
4700    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4701         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4702                        (match_operand:SI 2 "immediate_operand" "i")
4703                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4704          VMVNQ_M_N))
4705   ]
4706   "TARGET_HAVE_MVE"
4707   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4708   [(set_attr "type" "mve_move")
4709    (set_attr "length""8")])
4710 ;;
4711 ;; [vnegq_m_f])
4712 ;;
4713 (define_insn "mve_vnegq_m_f<mode>"
4714   [
4715    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4716         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4717                        (match_operand:MVE_0 2 "s_register_operand" "w")
4718                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4719          VNEGQ_M_F))
4720   ]
4721   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4722   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4723   [(set_attr "type" "mve_move")
4724    (set_attr "length""8")])
4725
4726 ;;
4727 ;; [vorrq_m_n_s, vorrq_m_n_u])
4728 ;;
4729 (define_insn "mve_vorrq_m_n_<supf><mode>"
4730   [
4731    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4732         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4733                        (match_operand:SI 2 "immediate_operand" "i")
4734                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4735          VORRQ_M_N))
4736   ]
4737   "TARGET_HAVE_MVE"
4738   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4739   [(set_attr "type" "mve_move")
4740    (set_attr "length""8")])
4741 ;;
4742 ;; [vpselq_f])
4743 ;;
4744 (define_insn "mve_vpselq_f<mode>"
4745   [
4746    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4747         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4748                        (match_operand:MVE_0 2 "s_register_operand" "w")
4749                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4750          VPSELQ_F))
4751   ]
4752   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4753   "vpsel %q0, %q1, %q2"
4754   [(set_attr "type" "mve_move")
4755 ])
4756
4757 ;;
4758 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4759 ;;
4760 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4761   [
4762    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4763         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4764                        (match_operand:MVE_5 2 "s_register_operand" "w")
4765                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4766          VQMOVNBQ_M))
4767   ]
4768   "TARGET_HAVE_MVE"
4769   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4770   [(set_attr "type" "mve_move")
4771    (set_attr "length""8")])
4772
4773 ;;
4774 ;; [vqmovntq_m_u, vqmovntq_m_s])
4775 ;;
4776 (define_insn "mve_vqmovntq_m_<supf><mode>"
4777   [
4778    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4779         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4780                        (match_operand:MVE_5 2 "s_register_operand" "w")
4781                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4782          VQMOVNTQ_M))
4783   ]
4784   "TARGET_HAVE_MVE"
4785   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4786   [(set_attr "type" "mve_move")
4787    (set_attr "length""8")])
4788
4789 ;;
4790 ;; [vqmovunbq_m_s])
4791 ;;
4792 (define_insn "mve_vqmovunbq_m_s<mode>"
4793   [
4794    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4795         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4796                        (match_operand:MVE_5 2 "s_register_operand" "w")
4797                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4798          VQMOVUNBQ_M_S))
4799   ]
4800   "TARGET_HAVE_MVE"
4801   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4802   [(set_attr "type" "mve_move")
4803    (set_attr "length""8")])
4804
4805 ;;
4806 ;; [vqmovuntq_m_s])
4807 ;;
4808 (define_insn "mve_vqmovuntq_m_s<mode>"
4809   [
4810    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4811         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4812                        (match_operand:MVE_5 2 "s_register_operand" "w")
4813                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4814          VQMOVUNTQ_M_S))
4815   ]
4816   "TARGET_HAVE_MVE"
4817   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4818   [(set_attr "type" "mve_move")
4819    (set_attr "length""8")])
4820
4821 ;;
4822 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4823 ;;
4824 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4825   [
4826    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4827         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4828                        (match_operand:MVE_5 2 "s_register_operand" "w")
4829                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4830          VQRSHRNTQ_N))
4831   ]
4832   "TARGET_HAVE_MVE"
4833   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4834   [(set_attr "type" "mve_move")
4835 ])
4836
4837 ;;
4838 ;; [vqrshruntq_n_s])
4839 ;;
4840 (define_insn "mve_vqrshruntq_n_s<mode>"
4841   [
4842    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4843         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4844                        (match_operand:MVE_5 2 "s_register_operand" "w")
4845                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4846          VQRSHRUNTQ_N_S))
4847   ]
4848   "TARGET_HAVE_MVE"
4849   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4850   [(set_attr "type" "mve_move")
4851 ])
4852
4853 ;;
4854 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4855 ;;
4856 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4857   [
4858    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4859         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4860                        (match_operand:MVE_5 2 "s_register_operand" "w")
4861                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4862          VQSHRNBQ_N))
4863   ]
4864   "TARGET_HAVE_MVE"
4865   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4866   [(set_attr "type" "mve_move")
4867 ])
4868
4869 ;;
4870 ;; [vqshrntq_n_u, vqshrntq_n_s])
4871 ;;
4872 (define_insn "mve_vqshrntq_n_<supf><mode>"
4873   [
4874    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4875         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4876                        (match_operand:MVE_5 2 "s_register_operand" "w")
4877                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4878          VQSHRNTQ_N))
4879   ]
4880   "TARGET_HAVE_MVE"
4881   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4882   [(set_attr "type" "mve_move")
4883 ])
4884
4885 ;;
4886 ;; [vqshrunbq_n_s])
4887 ;;
4888 (define_insn "mve_vqshrunbq_n_s<mode>"
4889   [
4890    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4891         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4892                        (match_operand:MVE_5 2 "s_register_operand" "w")
4893                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4894          VQSHRUNBQ_N_S))
4895   ]
4896   "TARGET_HAVE_MVE"
4897   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4898   [(set_attr "type" "mve_move")
4899 ])
4900
4901 ;;
4902 ;; [vqshruntq_n_s])
4903 ;;
4904 (define_insn "mve_vqshruntq_n_s<mode>"
4905   [
4906    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4907         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4908                        (match_operand:MVE_5 2 "s_register_operand" "w")
4909                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4910          VQSHRUNTQ_N_S))
4911   ]
4912   "TARGET_HAVE_MVE"
4913   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4914   [(set_attr "type" "mve_move")
4915 ])
4916
4917 ;;
4918 ;; [vrev32q_m_f])
4919 ;;
4920 (define_insn "mve_vrev32q_m_fv8hf"
4921   [
4922    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4923         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4924                        (match_operand:V8HF 2 "s_register_operand" "w")
4925                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4926          VREV32Q_M_F))
4927   ]
4928   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4929   "vpst\;vrev32t.16 %q0, %q2"
4930   [(set_attr "type" "mve_move")
4931    (set_attr "length""8")])
4932
4933 ;;
4934 ;; [vrev32q_m_s, vrev32q_m_u])
4935 ;;
4936 (define_insn "mve_vrev32q_m_<supf><mode>"
4937   [
4938    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4939         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4940                        (match_operand:MVE_3 2 "s_register_operand" "w")
4941                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4942          VREV32Q_M))
4943   ]
4944   "TARGET_HAVE_MVE"
4945   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4946   [(set_attr "type" "mve_move")
4947    (set_attr "length""8")])
4948
4949 ;;
4950 ;; [vrev64q_m_f])
4951 ;;
4952 (define_insn "mve_vrev64q_m_f<mode>"
4953   [
4954    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4955         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4956                        (match_operand:MVE_0 2 "s_register_operand" "w")
4957                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4958          VREV64Q_M_F))
4959   ]
4960   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4961   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4962   [(set_attr "type" "mve_move")
4963    (set_attr "length""8")])
4964
4965 ;;
4966 ;; [vrmlaldavhaxq_s])
4967 ;;
4968 (define_insn "mve_vrmlaldavhaxq_sv4si"
4969   [
4970    (set (match_operand:DI 0 "s_register_operand" "=r")
4971         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4972                        (match_operand:V4SI 2 "s_register_operand" "w")
4973                        (match_operand:V4SI 3 "s_register_operand" "w")]
4974          VRMLALDAVHAXQ_S))
4975   ]
4976   "TARGET_HAVE_MVE"
4977   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
4978   [(set_attr "type" "mve_move")
4979 ])
4980
4981 ;;
4982 ;; [vrmlaldavhxq_p_s])
4983 ;;
4984 (define_insn "mve_vrmlaldavhxq_p_sv4si"
4985   [
4986    (set (match_operand:DI 0 "s_register_operand" "=r")
4987         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4988                        (match_operand:V4SI 2 "s_register_operand" "w")
4989                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4990          VRMLALDAVHXQ_P_S))
4991   ]
4992   "TARGET_HAVE_MVE"
4993   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
4994   [(set_attr "type" "mve_move")
4995    (set_attr "length""8")])
4996
4997 ;;
4998 ;; [vrmlsldavhaxq_s])
4999 ;;
5000 (define_insn "mve_vrmlsldavhaxq_sv4si"
5001   [
5002    (set (match_operand:DI 0 "s_register_operand" "=r")
5003         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5004                        (match_operand:V4SI 2 "s_register_operand" "w")
5005                        (match_operand:V4SI 3 "s_register_operand" "w")]
5006          VRMLSLDAVHAXQ_S))
5007   ]
5008   "TARGET_HAVE_MVE"
5009   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5010   [(set_attr "type" "mve_move")
5011 ])
5012
5013 ;;
5014 ;; [vrmlsldavhq_p_s])
5015 ;;
5016 (define_insn "mve_vrmlsldavhq_p_sv4si"
5017   [
5018    (set (match_operand:DI 0 "s_register_operand" "=r")
5019         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5020                        (match_operand:V4SI 2 "s_register_operand" "w")
5021                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5022          VRMLSLDAVHQ_P_S))
5023   ]
5024   "TARGET_HAVE_MVE"
5025   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5026   [(set_attr "type" "mve_move")
5027    (set_attr "length""8")])
5028
5029 ;;
5030 ;; [vrmlsldavhxq_p_s])
5031 ;;
5032 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5033   [
5034    (set (match_operand:DI 0 "s_register_operand" "=r")
5035         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5036                        (match_operand:V4SI 2 "s_register_operand" "w")
5037                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5038          VRMLSLDAVHXQ_P_S))
5039   ]
5040   "TARGET_HAVE_MVE"
5041   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5042   [(set_attr "type" "mve_move")
5043    (set_attr "length""8")])
5044
5045 ;;
5046 ;; [vrndaq_m_f])
5047 ;;
5048 (define_insn "mve_vrndaq_m_f<mode>"
5049   [
5050    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5051         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5052                        (match_operand:MVE_0 2 "s_register_operand" "w")
5053                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5054          VRNDAQ_M_F))
5055   ]
5056   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5057   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5058   [(set_attr "type" "mve_move")
5059    (set_attr "length""8")])
5060
5061 ;;
5062 ;; [vrndmq_m_f])
5063 ;;
5064 (define_insn "mve_vrndmq_m_f<mode>"
5065   [
5066    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5067         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5068                        (match_operand:MVE_0 2 "s_register_operand" "w")
5069                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5070          VRNDMQ_M_F))
5071   ]
5072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5073   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5074   [(set_attr "type" "mve_move")
5075    (set_attr "length""8")])
5076
5077 ;;
5078 ;; [vrndnq_m_f])
5079 ;;
5080 (define_insn "mve_vrndnq_m_f<mode>"
5081   [
5082    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5083         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5084                        (match_operand:MVE_0 2 "s_register_operand" "w")
5085                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5086          VRNDNQ_M_F))
5087   ]
5088   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5089   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5090   [(set_attr "type" "mve_move")
5091    (set_attr "length""8")])
5092
5093 ;;
5094 ;; [vrndpq_m_f])
5095 ;;
5096 (define_insn "mve_vrndpq_m_f<mode>"
5097   [
5098    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5099         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5100                        (match_operand:MVE_0 2 "s_register_operand" "w")
5101                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5102          VRNDPQ_M_F))
5103   ]
5104   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5105   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5106   [(set_attr "type" "mve_move")
5107    (set_attr "length""8")])
5108
5109 ;;
5110 ;; [vrndxq_m_f])
5111 ;;
5112 (define_insn "mve_vrndxq_m_f<mode>"
5113   [
5114    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5115         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5116                        (match_operand:MVE_0 2 "s_register_operand" "w")
5117                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5118          VRNDXQ_M_F))
5119   ]
5120   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5121   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5122   [(set_attr "type" "mve_move")
5123    (set_attr "length""8")])
5124
5125 ;;
5126 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5127 ;;
5128 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5129   [
5130    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5131         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5132                        (match_operand:MVE_5 2 "s_register_operand" "w")
5133                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5134          VRSHRNBQ_N))
5135   ]
5136   "TARGET_HAVE_MVE"
5137   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5138   [(set_attr "type" "mve_move")
5139 ])
5140
5141 ;;
5142 ;; [vrshrntq_n_u, vrshrntq_n_s])
5143 ;;
5144 (define_insn "mve_vrshrntq_n_<supf><mode>"
5145   [
5146    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5147         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5148                        (match_operand:MVE_5 2 "s_register_operand" "w")
5149                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5150          VRSHRNTQ_N))
5151   ]
5152   "TARGET_HAVE_MVE"
5153   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5154   [(set_attr "type" "mve_move")
5155 ])
5156
5157 ;;
5158 ;; [vshrnbq_n_u, vshrnbq_n_s])
5159 ;;
5160 (define_insn "mve_vshrnbq_n_<supf><mode>"
5161   [
5162    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5163         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5164                        (match_operand:MVE_5 2 "s_register_operand" "w")
5165                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5166          VSHRNBQ_N))
5167   ]
5168   "TARGET_HAVE_MVE"
5169   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5170   [(set_attr "type" "mve_move")
5171 ])
5172
5173 ;;
5174 ;; [vshrntq_n_s, vshrntq_n_u])
5175 ;;
5176 (define_insn "mve_vshrntq_n_<supf><mode>"
5177   [
5178    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5179         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5180                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5181                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5182          VSHRNTQ_N))
5183   ]
5184   "TARGET_HAVE_MVE"
5185   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5186   [(set_attr "type" "mve_move")
5187 ])
5188
5189 ;;
5190 ;; [vcvtmq_m_s, vcvtmq_m_u])
5191 ;;
5192 (define_insn "mve_vcvtmq_m_<supf><mode>"
5193   [
5194    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5195         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5196                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5197                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5198          VCVTMQ_M))
5199   ]
5200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5201   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5202   [(set_attr "type" "mve_move")
5203    (set_attr "length""8")])
5204
5205 ;;
5206 ;; [vcvtpq_m_u, vcvtpq_m_s])
5207 ;;
5208 (define_insn "mve_vcvtpq_m_<supf><mode>"
5209   [
5210    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5211         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5212                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5213                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5214          VCVTPQ_M))
5215   ]
5216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5217   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5218   [(set_attr "type" "mve_move")
5219    (set_attr "length""8")])
5220
5221 ;;
5222 ;; [vcvtnq_m_s, vcvtnq_m_u])
5223 ;;
5224 (define_insn "mve_vcvtnq_m_<supf><mode>"
5225   [
5226    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5227         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5228                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5229                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5230          VCVTNQ_M))
5231   ]
5232   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5233   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5234   [(set_attr "type" "mve_move")
5235    (set_attr "length""8")])
5236
5237 ;;
5238 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5239 ;;
5240 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5241   [
5242    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5243         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5244                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5245                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5246                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5247          VCVTQ_M_N_FROM_F))
5248   ]
5249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5250   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5251   [(set_attr "type" "mve_move")
5252    (set_attr "length""8")])
5253
5254 ;;
5255 ;; [vrev16q_m_u, vrev16q_m_s])
5256 ;;
5257 (define_insn "mve_vrev16q_m_<supf>v16qi"
5258   [
5259    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5260         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5261                        (match_operand:V16QI 2 "s_register_operand" "w")
5262                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5263          VREV16Q_M))
5264   ]
5265   "TARGET_HAVE_MVE"
5266   "vpst\;vrev16t.8 %q0, %q2"
5267   [(set_attr "type" "mve_move")
5268    (set_attr "length""8")])
5269
5270 ;;
5271 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5272 ;;
5273 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5274   [
5275    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5276         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5277                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5278                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5279          VCVTQ_M_FROM_F))
5280   ]
5281   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5282   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5283   [(set_attr "type" "mve_move")
5284    (set_attr "length""8")])
5285
5286 ;;
5287 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5288 ;;
5289 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5290   [
5291    (set (match_operand:DI 0 "s_register_operand" "=r")
5292         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5293                     (match_operand:V4SI 2 "s_register_operand" "w")
5294                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5295          VRMLALDAVHQ_P))
5296   ]
5297   "TARGET_HAVE_MVE"
5298   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5299   [(set_attr "type" "mve_move")
5300    (set_attr "length""8")])
5301
5302 ;;
5303 ;; [vrmlsldavhaq_s])
5304 ;;
5305 (define_insn "mve_vrmlsldavhaq_sv4si"
5306   [
5307    (set (match_operand:DI 0 "s_register_operand" "=r")
5308         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5309                     (match_operand:V4SI 2 "s_register_operand" "w")
5310                     (match_operand:V4SI 3 "s_register_operand" "w")]
5311          VRMLSLDAVHAQ_S))
5312   ]
5313   "TARGET_HAVE_MVE"
5314   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5315   [(set_attr "type" "mve_move")
5316 ])
5317
5318 ;;
5319 ;; [vabavq_p_s, vabavq_p_u])
5320 ;;
5321 (define_insn "mve_vabavq_p_<supf><mode>"
5322   [
5323    (set (match_operand:SI 0 "s_register_operand" "=r")
5324         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5325                     (match_operand:MVE_2 2 "s_register_operand" "w")
5326                     (match_operand:MVE_2 3 "s_register_operand" "w")
5327                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5328          VABAVQ_P))
5329   ]
5330   "TARGET_HAVE_MVE"
5331   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5332   [(set_attr "type" "mve_move")
5333 ])
5334
5335 ;;
5336 ;; [vqshluq_m_n_s])
5337 ;;
5338 (define_insn "mve_vqshluq_m_n_s<mode>"
5339   [
5340    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5341         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5342                        (match_operand:MVE_2 2 "s_register_operand" "w")
5343                        (match_operand:SI 3 "mve_imm_7" "Ra")
5344                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5345          VQSHLUQ_M_N_S))
5346   ]
5347   "TARGET_HAVE_MVE"
5348   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5349   [(set_attr "type" "mve_move")])
5350
5351 ;;
5352 ;; [vshlq_m_s, vshlq_m_u])
5353 ;;
5354 (define_insn "mve_vshlq_m_<supf><mode>"
5355   [
5356    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5357         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5358                        (match_operand:MVE_2 2 "s_register_operand" "w")
5359                        (match_operand:MVE_2 3 "s_register_operand" "w")
5360                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5361          VSHLQ_M))
5362   ]
5363   "TARGET_HAVE_MVE"
5364   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5365   [(set_attr "type" "mve_move")])
5366
5367 ;;
5368 ;; [vsriq_m_n_s, vsriq_m_n_u])
5369 ;;
5370 (define_insn "mve_vsriq_m_n_<supf><mode>"
5371   [
5372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5374                        (match_operand:MVE_2 2 "s_register_operand" "w")
5375                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5376                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5377          VSRIQ_M_N))
5378   ]
5379   "TARGET_HAVE_MVE"
5380   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5381   [(set_attr "type" "mve_move")])
5382
5383 ;;
5384 ;; [vsubq_m_u, vsubq_m_s])
5385 ;;
5386 (define_insn "mve_vsubq_m_<supf><mode>"
5387   [
5388    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5389         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5390                        (match_operand:MVE_2 2 "s_register_operand" "w")
5391                        (match_operand:MVE_2 3 "s_register_operand" "w")
5392                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5393          VSUBQ_M))
5394   ]
5395   "TARGET_HAVE_MVE"
5396   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5397   [(set_attr "type" "mve_move")])
5398
5399 ;;
5400 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5401 ;;
5402 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5403   [
5404    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5405         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5406                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5407                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5408                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5409          VCVTQ_M_N_TO_F))
5410   ]
5411   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5412   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5413   [(set_attr "type" "mve_move")
5414    (set_attr "length""8")])
5415 ;;
5416 ;; [vabdq_m_s, vabdq_m_u])
5417 ;;
5418 (define_insn "mve_vabdq_m_<supf><mode>"
5419   [
5420    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5421         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5422                        (match_operand:MVE_2 2 "s_register_operand" "w")
5423                        (match_operand:MVE_2 3 "s_register_operand" "w")
5424                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5425          VABDQ_M))
5426   ]
5427   "TARGET_HAVE_MVE"
5428   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5429   [(set_attr "type" "mve_move")
5430    (set_attr "length""8")])
5431
5432 ;;
5433 ;; [vaddq_m_n_s, vaddq_m_n_u])
5434 ;;
5435 (define_insn "mve_vaddq_m_n_<supf><mode>"
5436   [
5437    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5438         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5439                        (match_operand:MVE_2 2 "s_register_operand" "w")
5440                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5441                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5442          VADDQ_M_N))
5443   ]
5444   "TARGET_HAVE_MVE"
5445   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5446   [(set_attr "type" "mve_move")
5447    (set_attr "length""8")])
5448
5449 ;;
5450 ;; [vaddq_m_u, vaddq_m_s])
5451 ;;
5452 (define_insn "mve_vaddq_m_<supf><mode>"
5453   [
5454    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5455         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5456                        (match_operand:MVE_2 2 "s_register_operand" "w")
5457                        (match_operand:MVE_2 3 "s_register_operand" "w")
5458                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5459          VADDQ_M))
5460   ]
5461   "TARGET_HAVE_MVE"
5462   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5463   [(set_attr "type" "mve_move")
5464    (set_attr "length""8")])
5465
5466 ;;
5467 ;; [vandq_m_u, vandq_m_s])
5468 ;;
5469 (define_insn "mve_vandq_m_<supf><mode>"
5470   [
5471    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5472         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5473                        (match_operand:MVE_2 2 "s_register_operand" "w")
5474                        (match_operand:MVE_2 3 "s_register_operand" "w")
5475                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5476          VANDQ_M))
5477   ]
5478   "TARGET_HAVE_MVE"
5479   "vpst\;vandt %q0, %q2, %q3"
5480   [(set_attr "type" "mve_move")
5481    (set_attr "length""8")])
5482
5483 ;;
5484 ;; [vbicq_m_u, vbicq_m_s])
5485 ;;
5486 (define_insn "mve_vbicq_m_<supf><mode>"
5487   [
5488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5489         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5490                        (match_operand:MVE_2 2 "s_register_operand" "w")
5491                        (match_operand:MVE_2 3 "s_register_operand" "w")
5492                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5493          VBICQ_M))
5494   ]
5495   "TARGET_HAVE_MVE"
5496   "vpst\;vbict %q0, %q2, %q3"
5497   [(set_attr "type" "mve_move")
5498    (set_attr "length""8")])
5499
5500 ;;
5501 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5502 ;;
5503 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5504   [
5505    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5506         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5507                        (match_operand:MVE_2 2 "s_register_operand" "w")
5508                        (match_operand:SI 3 "s_register_operand" "r")
5509                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5510          VBRSRQ_M_N))
5511   ]
5512   "TARGET_HAVE_MVE"
5513   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5514   [(set_attr "type" "mve_move")
5515    (set_attr "length""8")])
5516
5517 ;;
5518 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5519 ;;
5520 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5521   [
5522    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5523         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5524                        (match_operand:MVE_2 2 "s_register_operand" "w")
5525                        (match_operand:MVE_2 3 "s_register_operand" "w")
5526                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5527          VCADDQ_ROT270_M))
5528   ]
5529   "TARGET_HAVE_MVE"
5530   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5531   [(set_attr "type" "mve_move")
5532    (set_attr "length""8")])
5533
5534 ;;
5535 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5536 ;;
5537 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5538   [
5539    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5540         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5541                        (match_operand:MVE_2 2 "s_register_operand" "w")
5542                        (match_operand:MVE_2 3 "s_register_operand" "w")
5543                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5544          VCADDQ_ROT90_M))
5545   ]
5546   "TARGET_HAVE_MVE"
5547   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5548   [(set_attr "type" "mve_move")
5549    (set_attr "length""8")])
5550
5551 ;;
5552 ;; [veorq_m_s, veorq_m_u])
5553 ;;
5554 (define_insn "mve_veorq_m_<supf><mode>"
5555   [
5556    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5557         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5558                        (match_operand:MVE_2 2 "s_register_operand" "w")
5559                        (match_operand:MVE_2 3 "s_register_operand" "w")
5560                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5561          VEORQ_M))
5562   ]
5563   "TARGET_HAVE_MVE"
5564   "vpst\;veort %q0, %q2, %q3"
5565   [(set_attr "type" "mve_move")
5566    (set_attr "length""8")])
5567
5568 ;;
5569 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5570 ;;
5571 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5572   [
5573    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5574         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5575                        (match_operand:MVE_2 2 "s_register_operand" "w")
5576                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5577                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5578          VHADDQ_M_N))
5579   ]
5580   "TARGET_HAVE_MVE"
5581   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5582   [(set_attr "type" "mve_move")
5583    (set_attr "length""8")])
5584
5585 ;;
5586 ;; [vhaddq_m_s, vhaddq_m_u])
5587 ;;
5588 (define_insn "mve_vhaddq_m_<supf><mode>"
5589   [
5590    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5591         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5592                        (match_operand:MVE_2 2 "s_register_operand" "w")
5593                        (match_operand:MVE_2 3 "s_register_operand" "w")
5594                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5595          VHADDQ_M))
5596   ]
5597   "TARGET_HAVE_MVE"
5598   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5599   [(set_attr "type" "mve_move")
5600    (set_attr "length""8")])
5601
5602 ;;
5603 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5604 ;;
5605 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5606   [
5607    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5608         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5609                        (match_operand:MVE_2 2 "s_register_operand" "w")
5610                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5611                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5612          VHSUBQ_M_N))
5613   ]
5614   "TARGET_HAVE_MVE"
5615   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5616   [(set_attr "type" "mve_move")
5617    (set_attr "length""8")])
5618
5619 ;;
5620 ;; [vhsubq_m_s, vhsubq_m_u])
5621 ;;
5622 (define_insn "mve_vhsubq_m_<supf><mode>"
5623   [
5624    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5625         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5626                        (match_operand:MVE_2 2 "s_register_operand" "w")
5627                        (match_operand:MVE_2 3 "s_register_operand" "w")
5628                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5629          VHSUBQ_M))
5630   ]
5631   "TARGET_HAVE_MVE"
5632   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5633   [(set_attr "type" "mve_move")
5634    (set_attr "length""8")])
5635
5636 ;;
5637 ;; [vmaxq_m_s, vmaxq_m_u])
5638 ;;
5639 (define_insn "mve_vmaxq_m_<supf><mode>"
5640   [
5641    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5642         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5643                        (match_operand:MVE_2 2 "s_register_operand" "w")
5644                        (match_operand:MVE_2 3 "s_register_operand" "w")
5645                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5646          VMAXQ_M))
5647   ]
5648   "TARGET_HAVE_MVE"
5649   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5650   [(set_attr "type" "mve_move")
5651    (set_attr "length""8")])
5652
5653 ;;
5654 ;; [vminq_m_s, vminq_m_u])
5655 ;;
5656 (define_insn "mve_vminq_m_<supf><mode>"
5657   [
5658    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5659         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5660                        (match_operand:MVE_2 2 "s_register_operand" "w")
5661                        (match_operand:MVE_2 3 "s_register_operand" "w")
5662                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5663          VMINQ_M))
5664   ]
5665   "TARGET_HAVE_MVE"
5666   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5667   [(set_attr "type" "mve_move")
5668    (set_attr "length""8")])
5669
5670 ;;
5671 ;; [vmladavaq_p_u, vmladavaq_p_s])
5672 ;;
5673 (define_insn "mve_vmladavaq_p_<supf><mode>"
5674   [
5675    (set (match_operand:SI 0 "s_register_operand" "=Te")
5676         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5677                     (match_operand:MVE_2 2 "s_register_operand" "w")
5678                     (match_operand:MVE_2 3 "s_register_operand" "w")
5679                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5680          VMLADAVAQ_P))
5681   ]
5682   "TARGET_HAVE_MVE"
5683   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5684   [(set_attr "type" "mve_move")
5685    (set_attr "length""8")])
5686
5687 ;;
5688 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5689 ;;
5690 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5691   [
5692    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5693         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5694                        (match_operand:MVE_2 2 "s_register_operand" "w")
5695                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5696                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5697          VMLAQ_M_N))
5698   ]
5699   "TARGET_HAVE_MVE"
5700   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5701   [(set_attr "type" "mve_move")
5702    (set_attr "length""8")])
5703
5704 ;;
5705 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5706 ;;
5707 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5708   [
5709    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5710         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5711                        (match_operand:MVE_2 2 "s_register_operand" "w")
5712                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5713                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5714          VMLASQ_M_N))
5715   ]
5716   "TARGET_HAVE_MVE"
5717   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5718   [(set_attr "type" "mve_move")
5719    (set_attr "length""8")])
5720
5721 ;;
5722 ;; [vmulhq_m_s, vmulhq_m_u])
5723 ;;
5724 (define_insn "mve_vmulhq_m_<supf><mode>"
5725   [
5726    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5727         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5728                        (match_operand:MVE_2 2 "s_register_operand" "w")
5729                        (match_operand:MVE_2 3 "s_register_operand" "w")
5730                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5731          VMULHQ_M))
5732   ]
5733   "TARGET_HAVE_MVE"
5734   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5735   [(set_attr "type" "mve_move")
5736    (set_attr "length""8")])
5737
5738 ;;
5739 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5740 ;;
5741 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5742   [
5743    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5744         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5745                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5746                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5747                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5748          VMULLBQ_INT_M))
5749   ]
5750   "TARGET_HAVE_MVE"
5751   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5752   [(set_attr "type" "mve_move")
5753    (set_attr "length""8")])
5754
5755 ;;
5756 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5757 ;;
5758 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5759   [
5760    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5761         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5762                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5763                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5764                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5765          VMULLTQ_INT_M))
5766   ]
5767   "TARGET_HAVE_MVE"
5768   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5769   [(set_attr "type" "mve_move")
5770    (set_attr "length""8")])
5771
5772 ;;
5773 ;; [vmulq_m_n_u, vmulq_m_n_s])
5774 ;;
5775 (define_insn "mve_vmulq_m_n_<supf><mode>"
5776   [
5777    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5778         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5779                        (match_operand:MVE_2 2 "s_register_operand" "w")
5780                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5781                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5782          VMULQ_M_N))
5783   ]
5784   "TARGET_HAVE_MVE"
5785   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5786   [(set_attr "type" "mve_move")
5787    (set_attr "length""8")])
5788
5789 ;;
5790 ;; [vmulq_m_s, vmulq_m_u])
5791 ;;
5792 (define_insn "mve_vmulq_m_<supf><mode>"
5793   [
5794    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5795         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5796                        (match_operand:MVE_2 2 "s_register_operand" "w")
5797                        (match_operand:MVE_2 3 "s_register_operand" "w")
5798                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5799          VMULQ_M))
5800   ]
5801   "TARGET_HAVE_MVE"
5802   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5803   [(set_attr "type" "mve_move")
5804    (set_attr "length""8")])
5805
5806 ;;
5807 ;; [vornq_m_u, vornq_m_s])
5808 ;;
5809 (define_insn "mve_vornq_m_<supf><mode>"
5810   [
5811    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5812         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5813                        (match_operand:MVE_2 2 "s_register_operand" "w")
5814                        (match_operand:MVE_2 3 "s_register_operand" "w")
5815                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5816          VORNQ_M))
5817   ]
5818   "TARGET_HAVE_MVE"
5819   "vpst\;vornt %q0, %q2, %q3"
5820   [(set_attr "type" "mve_move")
5821    (set_attr "length""8")])
5822
5823 ;;
5824 ;; [vorrq_m_s, vorrq_m_u])
5825 ;;
5826 (define_insn "mve_vorrq_m_<supf><mode>"
5827   [
5828    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5829         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5830                        (match_operand:MVE_2 2 "s_register_operand" "w")
5831                        (match_operand:MVE_2 3 "s_register_operand" "w")
5832                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5833          VORRQ_M))
5834   ]
5835   "TARGET_HAVE_MVE"
5836   "vpst\;vorrt %q0, %q2, %q3"
5837   [(set_attr "type" "mve_move")
5838    (set_attr "length""8")])
5839
5840 ;;
5841 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5842 ;;
5843 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5844   [
5845    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5846         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5847                        (match_operand:MVE_2 2 "s_register_operand" "w")
5848                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5849                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5850          VQADDQ_M_N))
5851   ]
5852   "TARGET_HAVE_MVE"
5853   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5854   [(set_attr "type" "mve_move")
5855    (set_attr "length""8")])
5856
5857 ;;
5858 ;; [vqaddq_m_u, vqaddq_m_s])
5859 ;;
5860 (define_insn "mve_vqaddq_m_<supf><mode>"
5861   [
5862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5864                        (match_operand:MVE_2 2 "s_register_operand" "w")
5865                        (match_operand:MVE_2 3 "s_register_operand" "w")
5866                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5867          VQADDQ_M))
5868   ]
5869   "TARGET_HAVE_MVE"
5870   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5871   [(set_attr "type" "mve_move")
5872    (set_attr "length""8")])
5873
5874 ;;
5875 ;; [vqdmlahq_m_n_s])
5876 ;;
5877 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5878   [
5879    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5880         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5881                        (match_operand:MVE_2 2 "s_register_operand" "w")
5882                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5883                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5884          VQDMLAHQ_M_N_S))
5885   ]
5886   "TARGET_HAVE_MVE"
5887   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5888   [(set_attr "type" "mve_move")
5889    (set_attr "length""8")])
5890
5891 ;;
5892 ;; [vqdmlashq_m_n_s])
5893 ;;
5894 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5895   [
5896    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5897         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5898                        (match_operand:MVE_2 2 "s_register_operand" "w")
5899                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5900                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5901          VQDMLASHQ_M_N_S))
5902   ]
5903   "TARGET_HAVE_MVE"
5904   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5905   [(set_attr "type" "mve_move")
5906    (set_attr "length""8")])
5907
5908 ;;
5909 ;; [vqrdmlahq_m_n_s])
5910 ;;
5911 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5912   [
5913    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5914         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5915                        (match_operand:MVE_2 2 "s_register_operand" "w")
5916                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5917                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5918          VQRDMLAHQ_M_N_S))
5919   ]
5920   "TARGET_HAVE_MVE"
5921   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5922   [(set_attr "type" "mve_move")
5923    (set_attr "length""8")])
5924
5925 ;;
5926 ;; [vqrdmlashq_m_n_s])
5927 ;;
5928 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5929   [
5930    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5931         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5932                        (match_operand:MVE_2 2 "s_register_operand" "w")
5933                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5934                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5935          VQRDMLASHQ_M_N_S))
5936   ]
5937   "TARGET_HAVE_MVE"
5938   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5939   [(set_attr "type" "mve_move")
5940    (set_attr "length""8")])
5941
5942 ;;
5943 ;; [vqrshlq_m_u, vqrshlq_m_s])
5944 ;;
5945 (define_insn "mve_vqrshlq_m_<supf><mode>"
5946   [
5947    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5948         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5949                        (match_operand:MVE_2 2 "s_register_operand" "w")
5950                        (match_operand:MVE_2 3 "s_register_operand" "w")
5951                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5952          VQRSHLQ_M))
5953   ]
5954   "TARGET_HAVE_MVE"
5955   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5956   [(set_attr "type" "mve_move")
5957    (set_attr "length""8")])
5958
5959 ;;
5960 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5961 ;;
5962 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5963   [
5964    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5965         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5966                        (match_operand:MVE_2 2 "s_register_operand" "w")
5967                        (match_operand:SI 3 "immediate_operand" "i")
5968                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5969          VQSHLQ_M_N))
5970   ]
5971   "TARGET_HAVE_MVE"
5972   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5973   [(set_attr "type" "mve_move")
5974    (set_attr "length""8")])
5975
5976 ;;
5977 ;; [vqshlq_m_u, vqshlq_m_s])
5978 ;;
5979 (define_insn "mve_vqshlq_m_<supf><mode>"
5980   [
5981    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5982         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5983                        (match_operand:MVE_2 2 "s_register_operand" "w")
5984                        (match_operand:MVE_2 3 "s_register_operand" "w")
5985                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5986          VQSHLQ_M))
5987   ]
5988   "TARGET_HAVE_MVE"
5989   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5990   [(set_attr "type" "mve_move")
5991    (set_attr "length""8")])
5992
5993 ;;
5994 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
5995 ;;
5996 (define_insn "mve_vqsubq_m_n_<supf><mode>"
5997   [
5998    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5999         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6000                        (match_operand:MVE_2 2 "s_register_operand" "w")
6001                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6002                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6003          VQSUBQ_M_N))
6004   ]
6005   "TARGET_HAVE_MVE"
6006   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6007   [(set_attr "type" "mve_move")
6008    (set_attr "length""8")])
6009
6010 ;;
6011 ;; [vqsubq_m_u, vqsubq_m_s])
6012 ;;
6013 (define_insn "mve_vqsubq_m_<supf><mode>"
6014   [
6015    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6016         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6017                        (match_operand:MVE_2 2 "s_register_operand" "w")
6018                        (match_operand:MVE_2 3 "s_register_operand" "w")
6019                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6020          VQSUBQ_M))
6021   ]
6022   "TARGET_HAVE_MVE"
6023   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6024   [(set_attr "type" "mve_move")
6025    (set_attr "length""8")])
6026
6027 ;;
6028 ;; [vrhaddq_m_u, vrhaddq_m_s])
6029 ;;
6030 (define_insn "mve_vrhaddq_m_<supf><mode>"
6031   [
6032    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6033         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6034                        (match_operand:MVE_2 2 "s_register_operand" "w")
6035                        (match_operand:MVE_2 3 "s_register_operand" "w")
6036                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6037          VRHADDQ_M))
6038   ]
6039   "TARGET_HAVE_MVE"
6040   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6041   [(set_attr "type" "mve_move")
6042    (set_attr "length""8")])
6043
6044 ;;
6045 ;; [vrmulhq_m_u, vrmulhq_m_s])
6046 ;;
6047 (define_insn "mve_vrmulhq_m_<supf><mode>"
6048   [
6049    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6050         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6051                        (match_operand:MVE_2 2 "s_register_operand" "w")
6052                        (match_operand:MVE_2 3 "s_register_operand" "w")
6053                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6054          VRMULHQ_M))
6055   ]
6056   "TARGET_HAVE_MVE"
6057   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6058   [(set_attr "type" "mve_move")
6059    (set_attr "length""8")])
6060
6061 ;;
6062 ;; [vrshlq_m_s, vrshlq_m_u])
6063 ;;
6064 (define_insn "mve_vrshlq_m_<supf><mode>"
6065   [
6066    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6067         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6068                        (match_operand:MVE_2 2 "s_register_operand" "w")
6069                        (match_operand:MVE_2 3 "s_register_operand" "w")
6070                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6071          VRSHLQ_M))
6072   ]
6073   "TARGET_HAVE_MVE"
6074   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6075   [(set_attr "type" "mve_move")
6076    (set_attr "length""8")])
6077
6078 ;;
6079 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6080 ;;
6081 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6082   [
6083    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6084         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6085                        (match_operand:MVE_2 2 "s_register_operand" "w")
6086                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6087                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6088          VRSHRQ_M_N))
6089   ]
6090   "TARGET_HAVE_MVE"
6091   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6092   [(set_attr "type" "mve_move")
6093    (set_attr "length""8")])
6094
6095 ;;
6096 ;; [vshlq_m_n_s, vshlq_m_n_u])
6097 ;;
6098 (define_insn "mve_vshlq_m_n_<supf><mode>"
6099   [
6100    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6101         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6102                        (match_operand:MVE_2 2 "s_register_operand" "w")
6103                        (match_operand:SI 3 "immediate_operand" "i")
6104                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6105          VSHLQ_M_N))
6106   ]
6107   "TARGET_HAVE_MVE"
6108   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6109   [(set_attr "type" "mve_move")
6110    (set_attr "length""8")])
6111
6112 ;;
6113 ;; [vshrq_m_n_s, vshrq_m_n_u])
6114 ;;
6115 (define_insn "mve_vshrq_m_n_<supf><mode>"
6116   [
6117    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6118         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6119                        (match_operand:MVE_2 2 "s_register_operand" "w")
6120                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6121                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6122          VSHRQ_M_N))
6123   ]
6124   "TARGET_HAVE_MVE"
6125   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6126   [(set_attr "type" "mve_move")
6127    (set_attr "length""8")])
6128
6129 ;;
6130 ;; [vsliq_m_n_u, vsliq_m_n_s])
6131 ;;
6132 (define_insn "mve_vsliq_m_n_<supf><mode>"
6133    [
6134    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6135        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6136                        (match_operand:MVE_2 2 "s_register_operand" "w")
6137                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6138                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6139          VSLIQ_M_N))
6140   ]
6141   "TARGET_HAVE_MVE"
6142   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6143   [(set_attr "type" "mve_move")
6144    (set_attr "length""8")])
6145
6146 ;;
6147 ;; [vsubq_m_n_s, vsubq_m_n_u])
6148 ;;
6149 (define_insn "mve_vsubq_m_n_<supf><mode>"
6150   [
6151    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6152         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6153                        (match_operand:MVE_2 2 "s_register_operand" "w")
6154                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6155                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6156          VSUBQ_M_N))
6157   ]
6158   "TARGET_HAVE_MVE"
6159   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6160   [(set_attr "type" "mve_move")
6161    (set_attr "length""8")])
6162
6163 ;;
6164 ;; [vhcaddq_rot270_m_s])
6165 ;;
6166 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6167   [
6168    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6169         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6170                        (match_operand:MVE_2 2 "s_register_operand" "w")
6171                        (match_operand:MVE_2 3 "s_register_operand" "w")
6172                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6173          VHCADDQ_ROT270_M_S))
6174   ]
6175   "TARGET_HAVE_MVE"
6176   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6177   [(set_attr "type" "mve_move")
6178    (set_attr "length""8")])
6179
6180 ;;
6181 ;; [vhcaddq_rot90_m_s])
6182 ;;
6183 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6184   [
6185    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6186         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6187                        (match_operand:MVE_2 2 "s_register_operand" "w")
6188                        (match_operand:MVE_2 3 "s_register_operand" "w")
6189                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6190          VHCADDQ_ROT90_M_S))
6191   ]
6192   "TARGET_HAVE_MVE"
6193   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6194   [(set_attr "type" "mve_move")
6195    (set_attr "length""8")])
6196
6197 ;;
6198 ;; [vmladavaxq_p_s])
6199 ;;
6200 (define_insn "mve_vmladavaxq_p_s<mode>"
6201   [
6202    (set (match_operand:SI 0 "s_register_operand" "=Te")
6203         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6204                        (match_operand:MVE_2 2 "s_register_operand" "w")
6205                        (match_operand:MVE_2 3 "s_register_operand" "w")
6206                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6207          VMLADAVAXQ_P_S))
6208   ]
6209   "TARGET_HAVE_MVE"
6210   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6211   [(set_attr "type" "mve_move")
6212    (set_attr "length""8")])
6213
6214 ;;
6215 ;; [vmlsdavaq_p_s])
6216 ;;
6217 (define_insn "mve_vmlsdavaq_p_s<mode>"
6218   [
6219    (set (match_operand:SI 0 "s_register_operand" "=Te")
6220         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6221                        (match_operand:MVE_2 2 "s_register_operand" "w")
6222                        (match_operand:MVE_2 3 "s_register_operand" "w")
6223                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6224          VMLSDAVAQ_P_S))
6225   ]
6226   "TARGET_HAVE_MVE"
6227   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6228   [(set_attr "type" "mve_move")
6229    (set_attr "length""8")])
6230
6231 ;;
6232 ;; [vmlsdavaxq_p_s])
6233 ;;
6234 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6235   [
6236    (set (match_operand:SI 0 "s_register_operand" "=Te")
6237         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6238                        (match_operand:MVE_2 2 "s_register_operand" "w")
6239                        (match_operand:MVE_2 3 "s_register_operand" "w")
6240                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6241          VMLSDAVAXQ_P_S))
6242   ]
6243   "TARGET_HAVE_MVE"
6244   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6245   [(set_attr "type" "mve_move")
6246    (set_attr "length""8")])
6247
6248 ;;
6249 ;; [vqdmladhq_m_s])
6250 ;;
6251 (define_insn "mve_vqdmladhq_m_s<mode>"
6252   [
6253    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6254         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6255                        (match_operand:MVE_2 2 "s_register_operand" "w")
6256                        (match_operand:MVE_2 3 "s_register_operand" "w")
6257                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6258          VQDMLADHQ_M_S))
6259   ]
6260   "TARGET_HAVE_MVE"
6261   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6262   [(set_attr "type" "mve_move")
6263    (set_attr "length""8")])
6264
6265 ;;
6266 ;; [vqdmladhxq_m_s])
6267 ;;
6268 (define_insn "mve_vqdmladhxq_m_s<mode>"
6269   [
6270    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6271         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6272                        (match_operand:MVE_2 2 "s_register_operand" "w")
6273                        (match_operand:MVE_2 3 "s_register_operand" "w")
6274                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6275          VQDMLADHXQ_M_S))
6276   ]
6277   "TARGET_HAVE_MVE"
6278   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6279   [(set_attr "type" "mve_move")
6280    (set_attr "length""8")])
6281
6282 ;;
6283 ;; [vqdmlsdhq_m_s])
6284 ;;
6285 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6286   [
6287    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6288         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6289                        (match_operand:MVE_2 2 "s_register_operand" "w")
6290                        (match_operand:MVE_2 3 "s_register_operand" "w")
6291                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6292          VQDMLSDHQ_M_S))
6293   ]
6294   "TARGET_HAVE_MVE"
6295   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6296   [(set_attr "type" "mve_move")
6297    (set_attr "length""8")])
6298
6299 ;;
6300 ;; [vqdmlsdhxq_m_s])
6301 ;;
6302 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6303   [
6304    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6305         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6306                        (match_operand:MVE_2 2 "s_register_operand" "w")
6307                        (match_operand:MVE_2 3 "s_register_operand" "w")
6308                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6309          VQDMLSDHXQ_M_S))
6310   ]
6311   "TARGET_HAVE_MVE"
6312   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6313   [(set_attr "type" "mve_move")
6314    (set_attr "length""8")])
6315
6316 ;;
6317 ;; [vqdmulhq_m_n_s])
6318 ;;
6319 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6320   [
6321    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6322         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6323                        (match_operand:MVE_2 2 "s_register_operand" "w")
6324                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6325                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6326          VQDMULHQ_M_N_S))
6327   ]
6328   "TARGET_HAVE_MVE"
6329   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6330   [(set_attr "type" "mve_move")
6331    (set_attr "length""8")])
6332
6333 ;;
6334 ;; [vqdmulhq_m_s])
6335 ;;
6336 (define_insn "mve_vqdmulhq_m_s<mode>"
6337   [
6338    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6339         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6340                        (match_operand:MVE_2 2 "s_register_operand" "w")
6341                        (match_operand:MVE_2 3 "s_register_operand" "w")
6342                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6343          VQDMULHQ_M_S))
6344   ]
6345   "TARGET_HAVE_MVE"
6346   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6347   [(set_attr "type" "mve_move")
6348    (set_attr "length""8")])
6349
6350 ;;
6351 ;; [vqrdmladhq_m_s])
6352 ;;
6353 (define_insn "mve_vqrdmladhq_m_s<mode>"
6354   [
6355    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6356         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6357                        (match_operand:MVE_2 2 "s_register_operand" "w")
6358                        (match_operand:MVE_2 3 "s_register_operand" "w")
6359                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6360          VQRDMLADHQ_M_S))
6361   ]
6362   "TARGET_HAVE_MVE"
6363   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6364   [(set_attr "type" "mve_move")
6365    (set_attr "length""8")])
6366
6367 ;;
6368 ;; [vqrdmladhxq_m_s])
6369 ;;
6370 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6371   [
6372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6374                        (match_operand:MVE_2 2 "s_register_operand" "w")
6375                        (match_operand:MVE_2 3 "s_register_operand" "w")
6376                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6377          VQRDMLADHXQ_M_S))
6378   ]
6379   "TARGET_HAVE_MVE"
6380   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6381   [(set_attr "type" "mve_move")
6382    (set_attr "length""8")])
6383
6384 ;;
6385 ;; [vqrdmlsdhq_m_s])
6386 ;;
6387 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6388   [
6389    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6390         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6391                        (match_operand:MVE_2 2 "s_register_operand" "w")
6392                        (match_operand:MVE_2 3 "s_register_operand" "w")
6393                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6394          VQRDMLSDHQ_M_S))
6395   ]
6396   "TARGET_HAVE_MVE"
6397   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6398   [(set_attr "type" "mve_move")
6399    (set_attr "length""8")])
6400
6401 ;;
6402 ;; [vqrdmlsdhxq_m_s])
6403 ;;
6404 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6405   [
6406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6407         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6408                        (match_operand:MVE_2 2 "s_register_operand" "w")
6409                        (match_operand:MVE_2 3 "s_register_operand" "w")
6410                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6411          VQRDMLSDHXQ_M_S))
6412   ]
6413   "TARGET_HAVE_MVE"
6414   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6415   [(set_attr "type" "mve_move")
6416    (set_attr "length""8")])
6417
6418 ;;
6419 ;; [vqrdmulhq_m_n_s])
6420 ;;
6421 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6422   [
6423    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6424         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6425                        (match_operand:MVE_2 2 "s_register_operand" "w")
6426                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6427                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6428          VQRDMULHQ_M_N_S))
6429   ]
6430   "TARGET_HAVE_MVE"
6431   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6432   [(set_attr "type" "mve_move")
6433    (set_attr "length""8")])
6434
6435 ;;
6436 ;; [vqrdmulhq_m_s])
6437 ;;
6438 (define_insn "mve_vqrdmulhq_m_s<mode>"
6439   [
6440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6441         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6442                        (match_operand:MVE_2 2 "s_register_operand" "w")
6443                        (match_operand:MVE_2 3 "s_register_operand" "w")
6444                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6445          VQRDMULHQ_M_S))
6446   ]
6447   "TARGET_HAVE_MVE"
6448   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6449   [(set_attr "type" "mve_move")
6450    (set_attr "length""8")])
6451
6452 ;;
6453 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6454 ;;
6455 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6456   [
6457    (set (match_operand:DI 0 "s_register_operand" "=r")
6458         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6459                        (match_operand:MVE_5 2 "s_register_operand" "w")
6460                        (match_operand:MVE_5 3 "s_register_operand" "w")
6461                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6462          VMLALDAVAQ_P))
6463   ]
6464   "TARGET_HAVE_MVE"
6465   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6466   [(set_attr "type" "mve_move")
6467    (set_attr "length""8")])
6468
6469 ;;
6470 ;; [vmlaldavaxq_p_s])
6471 ;;
6472 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6473   [
6474    (set (match_operand:DI 0 "s_register_operand" "=r")
6475         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6476                        (match_operand:MVE_5 2 "s_register_operand" "w")
6477                        (match_operand:MVE_5 3 "s_register_operand" "w")
6478                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6479          VMLALDAVAXQ_P))
6480   ]
6481   "TARGET_HAVE_MVE"
6482   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6483   [(set_attr "type" "mve_move")
6484    (set_attr "length""8")])
6485
6486 ;;
6487 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6488 ;;
6489 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6490   [
6491    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6492         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6493                        (match_operand:MVE_5 2 "s_register_operand" "w")
6494                        (match_operand:SI 3 "mve_imm_8" "Rb")
6495                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6496          VQRSHRNBQ_M_N))
6497   ]
6498   "TARGET_HAVE_MVE"
6499   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6500   [(set_attr "type" "mve_move")
6501    (set_attr "length""8")])
6502
6503 ;;
6504 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6505 ;;
6506 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6507   [
6508    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6509         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6510                        (match_operand:MVE_5 2 "s_register_operand" "w")
6511                        (match_operand:SI 3 "mve_imm_8" "Rb")
6512                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6513          VQRSHRNTQ_M_N))
6514   ]
6515   "TARGET_HAVE_MVE"
6516   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6517   [(set_attr "type" "mve_move")
6518    (set_attr "length""8")])
6519
6520 ;;
6521 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6522 ;;
6523 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6524   [
6525    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6526         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6527                        (match_operand:MVE_5 2 "s_register_operand" "w")
6528                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6529                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6530          VQSHRNBQ_M_N))
6531   ]
6532   "TARGET_HAVE_MVE"
6533   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6534   [(set_attr "type" "mve_move")
6535    (set_attr "length""8")])
6536
6537 ;;
6538 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6539 ;;
6540 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6541   [
6542    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6543         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6544                        (match_operand:MVE_5 2 "s_register_operand" "w")
6545                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6546                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6547          VQSHRNTQ_M_N))
6548   ]
6549   "TARGET_HAVE_MVE"
6550   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6551   [(set_attr "type" "mve_move")
6552    (set_attr "length""8")])
6553
6554 ;;
6555 ;; [vrmlaldavhaq_p_s])
6556 ;;
6557 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6558   [
6559    (set (match_operand:DI 0 "s_register_operand" "=r")
6560         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6561                        (match_operand:V4SI 2 "s_register_operand" "w")
6562                        (match_operand:V4SI 3 "s_register_operand" "w")
6563                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6564          VRMLALDAVHAQ_P_S))
6565   ]
6566   "TARGET_HAVE_MVE"
6567   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6568   [(set_attr "type" "mve_move")
6569    (set_attr "length""8")])
6570
6571 ;;
6572 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6573 ;;
6574 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6575   [
6576    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6577         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6578                        (match_operand:MVE_5 2 "s_register_operand" "w")
6579                        (match_operand:SI 3 "mve_imm_8" "Rb")
6580                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6581          VRSHRNBQ_M_N))
6582   ]
6583   "TARGET_HAVE_MVE"
6584   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6585   [(set_attr "type" "mve_move")
6586    (set_attr "length""8")])
6587
6588 ;;
6589 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6590 ;;
6591 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6592   [
6593    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6594         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6595                        (match_operand:MVE_5 2 "s_register_operand" "w")
6596                        (match_operand:SI 3 "mve_imm_8" "Rb")
6597                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6598          VRSHRNTQ_M_N))
6599   ]
6600   "TARGET_HAVE_MVE"
6601   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6602   [(set_attr "type" "mve_move")
6603    (set_attr "length""8")])
6604
6605 ;;
6606 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6607 ;;
6608 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6609   [
6610    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6611         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6612                        (match_operand:MVE_3 2 "s_register_operand" "w")
6613                        (match_operand:SI 3 "immediate_operand" "i")
6614                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6615          VSHLLBQ_M_N))
6616   ]
6617   "TARGET_HAVE_MVE"
6618   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6619   [(set_attr "type" "mve_move")
6620    (set_attr "length""8")])
6621
6622 ;;
6623 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6624 ;;
6625 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6626   [
6627    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6628         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6629                        (match_operand:MVE_3 2 "s_register_operand" "w")
6630                        (match_operand:SI 3 "immediate_operand" "i")
6631                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6632          VSHLLTQ_M_N))
6633   ]
6634   "TARGET_HAVE_MVE"
6635   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6636   [(set_attr "type" "mve_move")
6637    (set_attr "length""8")])
6638
6639 ;;
6640 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6641 ;;
6642 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6643   [
6644    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6645         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6646                        (match_operand:MVE_5 2 "s_register_operand" "w")
6647                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6648                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6649          VSHRNBQ_M_N))
6650   ]
6651   "TARGET_HAVE_MVE"
6652   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6653   [(set_attr "type" "mve_move")
6654    (set_attr "length""8")])
6655
6656 ;;
6657 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6658 ;;
6659 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6660   [
6661    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6662         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6663                        (match_operand:MVE_5 2 "s_register_operand" "w")
6664                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6665                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6666          VSHRNTQ_M_N))
6667   ]
6668   "TARGET_HAVE_MVE"
6669   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6670   [(set_attr "type" "mve_move")
6671    (set_attr "length""8")])
6672
6673 ;;
6674 ;; [vmlsldavaq_p_s])
6675 ;;
6676 (define_insn "mve_vmlsldavaq_p_s<mode>"
6677   [
6678    (set (match_operand:DI 0 "s_register_operand" "=r")
6679         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6680                        (match_operand:MVE_5 2 "s_register_operand" "w")
6681                        (match_operand:MVE_5 3 "s_register_operand" "w")
6682                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6683          VMLSLDAVAQ_P_S))
6684   ]
6685   "TARGET_HAVE_MVE"
6686   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6687   [(set_attr "type" "mve_move")
6688    (set_attr "length""8")])
6689
6690 ;;
6691 ;; [vmlsldavaxq_p_s])
6692 ;;
6693 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6694   [
6695    (set (match_operand:DI 0 "s_register_operand" "=r")
6696         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6697                        (match_operand:MVE_5 2 "s_register_operand" "w")
6698                        (match_operand:MVE_5 3 "s_register_operand" "w")
6699                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6700          VMLSLDAVAXQ_P_S))
6701   ]
6702   "TARGET_HAVE_MVE"
6703   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6704   [(set_attr "type" "mve_move")
6705    (set_attr "length""8")])
6706
6707 ;;
6708 ;; [vmullbq_poly_m_p])
6709 ;;
6710 (define_insn "mve_vmullbq_poly_m_p<mode>"
6711   [
6712    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6713         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6714                        (match_operand:MVE_3 2 "s_register_operand" "w")
6715                        (match_operand:MVE_3 3 "s_register_operand" "w")
6716                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6717          VMULLBQ_POLY_M_P))
6718   ]
6719   "TARGET_HAVE_MVE"
6720   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6721   [(set_attr "type" "mve_move")
6722    (set_attr "length""8")])
6723
6724 ;;
6725 ;; [vmulltq_poly_m_p])
6726 ;;
6727 (define_insn "mve_vmulltq_poly_m_p<mode>"
6728   [
6729    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6730         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6731                        (match_operand:MVE_3 2 "s_register_operand" "w")
6732                        (match_operand:MVE_3 3 "s_register_operand" "w")
6733                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6734          VMULLTQ_POLY_M_P))
6735   ]
6736   "TARGET_HAVE_MVE"
6737   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6738   [(set_attr "type" "mve_move")
6739    (set_attr "length""8")])
6740
6741 ;;
6742 ;; [vqdmullbq_m_n_s])
6743 ;;
6744 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6745   [
6746    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6747         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6748                        (match_operand:MVE_5 2 "s_register_operand" "w")
6749                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6750                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6751          VQDMULLBQ_M_N_S))
6752   ]
6753   "TARGET_HAVE_MVE"
6754   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6755   [(set_attr "type" "mve_move")
6756    (set_attr "length""8")])
6757
6758 ;;
6759 ;; [vqdmullbq_m_s])
6760 ;;
6761 (define_insn "mve_vqdmullbq_m_s<mode>"
6762   [
6763    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6764         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6765                        (match_operand:MVE_5 2 "s_register_operand" "w")
6766                        (match_operand:MVE_5 3 "s_register_operand" "w")
6767                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6768          VQDMULLBQ_M_S))
6769   ]
6770   "TARGET_HAVE_MVE"
6771   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6772   [(set_attr "type" "mve_move")
6773    (set_attr "length""8")])
6774
6775 ;;
6776 ;; [vqdmulltq_m_n_s])
6777 ;;
6778 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6779   [
6780    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6781         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6782                        (match_operand:MVE_5 2 "s_register_operand" "w")
6783                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6784                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6785          VQDMULLTQ_M_N_S))
6786   ]
6787   "TARGET_HAVE_MVE"
6788   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6789   [(set_attr "type" "mve_move")
6790    (set_attr "length""8")])
6791
6792 ;;
6793 ;; [vqdmulltq_m_s])
6794 ;;
6795 (define_insn "mve_vqdmulltq_m_s<mode>"
6796   [
6797    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6798         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6799                        (match_operand:MVE_5 2 "s_register_operand" "w")
6800                        (match_operand:MVE_5 3 "s_register_operand" "w")
6801                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6802          VQDMULLTQ_M_S))
6803   ]
6804   "TARGET_HAVE_MVE"
6805   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6806   [(set_attr "type" "mve_move")
6807    (set_attr "length""8")])
6808
6809 ;;
6810 ;; [vqrshrunbq_m_n_s])
6811 ;;
6812 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6813   [
6814    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6815         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6816                        (match_operand:MVE_5 2 "s_register_operand" "w")
6817                        (match_operand:SI 3 "mve_imm_8" "Rb")
6818                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6819          VQRSHRUNBQ_M_N_S))
6820   ]
6821   "TARGET_HAVE_MVE"
6822   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6823   [(set_attr "type" "mve_move")
6824    (set_attr "length""8")])
6825
6826 ;;
6827 ;; [vqrshruntq_m_n_s])
6828 ;;
6829 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6830   [
6831    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6832         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6833                        (match_operand:MVE_5 2 "s_register_operand" "w")
6834                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6835                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6836          VQRSHRUNTQ_M_N_S))
6837   ]
6838   "TARGET_HAVE_MVE"
6839   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6840   [(set_attr "type" "mve_move")
6841    (set_attr "length""8")])
6842
6843 ;;
6844 ;; [vqshrunbq_m_n_s])
6845 ;;
6846 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6847   [
6848    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6849         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6850                        (match_operand:MVE_5 2 "s_register_operand" "w")
6851                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6852                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6853          VQSHRUNBQ_M_N_S))
6854   ]
6855   "TARGET_HAVE_MVE"
6856   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6857   [(set_attr "type" "mve_move")
6858    (set_attr "length""8")])
6859
6860 ;;
6861 ;; [vqshruntq_m_n_s])
6862 ;;
6863 (define_insn "mve_vqshruntq_m_n_s<mode>"
6864   [
6865    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6866         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6867                        (match_operand:MVE_5 2 "s_register_operand" "w")
6868                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6869                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6870          VQSHRUNTQ_M_N_S))
6871   ]
6872   "TARGET_HAVE_MVE"
6873   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6874   [(set_attr "type" "mve_move")
6875    (set_attr "length""8")])
6876
6877 ;;
6878 ;; [vrmlaldavhaq_p_u])
6879 ;;
6880 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6881   [
6882    (set (match_operand:DI 0 "s_register_operand" "=r")
6883         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6884                        (match_operand:V4SI 2 "s_register_operand" "w")
6885                        (match_operand:V4SI 3 "s_register_operand" "w")
6886                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6887          VRMLALDAVHAQ_P_U))
6888   ]
6889   "TARGET_HAVE_MVE"
6890   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6891   [(set_attr "type" "mve_move")
6892    (set_attr "length""8")])
6893
6894 ;;
6895 ;; [vrmlaldavhaxq_p_s])
6896 ;;
6897 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6898   [
6899    (set (match_operand:DI 0 "s_register_operand" "=r")
6900         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6901                        (match_operand:V4SI 2 "s_register_operand" "w")
6902                        (match_operand:V4SI 3 "s_register_operand" "w")
6903                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6904          VRMLALDAVHAXQ_P_S))
6905   ]
6906   "TARGET_HAVE_MVE"
6907   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6908   [(set_attr "type" "mve_move")
6909    (set_attr "length""8")])
6910
6911 ;;
6912 ;; [vrmlsldavhaq_p_s])
6913 ;;
6914 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6915   [
6916    (set (match_operand:DI 0 "s_register_operand" "=r")
6917         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6918                        (match_operand:V4SI 2 "s_register_operand" "w")
6919                        (match_operand:V4SI 3 "s_register_operand" "w")
6920                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6921          VRMLSLDAVHAQ_P_S))
6922   ]
6923   "TARGET_HAVE_MVE"
6924   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6925   [(set_attr "type" "mve_move")
6926    (set_attr "length""8")])
6927
6928 ;;
6929 ;; [vrmlsldavhaxq_p_s])
6930 ;;
6931 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6932   [
6933    (set (match_operand:DI 0 "s_register_operand" "=r")
6934         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6935                        (match_operand:V4SI 2 "s_register_operand" "w")
6936                        (match_operand:V4SI 3 "s_register_operand" "w")
6937                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6938          VRMLSLDAVHAXQ_P_S))
6939   ]
6940   "TARGET_HAVE_MVE"
6941   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6942   [(set_attr "type" "mve_move")
6943    (set_attr "length""8")])
6944 ;;
6945 ;; [vabdq_m_f])
6946 ;;
6947 (define_insn "mve_vabdq_m_f<mode>"
6948   [
6949    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6950         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6951                        (match_operand:MVE_0 2 "s_register_operand" "w")
6952                        (match_operand:MVE_0 3 "s_register_operand" "w")
6953                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6954          VABDQ_M_F))
6955   ]
6956   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6957   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6958   [(set_attr "type" "mve_move")
6959    (set_attr "length""8")])
6960
6961 ;;
6962 ;; [vaddq_m_f])
6963 ;;
6964 (define_insn "mve_vaddq_m_f<mode>"
6965   [
6966    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6967         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6968                        (match_operand:MVE_0 2 "s_register_operand" "w")
6969                        (match_operand:MVE_0 3 "s_register_operand" "w")
6970                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6971          VADDQ_M_F))
6972   ]
6973   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6974   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6975   [(set_attr "type" "mve_move")
6976    (set_attr "length""8")])
6977
6978 ;;
6979 ;; [vaddq_m_n_f])
6980 ;;
6981 (define_insn "mve_vaddq_m_n_f<mode>"
6982   [
6983    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6984         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6985                        (match_operand:MVE_0 2 "s_register_operand" "w")
6986                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6987                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6988          VADDQ_M_N_F))
6989   ]
6990   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6991   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
6992   [(set_attr "type" "mve_move")
6993    (set_attr "length""8")])
6994
6995 ;;
6996 ;; [vandq_m_f])
6997 ;;
6998 (define_insn "mve_vandq_m_f<mode>"
6999   [
7000    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7001         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7002                        (match_operand:MVE_0 2 "s_register_operand" "w")
7003                        (match_operand:MVE_0 3 "s_register_operand" "w")
7004                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7005          VANDQ_M_F))
7006   ]
7007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7008   "vpst\;vandt %q0, %q2, %q3"
7009   [(set_attr "type" "mve_move")
7010    (set_attr "length""8")])
7011
7012 ;;
7013 ;; [vbicq_m_f])
7014 ;;
7015 (define_insn "mve_vbicq_m_f<mode>"
7016   [
7017    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7018         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7019                        (match_operand:MVE_0 2 "s_register_operand" "w")
7020                        (match_operand:MVE_0 3 "s_register_operand" "w")
7021                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7022          VBICQ_M_F))
7023   ]
7024   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7025   "vpst\;vbict %q0, %q2, %q3"
7026   [(set_attr "type" "mve_move")
7027    (set_attr "length""8")])
7028
7029 ;;
7030 ;; [vbrsrq_m_n_f])
7031 ;;
7032 (define_insn "mve_vbrsrq_m_n_f<mode>"
7033   [
7034    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7035         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7036                        (match_operand:MVE_0 2 "s_register_operand" "w")
7037                        (match_operand:SI 3 "s_register_operand" "r")
7038                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7039          VBRSRQ_M_N_F))
7040   ]
7041   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7042   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7043   [(set_attr "type" "mve_move")
7044    (set_attr "length""8")])
7045
7046 ;;
7047 ;; [vcaddq_rot270_m_f])
7048 ;;
7049 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7050   [
7051    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7052         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7053                        (match_operand:MVE_0 2 "s_register_operand" "w")
7054                        (match_operand:MVE_0 3 "s_register_operand" "w")
7055                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7056          VCADDQ_ROT270_M_F))
7057   ]
7058   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7059   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7060   [(set_attr "type" "mve_move")
7061    (set_attr "length""8")])
7062
7063 ;;
7064 ;; [vcaddq_rot90_m_f])
7065 ;;
7066 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7067   [
7068    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7069         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7070                        (match_operand:MVE_0 2 "s_register_operand" "w")
7071                        (match_operand:MVE_0 3 "s_register_operand" "w")
7072                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7073          VCADDQ_ROT90_M_F))
7074   ]
7075   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7076   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7077   [(set_attr "type" "mve_move")
7078    (set_attr "length""8")])
7079
7080 ;;
7081 ;; [vcmlaq_m_f])
7082 ;;
7083 (define_insn "mve_vcmlaq_m_f<mode>"
7084   [
7085    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7086         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7087                        (match_operand:MVE_0 2 "s_register_operand" "w")
7088                        (match_operand:MVE_0 3 "s_register_operand" "w")
7089                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7090          VCMLAQ_M_F))
7091   ]
7092   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7093   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7094   [(set_attr "type" "mve_move")
7095    (set_attr "length""8")])
7096
7097 ;;
7098 ;; [vcmlaq_rot180_m_f])
7099 ;;
7100 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7101   [
7102    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7103         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7104                        (match_operand:MVE_0 2 "s_register_operand" "w")
7105                        (match_operand:MVE_0 3 "s_register_operand" "w")
7106                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7107          VCMLAQ_ROT180_M_F))
7108   ]
7109   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7110   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7111   [(set_attr "type" "mve_move")
7112    (set_attr "length""8")])
7113
7114 ;;
7115 ;; [vcmlaq_rot270_m_f])
7116 ;;
7117 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7118   [
7119    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7120         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7121                        (match_operand:MVE_0 2 "s_register_operand" "w")
7122                        (match_operand:MVE_0 3 "s_register_operand" "w")
7123                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7124          VCMLAQ_ROT270_M_F))
7125   ]
7126   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7127   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7128   [(set_attr "type" "mve_move")
7129    (set_attr "length""8")])
7130
7131 ;;
7132 ;; [vcmlaq_rot90_m_f])
7133 ;;
7134 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7135   [
7136    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7137         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7138                        (match_operand:MVE_0 2 "s_register_operand" "w")
7139                        (match_operand:MVE_0 3 "s_register_operand" "w")
7140                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7141          VCMLAQ_ROT90_M_F))
7142   ]
7143   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7144   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7145   [(set_attr "type" "mve_move")
7146    (set_attr "length""8")])
7147
7148 ;;
7149 ;; [vcmulq_m_f])
7150 ;;
7151 (define_insn "mve_vcmulq_m_f<mode>"
7152   [
7153    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7154         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7155                        (match_operand:MVE_0 2 "s_register_operand" "w")
7156                        (match_operand:MVE_0 3 "s_register_operand" "w")
7157                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7158          VCMULQ_M_F))
7159   ]
7160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7161   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7162   [(set_attr "type" "mve_move")
7163    (set_attr "length""8")])
7164
7165 ;;
7166 ;; [vcmulq_rot180_m_f])
7167 ;;
7168 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7169   [
7170    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7171         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7172                        (match_operand:MVE_0 2 "s_register_operand" "w")
7173                        (match_operand:MVE_0 3 "s_register_operand" "w")
7174                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7175          VCMULQ_ROT180_M_F))
7176   ]
7177   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7178   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7179   [(set_attr "type" "mve_move")
7180    (set_attr "length""8")])
7181
7182 ;;
7183 ;; [vcmulq_rot270_m_f])
7184 ;;
7185 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7186   [
7187    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7188         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7189                        (match_operand:MVE_0 2 "s_register_operand" "w")
7190                        (match_operand:MVE_0 3 "s_register_operand" "w")
7191                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7192          VCMULQ_ROT270_M_F))
7193   ]
7194   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7195   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7196   [(set_attr "type" "mve_move")
7197    (set_attr "length""8")])
7198
7199 ;;
7200 ;; [vcmulq_rot90_m_f])
7201 ;;
7202 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7203   [
7204    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7205         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7206                        (match_operand:MVE_0 2 "s_register_operand" "w")
7207                        (match_operand:MVE_0 3 "s_register_operand" "w")
7208                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7209          VCMULQ_ROT90_M_F))
7210   ]
7211   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7212   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7213   [(set_attr "type" "mve_move")
7214    (set_attr "length""8")])
7215
7216 ;;
7217 ;; [veorq_m_f])
7218 ;;
7219 (define_insn "mve_veorq_m_f<mode>"
7220   [
7221    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7222         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7223                        (match_operand:MVE_0 2 "s_register_operand" "w")
7224                        (match_operand:MVE_0 3 "s_register_operand" "w")
7225                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7226          VEORQ_M_F))
7227   ]
7228   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7229   "vpst\;veort %q0, %q2, %q3"
7230   [(set_attr "type" "mve_move")
7231    (set_attr "length""8")])
7232
7233 ;;
7234 ;; [vfmaq_m_f])
7235 ;;
7236 (define_insn "mve_vfmaq_m_f<mode>"
7237   [
7238    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7239         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7240                        (match_operand:MVE_0 2 "s_register_operand" "w")
7241                        (match_operand:MVE_0 3 "s_register_operand" "w")
7242                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7243          VFMAQ_M_F))
7244   ]
7245   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7246   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7247   [(set_attr "type" "mve_move")
7248    (set_attr "length""8")])
7249
7250 ;;
7251 ;; [vfmaq_m_n_f])
7252 ;;
7253 (define_insn "mve_vfmaq_m_n_f<mode>"
7254   [
7255    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7256         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7257                        (match_operand:MVE_0 2 "s_register_operand" "w")
7258                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7259                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7260          VFMAQ_M_N_F))
7261   ]
7262   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7263   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7264   [(set_attr "type" "mve_move")
7265    (set_attr "length""8")])
7266
7267 ;;
7268 ;; [vfmasq_m_n_f])
7269 ;;
7270 (define_insn "mve_vfmasq_m_n_f<mode>"
7271   [
7272    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7273         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7274                        (match_operand:MVE_0 2 "s_register_operand" "w")
7275                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7276                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7277          VFMASQ_M_N_F))
7278   ]
7279   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7280   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7281   [(set_attr "type" "mve_move")
7282    (set_attr "length""8")])
7283
7284 ;;
7285 ;; [vfmsq_m_f])
7286 ;;
7287 (define_insn "mve_vfmsq_m_f<mode>"
7288   [
7289    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7290         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7291                        (match_operand:MVE_0 2 "s_register_operand" "w")
7292                        (match_operand:MVE_0 3 "s_register_operand" "w")
7293                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7294          VFMSQ_M_F))
7295   ]
7296   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7297   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7298   [(set_attr "type" "mve_move")
7299    (set_attr "length""8")])
7300
7301 ;;
7302 ;; [vmaxnmq_m_f])
7303 ;;
7304 (define_insn "mve_vmaxnmq_m_f<mode>"
7305   [
7306    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7307         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7308                        (match_operand:MVE_0 2 "s_register_operand" "w")
7309                        (match_operand:MVE_0 3 "s_register_operand" "w")
7310                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7311          VMAXNMQ_M_F))
7312   ]
7313   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7314   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7315   [(set_attr "type" "mve_move")
7316    (set_attr "length""8")])
7317
7318 ;;
7319 ;; [vminnmq_m_f])
7320 ;;
7321 (define_insn "mve_vminnmq_m_f<mode>"
7322   [
7323    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7324         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7325                        (match_operand:MVE_0 2 "s_register_operand" "w")
7326                        (match_operand:MVE_0 3 "s_register_operand" "w")
7327                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7328          VMINNMQ_M_F))
7329   ]
7330   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7331   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7332   [(set_attr "type" "mve_move")
7333    (set_attr "length""8")])
7334
7335 ;;
7336 ;; [vmulq_m_f])
7337 ;;
7338 (define_insn "mve_vmulq_m_f<mode>"
7339   [
7340    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7341         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7342                        (match_operand:MVE_0 2 "s_register_operand" "w")
7343                        (match_operand:MVE_0 3 "s_register_operand" "w")
7344                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7345          VMULQ_M_F))
7346   ]
7347   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7348   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7349   [(set_attr "type" "mve_move")
7350    (set_attr "length""8")])
7351
7352 ;;
7353 ;; [vmulq_m_n_f])
7354 ;;
7355 (define_insn "mve_vmulq_m_n_f<mode>"
7356   [
7357    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7358         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7359                        (match_operand:MVE_0 2 "s_register_operand" "w")
7360                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7361                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7362          VMULQ_M_N_F))
7363   ]
7364   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7365   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7366   [(set_attr "type" "mve_move")
7367    (set_attr "length""8")])
7368
7369 ;;
7370 ;; [vornq_m_f])
7371 ;;
7372 (define_insn "mve_vornq_m_f<mode>"
7373   [
7374    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7375         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7376                        (match_operand:MVE_0 2 "s_register_operand" "w")
7377                        (match_operand:MVE_0 3 "s_register_operand" "w")
7378                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7379          VORNQ_M_F))
7380   ]
7381   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7382   "vpst\;vornt %q0, %q2, %q3"
7383   [(set_attr "type" "mve_move")
7384    (set_attr "length""8")])
7385
7386 ;;
7387 ;; [vorrq_m_f])
7388 ;;
7389 (define_insn "mve_vorrq_m_f<mode>"
7390   [
7391    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7392         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7393                        (match_operand:MVE_0 2 "s_register_operand" "w")
7394                        (match_operand:MVE_0 3 "s_register_operand" "w")
7395                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7396          VORRQ_M_F))
7397   ]
7398   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7399   "vpst\;vorrt %q0, %q2, %q3"
7400   [(set_attr "type" "mve_move")
7401    (set_attr "length""8")])
7402
7403 ;;
7404 ;; [vsubq_m_f])
7405 ;;
7406 (define_insn "mve_vsubq_m_f<mode>"
7407   [
7408    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7409         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7410                        (match_operand:MVE_0 2 "s_register_operand" "w")
7411                        (match_operand:MVE_0 3 "s_register_operand" "w")
7412                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7413          VSUBQ_M_F))
7414   ]
7415   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7416   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7417   [(set_attr "type" "mve_move")
7418    (set_attr "length""8")])
7419
7420 ;;
7421 ;; [vsubq_m_n_f])
7422 ;;
7423 (define_insn "mve_vsubq_m_n_f<mode>"
7424   [
7425    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7426         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7427                        (match_operand:MVE_0 2 "s_register_operand" "w")
7428                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7429                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7430          VSUBQ_M_N_F))
7431   ]
7432   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7433   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7434   [(set_attr "type" "mve_move")
7435    (set_attr "length""8")])
7436
7437 ;;
7438 ;; [vstrbq_s vstrbq_u]
7439 ;;
7440 (define_insn "mve_vstrbq_<supf><mode>"
7441   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7442         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7443          VSTRBQ))
7444   ]
7445   "TARGET_HAVE_MVE"
7446 {
7447    rtx ops[2];
7448    int regno = REGNO (operands[1]);
7449    ops[1] = gen_rtx_REG (TImode, regno);
7450    ops[0]  = operands[0];
7451    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7452    return "";
7453 }
7454   [(set_attr "length" "4")])
7455
7456 ;;
7457 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7458 ;;
7459 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7460   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7461    (match_operand:MVE_2 1 "s_register_operand")
7462    (match_operand:MVE_2 2 "s_register_operand")
7463    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7464   "TARGET_HAVE_MVE"
7465 {
7466   rtx ind = XEXP (operands[0], 0);
7467   gcc_assert (REG_P (ind));
7468   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7469                                                               operands[2]));
7470   DONE;
7471 })
7472
7473 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7474   [(set (mem:BLK (scratch))
7475         (unspec:BLK
7476           [(match_operand:SI 0 "register_operand" "r")
7477            (match_operand:MVE_2 1 "s_register_operand" "w")
7478            (match_operand:MVE_2 2 "s_register_operand" "w")]
7479           VSTRBSOQ))]
7480   "TARGET_HAVE_MVE"
7481   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7482   [(set_attr "length" "4")])
7483
7484 ;;
7485 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7486 ;;
7487 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7488   [(set (mem:BLK (scratch))
7489         (unspec:BLK
7490                 [(match_operand:V4SI 0 "s_register_operand" "w")
7491                  (match_operand:SI 1 "immediate_operand" "i")
7492                  (match_operand:V4SI 2 "s_register_operand" "w")]
7493          VSTRWSBQ))
7494   ]
7495   "TARGET_HAVE_MVE"
7496 {
7497    rtx ops[3];
7498    ops[0] = operands[0];
7499    ops[1] = operands[1];
7500    ops[2] = operands[2];
7501    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7502    return "";
7503 }
7504   [(set_attr "length" "4")])
7505
7506 ;;
7507 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7508 ;;
7509 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7510   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7511         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7512                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7513          VLDRBGOQ))
7514   ]
7515   "TARGET_HAVE_MVE"
7516 {
7517    rtx ops[3];
7518    ops[0] = operands[0];
7519    ops[1] = operands[1];
7520    ops[2] = operands[2];
7521    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7522      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7523    else
7524      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7525    return "";
7526 }
7527   [(set_attr "length" "4")])
7528
7529 ;;
7530 ;; [vldrbq_s vldrbq_u]
7531 ;;
7532 (define_insn "mve_vldrbq_<supf><mode>"
7533   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7534         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7535          VLDRBQ))
7536   ]
7537   "TARGET_HAVE_MVE"
7538 {
7539    rtx ops[2];
7540    int regno = REGNO (operands[0]);
7541    ops[0] = gen_rtx_REG (TImode, regno);
7542    ops[1]  = operands[1];
7543    if (<V_sz_elem> == 8)
7544      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7545    else
7546      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7547    return "";
7548 }
7549   [(set_attr "length" "4")])
7550
7551 ;;
7552 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7553 ;;
7554 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7555   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7556         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7557                       (match_operand:SI 2 "immediate_operand" "i")]
7558          VLDRWGBQ))
7559   ]
7560   "TARGET_HAVE_MVE"
7561 {
7562    rtx ops[3];
7563    ops[0] = operands[0];
7564    ops[1] = operands[1];
7565    ops[2] = operands[2];
7566    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7567    return "";
7568 }
7569   [(set_attr "length" "4")])
7570
7571 ;;
7572 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7573 ;;
7574 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7575   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7576    (match_operand:MVE_2 1 "s_register_operand")
7577    (match_operand:MVE_2 2 "s_register_operand")
7578    (match_operand:HI 3 "vpr_register_operand" "Up")
7579    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7580   "TARGET_HAVE_MVE"
7581 {
7582   rtx ind = XEXP (operands[0], 0);
7583   gcc_assert (REG_P (ind));
7584   emit_insn (
7585     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7586                                                        operands[2],
7587                                                        operands[3]));
7588   DONE;
7589 })
7590
7591 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7592   [(set (mem:BLK (scratch))
7593         (unspec:BLK
7594           [(match_operand:SI 0 "register_operand" "r")
7595            (match_operand:MVE_2 1 "s_register_operand" "w")
7596            (match_operand:MVE_2 2 "s_register_operand" "w")
7597            (match_operand:HI 3 "vpr_register_operand" "Up")]
7598           VSTRBSOQ))]
7599   "TARGET_HAVE_MVE"
7600   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7601   [(set_attr "length" "8")])
7602
7603 ;;
7604 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7605 ;;
7606 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7607   [(set (mem:BLK (scratch))
7608         (unspec:BLK
7609                 [(match_operand:V4SI 0 "s_register_operand" "w")
7610                  (match_operand:SI 1 "immediate_operand" "i")
7611                  (match_operand:V4SI 2 "s_register_operand" "w")
7612                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7613          VSTRWSBQ))
7614   ]
7615   "TARGET_HAVE_MVE"
7616 {
7617    rtx ops[3];
7618    ops[0] = operands[0];
7619    ops[1] = operands[1];
7620    ops[2] = operands[2];
7621    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7622    return "";
7623 }
7624   [(set_attr "length" "8")])
7625
7626 ;;
7627 ;; [vstrbq_p_s vstrbq_p_u]
7628 ;;
7629 (define_insn "mve_vstrbq_p_<supf><mode>"
7630   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7631         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7632                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7633          VSTRBQ))
7634   ]
7635   "TARGET_HAVE_MVE"
7636 {
7637    rtx ops[2];
7638    int regno = REGNO (operands[1]);
7639    ops[1] = gen_rtx_REG (TImode, regno);
7640    ops[0]  = operands[0];
7641    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7642    return "";
7643 }
7644   [(set_attr "length" "8")])
7645
7646 ;;
7647 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7648 ;;
7649 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7650   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7651         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7652                        (match_operand:MVE_2 2 "s_register_operand" "w")
7653                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7654          VLDRBGOQ))
7655   ]
7656   "TARGET_HAVE_MVE"
7657 {
7658    rtx ops[4];
7659    ops[0] = operands[0];
7660    ops[1] = operands[1];
7661    ops[2] = operands[2];
7662    ops[3] = operands[3];
7663    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7664      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7665    else
7666      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7667    return "";
7668 }
7669   [(set_attr "length" "8")])
7670
7671 ;;
7672 ;; [vldrbq_z_s vldrbq_z_u]
7673 ;;
7674 (define_insn "mve_vldrbq_z_<supf><mode>"
7675   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7676         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7677                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7678          VLDRBQ))
7679   ]
7680   "TARGET_HAVE_MVE"
7681 {
7682    rtx ops[2];
7683    int regno = REGNO (operands[0]);
7684    ops[0] = gen_rtx_REG (TImode, regno);
7685    ops[1]  = operands[1];
7686    if (<V_sz_elem> == 8)
7687      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7688    else
7689      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7690    return "";
7691 }
7692   [(set_attr "length" "8")])
7693
7694 ;;
7695 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7696 ;;
7697 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7698   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7699         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7700                       (match_operand:SI 2 "immediate_operand" "i")
7701                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7702          VLDRWGBQ))
7703   ]
7704   "TARGET_HAVE_MVE"
7705 {
7706    rtx ops[3];
7707    ops[0] = operands[0];
7708    ops[1] = operands[1];
7709    ops[2] = operands[2];
7710    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7711    return "";
7712 }
7713   [(set_attr "length" "8")])
7714
7715 ;;
7716 ;; [vldrhq_f]
7717 ;;
7718 (define_insn "mve_vldrhq_fv8hf"
7719   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7720         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7721          VLDRHQ_F))
7722   ]
7723   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7724 {
7725    rtx ops[2];
7726    int regno = REGNO (operands[0]);
7727    ops[0] = gen_rtx_REG (TImode, regno);
7728    ops[1]  = operands[1];
7729    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7730    return "";
7731 }
7732   [(set_attr "length" "4")])
7733
7734 ;;
7735 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7736 ;;
7737 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7738   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7739         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7740                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7741         VLDRHGOQ))
7742   ]
7743   "TARGET_HAVE_MVE"
7744 {
7745    rtx ops[3];
7746    ops[0] = operands[0];
7747    ops[1] = operands[1];
7748    ops[2] = operands[2];
7749    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7750      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7751    else
7752      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7753    return "";
7754 }
7755   [(set_attr "length" "4")])
7756
7757 ;;
7758 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7759 ;;
7760 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7761   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7762         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7763                        (match_operand:MVE_6 2 "s_register_operand" "w")
7764                        (match_operand:HI 3 "vpr_register_operand" "Up")
7765         ]VLDRHGOQ))
7766   ]
7767   "TARGET_HAVE_MVE"
7768 {
7769    rtx ops[4];
7770    ops[0] = operands[0];
7771    ops[1] = operands[1];
7772    ops[2] = operands[2];
7773    ops[3] = operands[3];
7774    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7775      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7776    else
7777      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7778    return "";
7779 }
7780  [(set_attr "length" "8")])
7781
7782 ;;
7783 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7784 ;;
7785 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7786   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7787         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7788                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7789         VLDRHGSOQ))
7790   ]
7791   "TARGET_HAVE_MVE"
7792 {
7793    rtx ops[3];
7794    ops[0] = operands[0];
7795    ops[1] = operands[1];
7796    ops[2] = operands[2];
7797       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7798      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7799    else
7800      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7801    return "";
7802 }
7803   [(set_attr "length" "4")])
7804
7805 ;;
7806 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7807 ;;
7808 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7809   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7810         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7811                        (match_operand:MVE_6 2 "s_register_operand" "w")
7812                        (match_operand:HI 3 "vpr_register_operand" "Up")
7813         ]VLDRHGSOQ))
7814   ]
7815   "TARGET_HAVE_MVE"
7816 {
7817    rtx ops[4];
7818    ops[0] = operands[0];
7819    ops[1] = operands[1];
7820    ops[2] = operands[2];
7821    ops[3] = operands[3];
7822    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7823      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7824    else
7825      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7826    return "";
7827 }
7828   [(set_attr "length" "8")])
7829
7830 ;;
7831 ;; [vldrhq_s, vldrhq_u]
7832 ;;
7833 (define_insn "mve_vldrhq_<supf><mode>"
7834   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7835         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7836          VLDRHQ))
7837   ]
7838   "TARGET_HAVE_MVE"
7839 {
7840    rtx ops[2];
7841    int regno = REGNO (operands[0]);
7842    ops[0] = gen_rtx_REG (TImode, regno);
7843    ops[1]  = operands[1];
7844    if (<V_sz_elem> == 16)
7845      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7846    else
7847      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7848    return "";
7849 }
7850   [(set_attr "length" "4")])
7851
7852 ;;
7853 ;; [vldrhq_z_f]
7854 ;;
7855 (define_insn "mve_vldrhq_z_fv8hf"
7856   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7857         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7858         (match_operand:HI 2 "vpr_register_operand" "Up")]
7859          VLDRHQ_F))
7860   ]
7861   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7862 {
7863    rtx ops[2];
7864    int regno = REGNO (operands[0]);
7865    ops[0] = gen_rtx_REG (TImode, regno);
7866    ops[1]  = operands[1];
7867    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7868    return "";
7869 }
7870   [(set_attr "length" "8")])
7871
7872 ;;
7873 ;; [vldrhq_z_s vldrhq_z_u]
7874 ;;
7875 (define_insn "mve_vldrhq_z_<supf><mode>"
7876   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7877         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7878         (match_operand:HI 2 "vpr_register_operand" "Up")]
7879          VLDRHQ))
7880   ]
7881   "TARGET_HAVE_MVE"
7882 {
7883    rtx ops[2];
7884    int regno = REGNO (operands[0]);
7885    ops[0] = gen_rtx_REG (TImode, regno);
7886    ops[1]  = operands[1];
7887    if (<V_sz_elem> == 16)
7888      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7889    else
7890      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7891    return "";
7892 }
7893   [(set_attr "length" "8")])
7894
7895 ;;
7896 ;; [vldrwq_f]
7897 ;;
7898 (define_insn "mve_vldrwq_fv4sf"
7899   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7900         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7901          VLDRWQ_F))
7902   ]
7903   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7904 {
7905    rtx ops[2];
7906    int regno = REGNO (operands[0]);
7907    ops[0] = gen_rtx_REG (TImode, regno);
7908    ops[1]  = operands[1];
7909    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7910    return "";
7911 }
7912   [(set_attr "length" "4")])
7913
7914 ;;
7915 ;; [vldrwq_s vldrwq_u]
7916 ;;
7917 (define_insn "mve_vldrwq_<supf>v4si"
7918   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7919         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
7920          VLDRWQ))
7921   ]
7922   "TARGET_HAVE_MVE"
7923 {
7924    rtx ops[2];
7925    int regno = REGNO (operands[0]);
7926    ops[0] = gen_rtx_REG (TImode, regno);
7927    ops[1]  = operands[1];
7928    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7929    return "";
7930 }
7931   [(set_attr "length" "4")])
7932
7933 ;;
7934 ;; [vldrwq_z_f]
7935 ;;
7936 (define_insn "mve_vldrwq_z_fv4sf"
7937   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7938         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
7939         (match_operand:HI 2 "vpr_register_operand" "Up")]
7940          VLDRWQ_F))
7941   ]
7942   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7943 {
7944    rtx ops[2];
7945    int regno = REGNO (operands[0]);
7946    ops[0] = gen_rtx_REG (TImode, regno);
7947    ops[1]  = operands[1];
7948    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7949    return "";
7950 }
7951   [(set_attr "length" "8")])
7952
7953 ;;
7954 ;; [vldrwq_z_s vldrwq_z_u]
7955 ;;
7956 (define_insn "mve_vldrwq_z_<supf>v4si"
7957   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7958         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
7959         (match_operand:HI 2 "vpr_register_operand" "Up")]
7960          VLDRWQ))
7961   ]
7962   "TARGET_HAVE_MVE"
7963 {
7964    rtx ops[2];
7965    int regno = REGNO (operands[0]);
7966    ops[0] = gen_rtx_REG (TImode, regno);
7967    ops[1]  = operands[1];
7968    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7969    return "";
7970 }
7971   [(set_attr "length" "8")])
7972
7973 (define_expand "mve_vld1q_f<mode>"
7974   [(match_operand:MVE_0 0 "s_register_operand")
7975    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7976   ]
7977   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
7978 {
7979   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
7980   DONE;
7981 })
7982
7983 (define_expand "mve_vld1q_<supf><mode>"
7984   [(match_operand:MVE_2 0 "s_register_operand")
7985    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
7986   ]
7987   "TARGET_HAVE_MVE"
7988 {
7989   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
7990   DONE;
7991 })
7992
7993 ;;
7994 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
7995 ;;
7996 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
7997   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7998         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7999                       (match_operand:SI 2 "immediate_operand" "i")]
8000          VLDRDGBQ))
8001   ]
8002   "TARGET_HAVE_MVE"
8003 {
8004    rtx ops[3];
8005    ops[0] = operands[0];
8006    ops[1] = operands[1];
8007    ops[2] = operands[2];
8008    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8009    return "";
8010 }
8011   [(set_attr "length" "4")])
8012
8013 ;;
8014 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8015 ;;
8016 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8017   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8018         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8019                       (match_operand:SI 2 "immediate_operand" "i")
8020                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8021          VLDRDGBQ))
8022   ]
8023   "TARGET_HAVE_MVE"
8024 {
8025    rtx ops[3];
8026    ops[0] = operands[0];
8027    ops[1] = operands[1];
8028    ops[2] = operands[2];
8029    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8030    return "";
8031 }
8032   [(set_attr "length" "8")])
8033
8034 ;;
8035 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8036 ;;
8037 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8038  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8039        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8040                      (match_operand:V2DI 2 "s_register_operand" "w")]
8041         VLDRDGOQ))
8042  ]
8043  "TARGET_HAVE_MVE"
8044 {
8045   rtx ops[3];
8046   ops[0] = operands[0];
8047   ops[1] = operands[1];
8048   ops[2] = operands[2];
8049   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8050   return "";
8051 }
8052  [(set_attr "length" "4")])
8053
8054 ;;
8055 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8056 ;;
8057 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8058  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8059        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8060                      (match_operand:V2DI 2 "s_register_operand" "w")
8061                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8062         VLDRDGOQ))
8063  ]
8064  "TARGET_HAVE_MVE"
8065 {
8066   rtx ops[3];
8067   ops[0] = operands[0];
8068   ops[1] = operands[1];
8069   ops[2] = operands[2];
8070   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8071   return "";
8072 }
8073  [(set_attr "length" "8")])
8074
8075 ;;
8076 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8077 ;;
8078 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8079   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8080         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8081                       (match_operand:V2DI 2 "s_register_operand" "w")]
8082          VLDRDGSOQ))
8083   ]
8084   "TARGET_HAVE_MVE"
8085 {
8086    rtx ops[3];
8087    ops[0] = operands[0];
8088    ops[1] = operands[1];
8089    ops[2] = operands[2];
8090    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8091    return "";
8092 }
8093   [(set_attr "length" "4")])
8094
8095 ;;
8096 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8097 ;;
8098 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8099   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8100         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8101                       (match_operand:V2DI 2 "s_register_operand" "w")
8102                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8103          VLDRDGSOQ))
8104   ]
8105   "TARGET_HAVE_MVE"
8106 {
8107    rtx ops[3];
8108    ops[0] = operands[0];
8109    ops[1] = operands[1];
8110    ops[2] = operands[2];
8111    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8112    return "";
8113 }
8114   [(set_attr "length" "8")])
8115
8116 ;;
8117 ;; [vldrhq_gather_offset_f]
8118 ;;
8119 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8120   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8121         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8122                       (match_operand:V8HI 2 "s_register_operand" "w")]
8123          VLDRHQGO_F))
8124   ]
8125   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8126 {
8127    rtx ops[3];
8128    ops[0] = operands[0];
8129    ops[1] = operands[1];
8130    ops[2] = operands[2];
8131    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8132    return "";
8133 }
8134   [(set_attr "length" "4")])
8135
8136 ;;
8137 ;; [vldrhq_gather_offset_z_f]
8138 ;;
8139 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8140   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8141         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8142                       (match_operand:V8HI 2 "s_register_operand" "w")
8143                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8144          VLDRHQGO_F))
8145   ]
8146   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8147 {
8148    rtx ops[4];
8149    ops[0] = operands[0];
8150    ops[1] = operands[1];
8151    ops[2] = operands[2];
8152    ops[3] = operands[3];
8153    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8154    return "";
8155 }
8156   [(set_attr "length" "8")])
8157
8158 ;;
8159 ;; [vldrhq_gather_shifted_offset_f]
8160 ;;
8161 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8162   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8163         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8164                       (match_operand:V8HI 2 "s_register_operand" "w")]
8165          VLDRHQGSO_F))
8166   ]
8167   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8168 {
8169    rtx ops[3];
8170    ops[0] = operands[0];
8171    ops[1] = operands[1];
8172    ops[2] = operands[2];
8173    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8174    return "";
8175 }
8176   [(set_attr "length" "4")])
8177
8178 ;;
8179 ;; [vldrhq_gather_shifted_offset_z_f]
8180 ;;
8181 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8182   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8183         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8184                       (match_operand:V8HI 2 "s_register_operand" "w")
8185                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8186          VLDRHQGSO_F))
8187   ]
8188   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8189 {
8190    rtx ops[4];
8191    ops[0] = operands[0];
8192    ops[1] = operands[1];
8193    ops[2] = operands[2];
8194    ops[3] = operands[3];
8195    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8196    return "";
8197 }
8198   [(set_attr "length" "8")])
8199
8200 ;;
8201 ;; [vldrwq_gather_base_f]
8202 ;;
8203 (define_insn "mve_vldrwq_gather_base_fv4sf"
8204   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8205         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8206                       (match_operand:SI 2 "immediate_operand" "i")]
8207          VLDRWQGB_F))
8208   ]
8209   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8210 {
8211    rtx ops[3];
8212    ops[0] = operands[0];
8213    ops[1] = operands[1];
8214    ops[2] = operands[2];
8215    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8216    return "";
8217 }
8218   [(set_attr "length" "4")])
8219
8220 ;;
8221 ;; [vldrwq_gather_base_z_f]
8222 ;;
8223 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8224   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8225         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8226                       (match_operand:SI 2 "immediate_operand" "i")
8227                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8228          VLDRWQGB_F))
8229   ]
8230   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8231 {
8232    rtx ops[3];
8233    ops[0] = operands[0];
8234    ops[1] = operands[1];
8235    ops[2] = operands[2];
8236    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8237    return "";
8238 }
8239   [(set_attr "length" "8")])
8240
8241 ;;
8242 ;; [vldrwq_gather_offset_f]
8243 ;;
8244 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8245   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8246         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8247                        (match_operand:V4SI 2 "s_register_operand" "w")]
8248          VLDRWQGO_F))
8249   ]
8250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8251 {
8252    rtx ops[3];
8253    ops[0] = operands[0];
8254    ops[1] = operands[1];
8255    ops[2] = operands[2];
8256    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8257    return "";
8258 }
8259   [(set_attr "length" "4")])
8260
8261 ;;
8262 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8263 ;;
8264 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8265   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8266         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8267                        (match_operand:V4SI 2 "s_register_operand" "w")]
8268          VLDRWGOQ))
8269   ]
8270   "TARGET_HAVE_MVE"
8271 {
8272    rtx ops[3];
8273    ops[0] = operands[0];
8274    ops[1] = operands[1];
8275    ops[2] = operands[2];
8276    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8277    return "";
8278 }
8279   [(set_attr "length" "4")])
8280
8281 ;;
8282 ;; [vldrwq_gather_offset_z_f]
8283 ;;
8284 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8285   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8286         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8287                       (match_operand:V4SI 2 "s_register_operand" "w")
8288                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8289          VLDRWQGO_F))
8290   ]
8291   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8292 {
8293    rtx ops[4];
8294    ops[0] = operands[0];
8295    ops[1] = operands[1];
8296    ops[2] = operands[2];
8297    ops[3] = operands[3];
8298    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8299    return "";
8300 }
8301   [(set_attr "length" "8")])
8302
8303 ;;
8304 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8305 ;;
8306 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8307   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8308         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8309                       (match_operand:V4SI 2 "s_register_operand" "w")
8310                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8311          VLDRWGOQ))
8312   ]
8313   "TARGET_HAVE_MVE"
8314 {
8315    rtx ops[4];
8316    ops[0] = operands[0];
8317    ops[1] = operands[1];
8318    ops[2] = operands[2];
8319    ops[3] = operands[3];
8320    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8321    return "";
8322 }
8323   [(set_attr "length" "8")])
8324
8325 ;;
8326 ;; [vldrwq_gather_shifted_offset_f]
8327 ;;
8328 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8329   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8330         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8331                       (match_operand:V4SI 2 "s_register_operand" "w")]
8332          VLDRWQGSO_F))
8333   ]
8334   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8335 {
8336    rtx ops[3];
8337    ops[0] = operands[0];
8338    ops[1] = operands[1];
8339    ops[2] = operands[2];
8340    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8341    return "";
8342 }
8343   [(set_attr "length" "4")])
8344
8345 ;;
8346 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8347 ;;
8348 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8349   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8350         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8351                       (match_operand:V4SI 2 "s_register_operand" "w")]
8352          VLDRWGSOQ))
8353   ]
8354   "TARGET_HAVE_MVE"
8355 {
8356    rtx ops[3];
8357    ops[0] = operands[0];
8358    ops[1] = operands[1];
8359    ops[2] = operands[2];
8360    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8361    return "";
8362 }
8363   [(set_attr "length" "4")])
8364
8365 ;;
8366 ;; [vldrwq_gather_shifted_offset_z_f]
8367 ;;
8368 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8369   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8370         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8371                       (match_operand:V4SI 2 "s_register_operand" "w")
8372                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8373          VLDRWQGSO_F))
8374   ]
8375   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8376 {
8377    rtx ops[4];
8378    ops[0] = operands[0];
8379    ops[1] = operands[1];
8380    ops[2] = operands[2];
8381    ops[3] = operands[3];
8382    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8383    return "";
8384 }
8385   [(set_attr "length" "8")])
8386
8387 ;;
8388 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8389 ;;
8390 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8391   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8392         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8393                       (match_operand:V4SI 2 "s_register_operand" "w")
8394                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8395          VLDRWGSOQ))
8396   ]
8397   "TARGET_HAVE_MVE"
8398 {
8399    rtx ops[4];
8400    ops[0] = operands[0];
8401    ops[1] = operands[1];
8402    ops[2] = operands[2];
8403    ops[3] = operands[3];
8404    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8405    return "";
8406 }
8407   [(set_attr "length" "8")])
8408
8409 ;;
8410 ;; [vstrhq_f]
8411 ;;
8412 (define_insn "mve_vstrhq_fv8hf"
8413   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8414         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8415          VSTRHQ_F))
8416   ]
8417   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8418 {
8419    rtx ops[2];
8420    int regno = REGNO (operands[1]);
8421    ops[1] = gen_rtx_REG (TImode, regno);
8422    ops[0]  = operands[0];
8423    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8424    return "";
8425 }
8426   [(set_attr "length" "4")])
8427
8428 ;;
8429 ;; [vstrhq_p_f]
8430 ;;
8431 (define_insn "mve_vstrhq_p_fv8hf"
8432   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8433         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8434                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8435          VSTRHQ_F))
8436   ]
8437   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8438 {
8439    rtx ops[2];
8440    int regno = REGNO (operands[1]);
8441    ops[1] = gen_rtx_REG (TImode, regno);
8442    ops[0]  = operands[0];
8443    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8444    return "";
8445 }
8446   [(set_attr "length" "8")])
8447
8448 ;;
8449 ;; [vstrhq_p_s vstrhq_p_u]
8450 ;;
8451 (define_insn "mve_vstrhq_p_<supf><mode>"
8452   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8453         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8454                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8455          VSTRHQ))
8456   ]
8457   "TARGET_HAVE_MVE"
8458 {
8459    rtx ops[2];
8460    int regno = REGNO (operands[1]);
8461    ops[1] = gen_rtx_REG (TImode, regno);
8462    ops[0]  = operands[0];
8463    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8464    return "";
8465 }
8466   [(set_attr "length" "8")])
8467
8468 ;;
8469 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8470 ;;
8471 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8472   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8473    (match_operand:MVE_6 1 "s_register_operand")
8474    (match_operand:MVE_6 2 "s_register_operand")
8475    (match_operand:HI 3 "vpr_register_operand")
8476    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8477   "TARGET_HAVE_MVE"
8478 {
8479   rtx ind = XEXP (operands[0], 0);
8480   gcc_assert (REG_P (ind));
8481   emit_insn (
8482     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8483                                                        operands[2],
8484                                                        operands[3]));
8485   DONE;
8486 })
8487
8488 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8489   [(set (mem:BLK (scratch))
8490         (unspec:BLK
8491           [(match_operand:SI 0 "register_operand" "r")
8492            (match_operand:MVE_6 1 "s_register_operand" "w")
8493            (match_operand:MVE_6 2 "s_register_operand" "w")
8494            (match_operand:HI 3 "vpr_register_operand" "Up")]
8495           VSTRHSOQ))]
8496   "TARGET_HAVE_MVE"
8497   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8498   [(set_attr "length" "8")])
8499
8500 ;;
8501 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8502 ;;
8503 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8504   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8505    (match_operand:MVE_6 1 "s_register_operand")
8506    (match_operand:MVE_6 2 "s_register_operand")
8507    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8508   "TARGET_HAVE_MVE"
8509 {
8510   rtx ind = XEXP (operands[0], 0);
8511   gcc_assert (REG_P (ind));
8512   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8513                                                               operands[2]));
8514   DONE;
8515 })
8516
8517 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8518   [(set (mem:BLK (scratch))
8519         (unspec:BLK
8520           [(match_operand:SI 0 "register_operand" "r")
8521            (match_operand:MVE_6 1 "s_register_operand" "w")
8522            (match_operand:MVE_6 2 "s_register_operand" "w")]
8523           VSTRHSOQ))]
8524   "TARGET_HAVE_MVE"
8525   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8526   [(set_attr "length" "4")])
8527
8528 ;;
8529 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8530 ;;
8531 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8532   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8533    (match_operand:MVE_6 1 "s_register_operand")
8534    (match_operand:MVE_6 2 "s_register_operand")
8535    (match_operand:HI 3 "vpr_register_operand")
8536    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8537   "TARGET_HAVE_MVE"
8538 {
8539   rtx ind = XEXP (operands[0], 0);
8540   gcc_assert (REG_P (ind));
8541   emit_insn (
8542     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8543                                                                operands[2],
8544                                                                operands[3]));
8545   DONE;
8546 })
8547
8548 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8549   [(set (mem:BLK (scratch))
8550         (unspec:BLK
8551           [(match_operand:SI 0 "register_operand" "r")
8552            (match_operand:MVE_6 1 "s_register_operand" "w")
8553            (match_operand:MVE_6 2 "s_register_operand" "w")
8554            (match_operand:HI 3 "vpr_register_operand" "Up")]
8555           VSTRHSSOQ))]
8556   "TARGET_HAVE_MVE"
8557   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8558   [(set_attr "length" "8")])
8559
8560 ;;
8561 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8562 ;;
8563 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8564   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8565    (match_operand:MVE_6 1 "s_register_operand")
8566    (match_operand:MVE_6 2 "s_register_operand")
8567    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8568   "TARGET_HAVE_MVE"
8569 {
8570   rtx ind = XEXP (operands[0], 0);
8571   gcc_assert (REG_P (ind));
8572   emit_insn (
8573     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8574                                                              operands[2]));
8575   DONE;
8576 })
8577
8578 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8579   [(set (mem:BLK (scratch))
8580         (unspec:BLK
8581           [(match_operand:SI 0 "register_operand" "r")
8582            (match_operand:MVE_6 1 "s_register_operand" "w")
8583            (match_operand:MVE_6 2 "s_register_operand" "w")]
8584           VSTRHSSOQ))]
8585   "TARGET_HAVE_MVE"
8586   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8587   [(set_attr "length" "4")])
8588
8589 ;;
8590 ;; [vstrhq_s, vstrhq_u]
8591 ;;
8592 (define_insn "mve_vstrhq_<supf><mode>"
8593   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8594         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8595          VSTRHQ))
8596   ]
8597   "TARGET_HAVE_MVE"
8598 {
8599    rtx ops[2];
8600    int regno = REGNO (operands[1]);
8601    ops[1] = gen_rtx_REG (TImode, regno);
8602    ops[0]  = operands[0];
8603    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8604    return "";
8605 }
8606   [(set_attr "length" "4")])
8607
8608 ;;
8609 ;; [vstrwq_f]
8610 ;;
8611 (define_insn "mve_vstrwq_fv4sf"
8612   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8613         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8614          VSTRWQ_F))
8615   ]
8616   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8617 {
8618    rtx ops[2];
8619    int regno = REGNO (operands[1]);
8620    ops[1] = gen_rtx_REG (TImode, regno);
8621    ops[0]  = operands[0];
8622    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8623    return "";
8624 }
8625   [(set_attr "length" "4")])
8626
8627 ;;
8628 ;; [vstrwq_p_f]
8629 ;;
8630 (define_insn "mve_vstrwq_p_fv4sf"
8631   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8632         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8633                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8634          VSTRWQ_F))
8635   ]
8636   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8637 {
8638    rtx ops[2];
8639    int regno = REGNO (operands[1]);
8640    ops[1] = gen_rtx_REG (TImode, regno);
8641    ops[0]  = operands[0];
8642    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8643    return "";
8644 }
8645   [(set_attr "length" "8")])
8646
8647 ;;
8648 ;; [vstrwq_p_s vstrwq_p_u]
8649 ;;
8650 (define_insn "mve_vstrwq_p_<supf>v4si"
8651   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8652         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8653                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8654          VSTRWQ))
8655   ]
8656   "TARGET_HAVE_MVE"
8657 {
8658    rtx ops[2];
8659    int regno = REGNO (operands[1]);
8660    ops[1] = gen_rtx_REG (TImode, regno);
8661    ops[0]  = operands[0];
8662    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8663    return "";
8664 }
8665   [(set_attr "length" "8")])
8666
8667 ;;
8668 ;; [vstrwq_s vstrwq_u]
8669 ;;
8670 (define_insn "mve_vstrwq_<supf>v4si"
8671   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8672         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8673          VSTRWQ))
8674   ]
8675   "TARGET_HAVE_MVE"
8676 {
8677    rtx ops[2];
8678    int regno = REGNO (operands[1]);
8679    ops[1] = gen_rtx_REG (TImode, regno);
8680    ops[0]  = operands[0];
8681    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8682    return "";
8683 }
8684   [(set_attr "length" "4")])
8685
8686 (define_expand "mve_vst1q_f<mode>"
8687   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8688    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8689   ]
8690   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8691 {
8692   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8693   DONE;
8694 })
8695
8696 (define_expand "mve_vst1q_<supf><mode>"
8697   [(match_operand:MVE_2 0 "mve_memory_operand")
8698    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8699   ]
8700   "TARGET_HAVE_MVE"
8701 {
8702   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8703   DONE;
8704 })
8705
8706 ;;
8707 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8708 ;;
8709 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8710   [(set (mem:BLK (scratch))
8711         (unspec:BLK
8712                 [(match_operand:V2DI 0 "s_register_operand" "w")
8713                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8714                  (match_operand:V2DI 2 "s_register_operand" "w")
8715                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8716          VSTRDSBQ))
8717   ]
8718   "TARGET_HAVE_MVE"
8719 {
8720    rtx ops[3];
8721    ops[0] = operands[0];
8722    ops[1] = operands[1];
8723    ops[2] = operands[2];
8724    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8725    return "";
8726 }
8727   [(set_attr "length" "8")])
8728
8729 ;;
8730 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8731 ;;
8732 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8733   [(set (mem:BLK (scratch))
8734         (unspec:BLK
8735                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8736                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8737                  (match_operand:V2DI 2 "s_register_operand" "w")]
8738          VSTRDSBQ))
8739   ]
8740   "TARGET_HAVE_MVE"
8741 {
8742    rtx ops[3];
8743    ops[0] = operands[0];
8744    ops[1] = operands[1];
8745    ops[2] = operands[2];
8746    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8747    return "";
8748 }
8749   [(set_attr "length" "4")])
8750
8751 ;;
8752 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8753 ;;
8754 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8755   [(match_operand:V2DI 0 "mve_scatter_memory")
8756    (match_operand:V2DI 1 "s_register_operand")
8757    (match_operand:V2DI 2 "s_register_operand")
8758    (match_operand:HI 3 "vpr_register_operand")
8759    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8760   "TARGET_HAVE_MVE"
8761 {
8762   rtx ind = XEXP (operands[0], 0);
8763   gcc_assert (REG_P (ind));
8764   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8765                                                               operands[2],
8766                                                               operands[3]));
8767   DONE;
8768 })
8769
8770 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8771   [(set (mem:BLK (scratch))
8772         (unspec:BLK
8773           [(match_operand:SI 0 "register_operand" "r")
8774            (match_operand:V2DI 1 "s_register_operand" "w")
8775            (match_operand:V2DI 2 "s_register_operand" "w")
8776            (match_operand:HI 3 "vpr_register_operand" "Up")]
8777           VSTRDSOQ))]
8778   "TARGET_HAVE_MVE"
8779   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8780   [(set_attr "length" "8")])
8781
8782 ;;
8783 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8784 ;;
8785 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8786   [(match_operand:V2DI 0 "mve_scatter_memory")
8787    (match_operand:V2DI 1 "s_register_operand")
8788    (match_operand:V2DI 2 "s_register_operand")
8789    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8790   "TARGET_HAVE_MVE"
8791 {
8792   rtx ind = XEXP (operands[0], 0);
8793   gcc_assert (REG_P (ind));
8794   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8795                                                             operands[2]));
8796   DONE;
8797 })
8798
8799 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8800   [(set (mem:BLK (scratch))
8801         (unspec:BLK
8802           [(match_operand:SI 0 "register_operand" "r")
8803            (match_operand:V2DI 1 "s_register_operand" "w")
8804            (match_operand:V2DI 2 "s_register_operand" "w")]
8805           VSTRDSOQ))]
8806   "TARGET_HAVE_MVE"
8807   "vstrd.64\t%q2, [%0, %q1]"
8808   [(set_attr "length" "4")])
8809
8810 ;;
8811 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8812 ;;
8813 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8814   [(match_operand:V2DI 0 "mve_scatter_memory")
8815    (match_operand:V2DI 1 "s_register_operand")
8816    (match_operand:V2DI 2 "s_register_operand")
8817    (match_operand:HI 3 "vpr_register_operand")
8818    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8819   "TARGET_HAVE_MVE"
8820 {
8821   rtx ind = XEXP (operands[0], 0);
8822   gcc_assert (REG_P (ind));
8823   emit_insn (
8824     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8825                                                              operands[2],
8826                                                              operands[3]));
8827   DONE;
8828 })
8829
8830 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8831   [(set (mem:BLK (scratch))
8832         (unspec:BLK
8833           [(match_operand:SI 0 "register_operand" "r")
8834            (match_operand:V2DI 1 "s_register_operand" "w")
8835            (match_operand:V2DI 2 "s_register_operand" "w")
8836            (match_operand:HI 3 "vpr_register_operand" "Up")]
8837           VSTRDSSOQ))]
8838   "TARGET_HAVE_MVE"
8839   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8840   [(set_attr "length" "8")])
8841
8842 ;;
8843 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8844 ;;
8845 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8846   [(match_operand:V2DI 0 "mve_scatter_memory")
8847    (match_operand:V2DI 1 "s_register_operand")
8848    (match_operand:V2DI 2 "s_register_operand")
8849    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8850   "TARGET_HAVE_MVE"
8851 {
8852   rtx ind = XEXP (operands[0], 0);
8853   gcc_assert (REG_P (ind));
8854   emit_insn (
8855     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8856                                                            operands[2]));
8857   DONE;
8858 })
8859
8860 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8861   [(set (mem:BLK (scratch))
8862         (unspec:BLK
8863           [(match_operand:SI 0 "register_operand" "r")
8864            (match_operand:V2DI 1 "s_register_operand" "w")
8865            (match_operand:V2DI 2 "s_register_operand" "w")]
8866           VSTRDSSOQ))]
8867   "TARGET_HAVE_MVE"
8868   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8869   [(set_attr "length" "4")])
8870
8871 ;;
8872 ;; [vstrhq_scatter_offset_f]
8873 ;;
8874 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8875   [(match_operand:V8HI 0 "mve_scatter_memory")
8876    (match_operand:V8HI 1 "s_register_operand")
8877    (match_operand:V8HF 2 "s_register_operand")
8878    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8879   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8880 {
8881   rtx ind = XEXP (operands[0], 0);
8882   gcc_assert (REG_P (ind));
8883   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8884                                                        operands[2]));
8885   DONE;
8886 })
8887
8888 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8889   [(set (mem:BLK (scratch))
8890         (unspec:BLK
8891           [(match_operand:SI 0 "register_operand" "r")
8892            (match_operand:V8HI 1 "s_register_operand" "w")
8893            (match_operand:V8HF 2 "s_register_operand" "w")]
8894           VSTRHQSO_F))]
8895   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8896   "vstrh.16\t%q2, [%0, %q1]"
8897   [(set_attr "length" "4")])
8898
8899 ;;
8900 ;; [vstrhq_scatter_offset_p_f]
8901 ;;
8902 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8903   [(match_operand:V8HI 0 "mve_scatter_memory")
8904    (match_operand:V8HI 1 "s_register_operand")
8905    (match_operand:V8HF 2 "s_register_operand")
8906    (match_operand:HI 3 "vpr_register_operand")
8907    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8908   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8909 {
8910   rtx ind = XEXP (operands[0], 0);
8911   gcc_assert (REG_P (ind));
8912   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8913                                                          operands[2],
8914                                                          operands[3]));
8915   DONE;
8916 })
8917
8918 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8919   [(set (mem:BLK (scratch))
8920         (unspec:BLK
8921           [(match_operand:SI 0 "register_operand" "r")
8922            (match_operand:V8HI 1 "s_register_operand" "w")
8923            (match_operand:V8HF 2 "s_register_operand" "w")
8924            (match_operand:HI 3 "vpr_register_operand" "Up")]
8925           VSTRHQSO_F))]
8926   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8927   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8928   [(set_attr "length" "8")])
8929
8930 ;;
8931 ;; [vstrhq_scatter_shifted_offset_f]
8932 ;;
8933 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8934   [(match_operand:V8HI 0 "memory_operand" "=Us")
8935    (match_operand:V8HI 1 "s_register_operand" "w")
8936    (match_operand:V8HF 2 "s_register_operand" "w")
8937    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8938   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8939 {
8940   rtx ind = XEXP (operands[0], 0);
8941   gcc_assert (REG_P (ind));
8942   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8943                                                                operands[2]));
8944   DONE;
8945 })
8946
8947 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8948   [(set (mem:BLK (scratch))
8949         (unspec:BLK
8950           [(match_operand:SI 0 "register_operand" "r")
8951            (match_operand:V8HI 1 "s_register_operand" "w")
8952            (match_operand:V8HF 2 "s_register_operand" "w")]
8953           VSTRHQSSO_F))]
8954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8955   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8956   [(set_attr "length" "4")])
8957
8958 ;;
8959 ;; [vstrhq_scatter_shifted_offset_p_f]
8960 ;;
8961 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8962   [(match_operand:V8HI 0 "memory_operand" "=Us")
8963    (match_operand:V8HI 1 "s_register_operand" "w")
8964    (match_operand:V8HF 2 "s_register_operand" "w")
8965    (match_operand:HI 3 "vpr_register_operand" "Up")
8966    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8967   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8968 {
8969   rtx ind = XEXP (operands[0], 0);
8970   gcc_assert (REG_P (ind));
8971   emit_insn (
8972     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8973                                                         operands[2],
8974                                                         operands[3]));
8975   DONE;
8976 })
8977
8978 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
8979   [(set (mem:BLK (scratch))
8980         (unspec:BLK
8981           [(match_operand:SI 0 "register_operand" "r")
8982            (match_operand:V8HI 1 "s_register_operand" "w")
8983            (match_operand:V8HF 2 "s_register_operand" "w")
8984            (match_operand:HI 3 "vpr_register_operand" "Up")]
8985           VSTRHQSSO_F))]
8986   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8987   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
8988   [(set_attr "length" "8")])
8989
8990 ;;
8991 ;; [vstrwq_scatter_base_f]
8992 ;;
8993 (define_insn "mve_vstrwq_scatter_base_fv4sf"
8994   [(set (mem:BLK (scratch))
8995         (unspec:BLK
8996                 [(match_operand:V4SI 0 "s_register_operand" "w")
8997                  (match_operand:SI 1 "immediate_operand" "i")
8998                  (match_operand:V4SF 2 "s_register_operand" "w")]
8999          VSTRWQSB_F))
9000   ]
9001   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9002 {
9003    rtx ops[3];
9004    ops[0] = operands[0];
9005    ops[1] = operands[1];
9006    ops[2] = operands[2];
9007    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9008    return "";
9009 }
9010   [(set_attr "length" "4")])
9011
9012 ;;
9013 ;; [vstrwq_scatter_base_p_f]
9014 ;;
9015 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9016   [(set (mem:BLK (scratch))
9017         (unspec:BLK
9018                 [(match_operand:V4SI 0 "s_register_operand" "w")
9019                  (match_operand:SI 1 "immediate_operand" "i")
9020                  (match_operand:V4SF 2 "s_register_operand" "w")
9021                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9022          VSTRWQSB_F))
9023   ]
9024   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9025 {
9026    rtx ops[3];
9027    ops[0] = operands[0];
9028    ops[1] = operands[1];
9029    ops[2] = operands[2];
9030    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9031    return "";
9032 }
9033   [(set_attr "length" "8")])
9034
9035 ;;
9036 ;; [vstrwq_scatter_offset_f]
9037 ;;
9038 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9039   [(match_operand:V4SI 0 "mve_scatter_memory")
9040    (match_operand:V4SI 1 "s_register_operand")
9041    (match_operand:V4SF 2 "s_register_operand")
9042    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9043   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9044 {
9045   rtx ind = XEXP (operands[0], 0);
9046   gcc_assert (REG_P (ind));
9047   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9048                                                        operands[2]));
9049   DONE;
9050 })
9051
9052 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9053   [(set (mem:BLK (scratch))
9054         (unspec:BLK
9055           [(match_operand:SI 0 "register_operand" "r")
9056            (match_operand:V4SI 1 "s_register_operand" "w")
9057            (match_operand:V4SF 2 "s_register_operand" "w")]
9058           VSTRWQSO_F))]
9059   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9060   "vstrw.32\t%q2, [%0, %q1]"
9061   [(set_attr "length" "4")])
9062
9063 ;;
9064 ;; [vstrwq_scatter_offset_p_f]
9065 ;;
9066 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9067   [(match_operand:V4SI 0 "mve_scatter_memory")
9068    (match_operand:V4SI 1 "s_register_operand")
9069    (match_operand:V4SF 2 "s_register_operand")
9070    (match_operand:HI 3 "vpr_register_operand")
9071    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9073 {
9074   rtx ind = XEXP (operands[0], 0);
9075   gcc_assert (REG_P (ind));
9076   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9077                                                          operands[2],
9078                                                          operands[3]));
9079   DONE;
9080 })
9081
9082 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9083   [(set (mem:BLK (scratch))
9084         (unspec:BLK
9085           [(match_operand:SI 0 "register_operand" "r")
9086            (match_operand:V4SI 1 "s_register_operand" "w")
9087            (match_operand:V4SF 2 "s_register_operand" "w")
9088            (match_operand:HI 3 "vpr_register_operand" "Up")]
9089           VSTRWQSO_F))]
9090   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9091   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9092   [(set_attr "length" "8")])
9093
9094 ;;
9095 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9096 ;;
9097 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9098   [(match_operand:V4SI 0 "mve_scatter_memory")
9099    (match_operand:V4SI 1 "s_register_operand")
9100    (match_operand:V4SI 2 "s_register_operand")
9101    (match_operand:HI 3 "vpr_register_operand")
9102    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9103   "TARGET_HAVE_MVE"
9104 {
9105   rtx ind = XEXP (operands[0], 0);
9106   gcc_assert (REG_P (ind));
9107   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9108                                                               operands[2],
9109                                                               operands[3]));
9110   DONE;
9111 })
9112
9113 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9114   [(set (mem:BLK (scratch))
9115         (unspec:BLK
9116           [(match_operand:SI 0 "register_operand" "r")
9117            (match_operand:V4SI 1 "s_register_operand" "w")
9118            (match_operand:V4SI 2 "s_register_operand" "w")
9119            (match_operand:HI 3 "vpr_register_operand" "Up")]
9120           VSTRWSOQ))]
9121   "TARGET_HAVE_MVE"
9122   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9123   [(set_attr "length" "8")])
9124
9125 ;;
9126 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9127 ;;
9128 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9129   [(match_operand:V4SI 0 "mve_scatter_memory")
9130    (match_operand:V4SI 1 "s_register_operand")
9131    (match_operand:V4SI 2 "s_register_operand")
9132    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9133   "TARGET_HAVE_MVE"
9134 {
9135   rtx ind = XEXP (operands[0], 0);
9136   gcc_assert (REG_P (ind));
9137   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9138                                                             operands[2]));
9139   DONE;
9140 })
9141
9142 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9143   [(set (mem:BLK (scratch))
9144         (unspec:BLK
9145           [(match_operand:SI 0 "register_operand" "r")
9146            (match_operand:V4SI 1 "s_register_operand" "w")
9147            (match_operand:V4SI 2 "s_register_operand" "w")]
9148           VSTRWSOQ))]
9149   "TARGET_HAVE_MVE"
9150   "vstrw.32\t%q2, [%0, %q1]"
9151   [(set_attr "length" "4")])
9152
9153 ;;
9154 ;; [vstrwq_scatter_shifted_offset_f]
9155 ;;
9156 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9157   [(match_operand:V4SI 0 "mve_scatter_memory")
9158    (match_operand:V4SI 1 "s_register_operand")
9159    (match_operand:V4SF 2 "s_register_operand")
9160    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9161   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9162 {
9163   rtx ind = XEXP (operands[0], 0);
9164   gcc_assert (REG_P (ind));
9165   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9166                                                                operands[2]));
9167   DONE;
9168 })
9169
9170 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9171   [(set (mem:BLK (scratch))
9172         (unspec:BLK
9173           [(match_operand:SI 0 "register_operand" "r")
9174            (match_operand:V4SI 1 "s_register_operand" "w")
9175            (match_operand:V4SF 2 "s_register_operand" "w")]
9176          VSTRWQSSO_F))]
9177   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9178   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9179   [(set_attr "length" "8")])
9180
9181 ;;
9182 ;; [vstrwq_scatter_shifted_offset_p_f]
9183 ;;
9184 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9185   [(match_operand:V4SI 0 "mve_scatter_memory")
9186    (match_operand:V4SI 1 "s_register_operand")
9187    (match_operand:V4SF 2 "s_register_operand")
9188    (match_operand:HI 3 "vpr_register_operand")
9189    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9190   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9191 {
9192   rtx ind = XEXP (operands[0], 0);
9193   gcc_assert (REG_P (ind));
9194   emit_insn (
9195     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9196                                                         operands[2],
9197                                                         operands[3]));
9198   DONE;
9199 })
9200
9201 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9202   [(set (mem:BLK (scratch))
9203         (unspec:BLK
9204           [(match_operand:SI 0 "register_operand" "r")
9205            (match_operand:V4SI 1 "s_register_operand" "w")
9206            (match_operand:V4SF 2 "s_register_operand" "w")
9207            (match_operand:HI 3 "vpr_register_operand" "Up")]
9208           VSTRWQSSO_F))]
9209   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9210   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9211   [(set_attr "length" "8")])
9212
9213 ;;
9214 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9215 ;;
9216 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9217   [(match_operand:V4SI 0 "mve_scatter_memory")
9218    (match_operand:V4SI 1 "s_register_operand")
9219    (match_operand:V4SI 2 "s_register_operand")
9220    (match_operand:HI 3 "vpr_register_operand")
9221    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9222   "TARGET_HAVE_MVE"
9223 {
9224   rtx ind = XEXP (operands[0], 0);
9225   gcc_assert (REG_P (ind));
9226   emit_insn (
9227     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9228                                                              operands[2],
9229                                                              operands[3]));
9230   DONE;
9231 })
9232
9233 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9234   [(set (mem:BLK (scratch))
9235         (unspec:BLK
9236           [(match_operand:SI 0 "register_operand" "r")
9237            (match_operand:V4SI 1 "s_register_operand" "w")
9238            (match_operand:V4SI 2 "s_register_operand" "w")
9239            (match_operand:HI 3 "vpr_register_operand" "Up")]
9240           VSTRWSSOQ))]
9241   "TARGET_HAVE_MVE"
9242   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9243   [(set_attr "length" "8")])
9244
9245 ;;
9246 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9247 ;;
9248 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9249   [(match_operand:V4SI 0 "mve_scatter_memory")
9250    (match_operand:V4SI 1 "s_register_operand")
9251    (match_operand:V4SI 2 "s_register_operand")
9252    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9253   "TARGET_HAVE_MVE"
9254 {
9255   rtx ind = XEXP (operands[0], 0);
9256   gcc_assert (REG_P (ind));
9257   emit_insn (
9258     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9259                                                            operands[2]));
9260   DONE;
9261 })
9262
9263 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9264   [(set (mem:BLK (scratch))
9265         (unspec:BLK
9266           [(match_operand:SI 0 "register_operand" "r")
9267            (match_operand:V4SI 1 "s_register_operand" "w")
9268            (match_operand:V4SI 2 "s_register_operand" "w")]
9269           VSTRWSSOQ))]
9270   "TARGET_HAVE_MVE"
9271   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9272   [(set_attr "length" "4")])
9273
9274 ;;
9275 ;; [vaddq_s, vaddq_u])
9276 ;;
9277 (define_insn "mve_vaddq<mode>"
9278   [
9279    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9280         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9281                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9282   ]
9283   "TARGET_HAVE_MVE"
9284   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9285   [(set_attr "type" "mve_move")
9286 ])
9287
9288 ;;
9289 ;; [vaddq_f])
9290 ;;
9291 (define_insn "mve_vaddq_f<mode>"
9292   [
9293    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9294         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9295                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9296   ]
9297   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9298   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9299   [(set_attr "type" "mve_move")
9300 ])
9301
9302 ;;
9303 ;; [vidupq_n_u])
9304 ;;
9305 (define_expand "mve_vidupq_n_u<mode>"
9306  [(match_operand:MVE_2 0 "s_register_operand")
9307   (match_operand:SI 1 "s_register_operand")
9308   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9309  "TARGET_HAVE_MVE"
9310 {
9311   rtx temp = gen_reg_rtx (SImode);
9312   emit_move_insn (temp, operands[1]);
9313   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9314   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9315                                           operands[2], inc));
9316   DONE;
9317 })
9318
9319 ;;
9320 ;; [vidupq_u_insn])
9321 ;;
9322 (define_insn "mve_vidupq_u<mode>_insn"
9323  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9324        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9325                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9326          VIDUPQ))
9327   (set (match_operand:SI 1 "s_register_operand" "=Te")
9328        (plus:SI (match_dup 2)
9329                 (match_operand:SI 4 "immediate_operand" "i")))]
9330  "TARGET_HAVE_MVE"
9331  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9332
9333 ;;
9334 ;; [vidupq_m_n_u])
9335 ;;
9336 (define_expand "mve_vidupq_m_n_u<mode>"
9337   [(match_operand:MVE_2 0 "s_register_operand")
9338    (match_operand:MVE_2 1 "s_register_operand")
9339    (match_operand:SI 2 "s_register_operand")
9340    (match_operand:SI 3 "mve_imm_selective_upto_8")
9341    (match_operand:HI 4 "vpr_register_operand")]
9342   "TARGET_HAVE_MVE"
9343 {
9344   rtx temp = gen_reg_rtx (SImode);
9345   emit_move_insn (temp, operands[2]);
9346   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9347   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9348                                              operands[2], operands[3],
9349                                              operands[4], inc));
9350   DONE;
9351 })
9352
9353 ;;
9354 ;; [vidupq_m_wb_u_insn])
9355 ;;
9356 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9357  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9358        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9359                       (match_operand:SI 3 "s_register_operand" "2")
9360                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9361                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9362         VIDUPQ_M))
9363   (set (match_operand:SI 2 "s_register_operand" "=Te")
9364        (plus:SI (match_dup 3)
9365                 (match_operand:SI 6 "immediate_operand" "i")))]
9366  "TARGET_HAVE_MVE"
9367  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9368  [(set_attr "length""8")])
9369
9370 ;;
9371 ;; [vddupq_n_u])
9372 ;;
9373 (define_expand "mve_vddupq_n_u<mode>"
9374  [(match_operand:MVE_2 0 "s_register_operand")
9375   (match_operand:SI 1 "s_register_operand")
9376   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9377  "TARGET_HAVE_MVE"
9378 {
9379   rtx temp = gen_reg_rtx (SImode);
9380   emit_move_insn (temp, operands[1]);
9381   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9382   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9383                                           operands[2], inc));
9384   DONE;
9385 })
9386
9387 ;;
9388 ;; [vddupq_u_insn])
9389 ;;
9390 (define_insn "mve_vddupq_u<mode>_insn"
9391  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9392        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9393                       (match_operand:SI 3 "immediate_operand" "i")]
9394         VDDUPQ))
9395   (set (match_operand:SI 1 "s_register_operand" "=Te")
9396        (minus:SI (match_dup 2)
9397                  (match_operand:SI 4 "immediate_operand" "i")))]
9398  "TARGET_HAVE_MVE"
9399  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9400
9401 ;;
9402 ;; [vddupq_m_n_u])
9403 ;;
9404 (define_expand "mve_vddupq_m_n_u<mode>"
9405   [(match_operand:MVE_2 0 "s_register_operand")
9406    (match_operand:MVE_2 1 "s_register_operand")
9407    (match_operand:SI 2 "s_register_operand")
9408    (match_operand:SI 3 "mve_imm_selective_upto_8")
9409    (match_operand:HI 4 "vpr_register_operand")]
9410   "TARGET_HAVE_MVE"
9411 {
9412   rtx temp = gen_reg_rtx (SImode);
9413   emit_move_insn (temp, operands[2]);
9414   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9415   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9416                                              operands[2], operands[3],
9417                                              operands[4], inc));
9418   DONE;
9419 })
9420
9421 ;;
9422 ;; [vddupq_m_wb_u_insn])
9423 ;;
9424 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9425  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9426        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9427                       (match_operand:SI 3 "s_register_operand" "2")
9428                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9429                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9430         VDDUPQ_M))
9431   (set (match_operand:SI 2 "s_register_operand" "=Te")
9432        (minus:SI (match_dup 3)
9433                  (match_operand:SI 6 "immediate_operand" "i")))]
9434  "TARGET_HAVE_MVE"
9435  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9436  [(set_attr "length""8")])
9437
9438 ;;
9439 ;; [vdwdupq_n_u])
9440 ;;
9441 (define_expand "mve_vdwdupq_n_u<mode>"
9442  [(match_operand:MVE_2 0 "s_register_operand")
9443   (match_operand:SI 1 "s_register_operand")
9444   (match_operand:DI 2 "s_register_operand")
9445   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9446  "TARGET_HAVE_MVE"
9447 {
9448   rtx ignore_wb = gen_reg_rtx (SImode);
9449   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9450                                               operands[1], operands[2],
9451                                               operands[3]));
9452   DONE;
9453 })
9454
9455 ;;
9456 ;; [vdwdupq_wb_u])
9457 ;;
9458 (define_expand "mve_vdwdupq_wb_u<mode>"
9459  [(match_operand:SI 0 "s_register_operand")
9460   (match_operand:SI 1 "s_register_operand")
9461   (match_operand:DI 2 "s_register_operand")
9462   (match_operand:SI 3 "mve_imm_selective_upto_8")
9463   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9464  "TARGET_HAVE_MVE"
9465 {
9466   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9467   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9468                                               operands[1], operands[2],
9469                                               operands[3]));
9470   DONE;
9471 })
9472
9473 ;;
9474 ;; [vdwdupq_wb_u_insn])
9475 ;;
9476 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9477   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9478         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9479                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9480                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9481          VDWDUPQ))
9482    (set (match_operand:SI 1 "s_register_operand" "=Te")
9483         (unspec:SI [(match_dup 2)
9484                     (subreg:SI (match_dup 3) 4)
9485                     (match_dup 4)]
9486          VDWDUPQ))]
9487   "TARGET_HAVE_MVE"
9488   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9489 )
9490
9491 ;;
9492 ;; [vdwdupq_m_n_u])
9493 ;;
9494 (define_expand "mve_vdwdupq_m_n_u<mode>"
9495  [(match_operand:MVE_2 0 "s_register_operand")
9496   (match_operand:MVE_2 1 "s_register_operand")
9497   (match_operand:SI 2 "s_register_operand")
9498   (match_operand:DI 3 "s_register_operand")
9499   (match_operand:SI 4 "mve_imm_selective_upto_8")
9500   (match_operand:HI 5 "vpr_register_operand")]
9501  "TARGET_HAVE_MVE"
9502 {
9503   rtx ignore_wb = gen_reg_rtx (SImode);
9504   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9505                                                 operands[1], operands[2],
9506                                                 operands[3], operands[4],
9507                                                 operands[5]));
9508   DONE;
9509 })
9510
9511 ;;
9512 ;; [vdwdupq_m_wb_u])
9513 ;;
9514 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9515  [(match_operand:SI 0 "s_register_operand")
9516   (match_operand:MVE_2 1 "s_register_operand")
9517   (match_operand:SI 2 "s_register_operand")
9518   (match_operand:DI 3 "s_register_operand")
9519   (match_operand:SI 4 "mve_imm_selective_upto_8")
9520   (match_operand:HI 5 "vpr_register_operand")]
9521  "TARGET_HAVE_MVE"
9522 {
9523   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9524   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9525                                                 operands[1], operands[2],
9526                                                 operands[3], operands[4],
9527                                                 operands[5]));
9528   DONE;
9529 })
9530
9531 ;;
9532 ;; [vdwdupq_m_wb_u_insn])
9533 ;;
9534 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9535   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9536         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9537                        (match_operand:SI 3 "s_register_operand" "1")
9538                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9539                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9540                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9541          VDWDUPQ_M))
9542    (set (match_operand:SI 1 "s_register_operand" "=Te")
9543         (unspec:SI [(match_dup 2)
9544                     (match_dup 3)
9545                     (subreg:SI (match_dup 4) 4)
9546                     (match_dup 5)
9547                     (match_dup 6)]
9548          VDWDUPQ_M))
9549   ]
9550   "TARGET_HAVE_MVE"
9551   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9552   [(set_attr "type" "mve_move")
9553    (set_attr "length""8")])
9554
9555 ;;
9556 ;; [viwdupq_n_u])
9557 ;;
9558 (define_expand "mve_viwdupq_n_u<mode>"
9559  [(match_operand:MVE_2 0 "s_register_operand")
9560   (match_operand:SI 1 "s_register_operand")
9561   (match_operand:DI 2 "s_register_operand")
9562   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9563  "TARGET_HAVE_MVE"
9564 {
9565   rtx ignore_wb = gen_reg_rtx (SImode);
9566   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9567                                               operands[1], operands[2],
9568                                               operands[3]));
9569   DONE;
9570 })
9571
9572 ;;
9573 ;; [viwdupq_wb_u])
9574 ;;
9575 (define_expand "mve_viwdupq_wb_u<mode>"
9576  [(match_operand:SI 0 "s_register_operand")
9577   (match_operand:SI 1 "s_register_operand")
9578   (match_operand:DI 2 "s_register_operand")
9579   (match_operand:SI 3 "mve_imm_selective_upto_8")
9580   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9581  "TARGET_HAVE_MVE"
9582 {
9583   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9584   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9585                                               operands[1], operands[2],
9586                                               operands[3]));
9587   DONE;
9588 })
9589
9590 ;;
9591 ;; [viwdupq_wb_u_insn])
9592 ;;
9593 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9594   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9595         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9596                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9597                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9598          VIWDUPQ))
9599    (set (match_operand:SI 1 "s_register_operand" "=Te")
9600         (unspec:SI [(match_dup 2)
9601                     (subreg:SI (match_dup 3) 4)
9602                     (match_dup 4)]
9603          VIWDUPQ))]
9604   "TARGET_HAVE_MVE"
9605   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9606 )
9607
9608 ;;
9609 ;; [viwdupq_m_n_u])
9610 ;;
9611 (define_expand "mve_viwdupq_m_n_u<mode>"
9612  [(match_operand:MVE_2 0 "s_register_operand")
9613   (match_operand:MVE_2 1 "s_register_operand")
9614   (match_operand:SI 2 "s_register_operand")
9615   (match_operand:DI 3 "s_register_operand")
9616   (match_operand:SI 4 "mve_imm_selective_upto_8")
9617   (match_operand:HI 5 "vpr_register_operand")]
9618  "TARGET_HAVE_MVE"
9619 {
9620   rtx ignore_wb = gen_reg_rtx (SImode);
9621   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9622                                                 operands[1], operands[2],
9623                                                 operands[3], operands[4],
9624                                                 operands[5]));
9625   DONE;
9626 })
9627
9628 ;;
9629 ;; [viwdupq_m_wb_u])
9630 ;;
9631 (define_expand "mve_viwdupq_m_wb_u<mode>"
9632  [(match_operand:SI 0 "s_register_operand")
9633   (match_operand:MVE_2 1 "s_register_operand")
9634   (match_operand:SI 2 "s_register_operand")
9635   (match_operand:DI 3 "s_register_operand")
9636   (match_operand:SI 4 "mve_imm_selective_upto_8")
9637   (match_operand:HI 5 "vpr_register_operand")]
9638  "TARGET_HAVE_MVE"
9639 {
9640   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9641   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9642                                                 operands[1], operands[2],
9643                                                 operands[3], operands[4],
9644                                                 operands[5]));
9645   DONE;
9646 })
9647
9648 ;;
9649 ;; [viwdupq_m_wb_u_insn])
9650 ;;
9651 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9652   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9653         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9654                        (match_operand:SI 3 "s_register_operand" "1")
9655                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9656                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9657                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9658          VIWDUPQ_M))
9659    (set (match_operand:SI 1 "s_register_operand" "=Te")
9660         (unspec:SI [(match_dup 2)
9661                     (match_dup 3)
9662                     (subreg:SI (match_dup 4) 4)
9663                     (match_dup 5)
9664                     (match_dup 6)]
9665          VIWDUPQ_M))
9666   ]
9667   "TARGET_HAVE_MVE"
9668   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9669   [(set_attr "type" "mve_move")
9670    (set_attr "length""8")])
9671
9672 ;;
9673 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9674 ;;
9675 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9676   [(set (mem:BLK (scratch))
9677         (unspec:BLK
9678                 [(match_operand:V4SI 1 "s_register_operand" "0")
9679                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9680                  (match_operand:V4SI 3 "s_register_operand" "w")]
9681          VSTRWSBWBQ))
9682    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9683         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9684          VSTRWSBWBQ))
9685   ]
9686   "TARGET_HAVE_MVE"
9687 {
9688    rtx ops[3];
9689    ops[0] = operands[1];
9690    ops[1] = operands[2];
9691    ops[2] = operands[3];
9692    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9693    return "";
9694 }
9695   [(set_attr "length" "4")])
9696
9697 ;;
9698 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9699 ;;
9700 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9701  [(set (mem:BLK (scratch))
9702        (unspec:BLK
9703                 [(match_operand:V4SI 1 "s_register_operand" "0")
9704                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9705                  (match_operand:V4SI 3 "s_register_operand" "w")
9706                  (match_operand:HI 4 "vpr_register_operand")]
9707         VSTRWSBWBQ))
9708    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9709         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9710          VSTRWSBWBQ))
9711   ]
9712   "TARGET_HAVE_MVE"
9713 {
9714    rtx ops[3];
9715    ops[0] = operands[1];
9716    ops[1] = operands[2];
9717    ops[2] = operands[3];
9718    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9719    return "";
9720 }
9721   [(set_attr "length" "8")])
9722
9723 ;;
9724 ;; [vstrwq_scatter_base_wb_f]
9725 ;;
9726 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9727  [(set (mem:BLK (scratch))
9728        (unspec:BLK
9729                 [(match_operand:V4SI 1 "s_register_operand" "0")
9730                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9731                  (match_operand:V4SF 3 "s_register_operand" "w")]
9732          VSTRWQSBWB_F))
9733    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9734         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9735          VSTRWQSBWB_F))
9736   ]
9737   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9738 {
9739    rtx ops[3];
9740    ops[0] = operands[1];
9741    ops[1] = operands[2];
9742    ops[2] = operands[3];
9743    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9744    return "";
9745 }
9746   [(set_attr "length" "4")])
9747
9748 ;;
9749 ;; [vstrwq_scatter_base_wb_p_f]
9750 ;;
9751 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9752  [(set (mem:BLK (scratch))
9753        (unspec:BLK
9754                 [(match_operand:V4SI 1 "s_register_operand" "0")
9755                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9756                  (match_operand:V4SF 3 "s_register_operand" "w")
9757                  (match_operand:HI 4 "vpr_register_operand")]
9758         VSTRWQSBWB_F))
9759    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9760         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9761          VSTRWQSBWB_F))
9762   ]
9763   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9764 {
9765    rtx ops[3];
9766    ops[0] = operands[1];
9767    ops[1] = operands[2];
9768    ops[2] = operands[3];
9769    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9770    return "";
9771 }
9772   [(set_attr "length" "8")])
9773
9774 ;;
9775 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9776 ;;
9777 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9778   [(set (mem:BLK (scratch))
9779         (unspec:BLK
9780                 [(match_operand:V2DI 1 "s_register_operand" "0")
9781                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9782                  (match_operand:V2DI 3 "s_register_operand" "w")]
9783          VSTRDSBWBQ))
9784    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9785         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9786          VSTRDSBWBQ))
9787   ]
9788   "TARGET_HAVE_MVE"
9789 {
9790    rtx ops[3];
9791    ops[0] = operands[1];
9792    ops[1] = operands[2];
9793    ops[2] = operands[3];
9794    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9795    return "";
9796 }
9797   [(set_attr "length" "4")])
9798
9799 ;;
9800 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9801 ;;
9802 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9803   [(set (mem:BLK (scratch))
9804         (unspec:BLK
9805                 [(match_operand:V2DI 1 "s_register_operand" "0")
9806                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9807                  (match_operand:V2DI 3 "s_register_operand" "w")
9808                  (match_operand:HI 4 "vpr_register_operand")]
9809          VSTRDSBWBQ))
9810    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9811         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9812          VSTRDSBWBQ))
9813   ]
9814   "TARGET_HAVE_MVE"
9815 {
9816    rtx ops[3];
9817    ops[0] = operands[1];
9818    ops[1] = operands[2];
9819    ops[2] = operands[3];
9820    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9821    return "";
9822 }
9823   [(set_attr "length" "8")])
9824
9825 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9826   [(match_operand:V4SI 0 "s_register_operand")
9827    (match_operand:V4SI 1 "s_register_operand")
9828    (match_operand:SI 2 "mve_vldrd_immediate")
9829    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9830   "TARGET_HAVE_MVE"
9831 {
9832   rtx ignore_result = gen_reg_rtx (V4SImode);
9833   emit_insn (
9834   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9835                                                  operands[1], operands[2]));
9836   DONE;
9837 })
9838
9839 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9840   [(match_operand:V4SI 0 "s_register_operand")
9841    (match_operand:V4SI 1 "s_register_operand")
9842    (match_operand:SI 2 "mve_vldrd_immediate")
9843    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9844   "TARGET_HAVE_MVE"
9845 {
9846   rtx ignore_wb = gen_reg_rtx (V4SImode);
9847   emit_insn (
9848   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9849                                                  operands[1], operands[2]));
9850   DONE;
9851 })
9852
9853 ;;
9854 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9855 ;;
9856 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9857   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9858         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9859                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9860                       (mem:BLK (scratch))]
9861          VLDRWGBWBQ))
9862    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9863         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9864          VLDRWGBWBQ))
9865   ]
9866   "TARGET_HAVE_MVE"
9867 {
9868    rtx ops[3];
9869    ops[0] = operands[0];
9870    ops[1] = operands[2];
9871    ops[2] = operands[3];
9872    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9873    return "";
9874 }
9875   [(set_attr "length" "4")])
9876
9877 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9878   [(match_operand:V4SI 0 "s_register_operand")
9879    (match_operand:V4SI 1 "s_register_operand")
9880    (match_operand:SI 2 "mve_vldrd_immediate")
9881    (match_operand:HI 3 "vpr_register_operand")
9882    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9883   "TARGET_HAVE_MVE"
9884 {
9885   rtx ignore_result = gen_reg_rtx (V4SImode);
9886   emit_insn (
9887   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9888                                                    operands[1], operands[2],
9889                                                    operands[3]));
9890   DONE;
9891 })
9892 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9893   [(match_operand:V4SI 0 "s_register_operand")
9894    (match_operand:V4SI 1 "s_register_operand")
9895    (match_operand:SI 2 "mve_vldrd_immediate")
9896    (match_operand:HI 3 "vpr_register_operand")
9897    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9898   "TARGET_HAVE_MVE"
9899 {
9900   rtx ignore_wb = gen_reg_rtx (V4SImode);
9901   emit_insn (
9902   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9903                                                    operands[1], operands[2],
9904                                                    operands[3]));
9905   DONE;
9906 })
9907
9908 ;;
9909 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9910 ;;
9911 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9912   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9913         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9914                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9915                       (match_operand:HI 4 "vpr_register_operand" "Up")
9916                       (mem:BLK (scratch))]
9917          VLDRWGBWBQ))
9918    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9919         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9920          VLDRWGBWBQ))
9921   ]
9922   "TARGET_HAVE_MVE"
9923 {
9924    rtx ops[3];
9925    ops[0] = operands[0];
9926    ops[1] = operands[2];
9927    ops[2] = operands[3];
9928    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9929    return "";
9930 }
9931   [(set_attr "length" "8")])
9932
9933 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9934   [(match_operand:V4SI 0 "s_register_operand")
9935    (match_operand:V4SI 1 "s_register_operand")
9936    (match_operand:SI 2 "mve_vldrd_immediate")
9937    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9938   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9939 {
9940   rtx ignore_result = gen_reg_rtx (V4SFmode);
9941   emit_insn (
9942   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9943                                             operands[1], operands[2]));
9944   DONE;
9945 })
9946
9947 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9948   [(match_operand:V4SF 0 "s_register_operand")
9949    (match_operand:V4SI 1 "s_register_operand")
9950    (match_operand:SI 2 "mve_vldrd_immediate")
9951    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9952   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9953 {
9954   rtx ignore_wb = gen_reg_rtx (V4SImode);
9955   emit_insn (
9956   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9957                                             operands[1], operands[2]));
9958   DONE;
9959 })
9960
9961 ;;
9962 ;; [vldrwq_gather_base_wb_f]
9963 ;;
9964 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9965   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9966         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9967                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9968                       (mem:BLK (scratch))]
9969          VLDRWQGBWB_F))
9970    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9971         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9972          VLDRWQGBWB_F))
9973   ]
9974   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9975 {
9976    rtx ops[3];
9977    ops[0] = operands[0];
9978    ops[1] = operands[2];
9979    ops[2] = operands[3];
9980    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9981    return "";
9982 }
9983   [(set_attr "length" "4")])
9984
9985 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
9986   [(match_operand:V4SI 0 "s_register_operand")
9987    (match_operand:V4SI 1 "s_register_operand")
9988    (match_operand:SI 2 "mve_vldrd_immediate")
9989    (match_operand:HI 3 "vpr_register_operand")
9990    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9991   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9992 {
9993   rtx ignore_result = gen_reg_rtx (V4SFmode);
9994   emit_insn (
9995   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
9996                                               operands[1], operands[2],
9997                                               operands[3]));
9998   DONE;
9999 })
10000
10001 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10002   [(match_operand:V4SF 0 "s_register_operand")
10003    (match_operand:V4SI 1 "s_register_operand")
10004    (match_operand:SI 2 "mve_vldrd_immediate")
10005    (match_operand:HI 3 "vpr_register_operand")
10006    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10008 {
10009   rtx ignore_wb = gen_reg_rtx (V4SImode);
10010   emit_insn (
10011   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10012                                               operands[1], operands[2],
10013                                               operands[3]));
10014   DONE;
10015 })
10016
10017 ;;
10018 ;; [vldrwq_gather_base_wb_z_f]
10019 ;;
10020 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10021   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10022         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10023                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10024                       (match_operand:HI 4 "vpr_register_operand" "Up")
10025                       (mem:BLK (scratch))]
10026          VLDRWQGBWB_F))
10027    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10028         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10029          VLDRWQGBWB_F))
10030   ]
10031   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10032 {
10033    rtx ops[3];
10034    ops[0] = operands[0];
10035    ops[1] = operands[2];
10036    ops[2] = operands[3];
10037    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10038    return "";
10039 }
10040   [(set_attr "length" "8")])
10041
10042 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10043   [(match_operand:V2DI 0 "s_register_operand")
10044    (match_operand:V2DI 1 "s_register_operand")
10045    (match_operand:SI 2 "mve_vldrd_immediate")
10046    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10047   "TARGET_HAVE_MVE"
10048 {
10049   rtx ignore_result = gen_reg_rtx (V2DImode);
10050   emit_insn (
10051   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10052                                                  operands[1], operands[2]));
10053   DONE;
10054 })
10055
10056 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10057   [(match_operand:V2DI 0 "s_register_operand")
10058    (match_operand:V2DI 1 "s_register_operand")
10059    (match_operand:SI 2 "mve_vldrd_immediate")
10060    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10061   "TARGET_HAVE_MVE"
10062 {
10063   rtx ignore_wb = gen_reg_rtx (V2DImode);
10064   emit_insn (
10065   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10066                                                  operands[1], operands[2]));
10067   DONE;
10068 })
10069
10070
10071 ;;
10072 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10073 ;;
10074 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10075   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10076         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10077                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10078                       (mem:BLK (scratch))]
10079          VLDRDGBWBQ))
10080    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10081         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10082          VLDRDGBWBQ))
10083   ]
10084   "TARGET_HAVE_MVE"
10085 {
10086    rtx ops[3];
10087    ops[0] = operands[0];
10088    ops[1] = operands[2];
10089    ops[2] = operands[3];
10090    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10091    return "";
10092 }
10093   [(set_attr "length" "4")])
10094
10095 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10096   [(match_operand:V2DI 0 "s_register_operand")
10097    (match_operand:V2DI 1 "s_register_operand")
10098    (match_operand:SI 2 "mve_vldrd_immediate")
10099    (match_operand:HI 3 "vpr_register_operand")
10100    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10101   "TARGET_HAVE_MVE"
10102 {
10103   rtx ignore_result = gen_reg_rtx (V2DImode);
10104   emit_insn (
10105   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10106                                                    operands[1], operands[2],
10107                                                    operands[3]));
10108   DONE;
10109 })
10110
10111 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10112   [(match_operand:V2DI 0 "s_register_operand")
10113    (match_operand:V2DI 1 "s_register_operand")
10114    (match_operand:SI 2 "mve_vldrd_immediate")
10115    (match_operand:HI 3 "vpr_register_operand")
10116    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10117   "TARGET_HAVE_MVE"
10118 {
10119   rtx ignore_wb = gen_reg_rtx (V2DImode);
10120   emit_insn (
10121   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10122                                                    operands[1], operands[2],
10123                                                    operands[3]));
10124   DONE;
10125 })
10126
10127 (define_insn "get_fpscr_nzcvqc"
10128  [(set (match_operand:SI 0 "register_operand" "=r")
10129    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10130  "TARGET_HAVE_MVE"
10131  "vmrs\\t%0, FPSCR_nzcvqc"
10132   [(set_attr "type" "mve_move")])
10133
10134 (define_insn "set_fpscr_nzcvqc"
10135  [(set (reg:SI VFPCC_REGNUM)
10136    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10137     VUNSPEC_SET_FPSCR_NZCVQC))]
10138  "TARGET_HAVE_MVE"
10139  "vmsr\\tFPSCR_nzcvqc, %0"
10140   [(set_attr "type" "mve_move")])
10141
10142 ;;
10143 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10144 ;;
10145 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10146   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10147         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10148                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10149                       (match_operand:HI 4 "vpr_register_operand" "Up")
10150                       (mem:BLK (scratch))]
10151          VLDRDGBWBQ))
10152    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10153         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10154          VLDRDGBWBQ))
10155   ]
10156   "TARGET_HAVE_MVE"
10157 {
10158    rtx ops[3];
10159    ops[0] = operands[0];
10160    ops[1] = operands[2];
10161    ops[2] = operands[3];
10162    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10163    return "";
10164 }
10165   [(set_attr "length" "8")])
10166 ;;
10167 ;; [vadciq_m_s, vadciq_m_u])
10168 ;;
10169 (define_insn "mve_vadciq_m_<supf>v4si"
10170   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10171         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10172                       (match_operand:V4SI 2 "s_register_operand" "w")
10173                       (match_operand:V4SI 3 "s_register_operand" "w")
10174                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10175          VADCIQ_M))
10176    (set (reg:SI VFPCC_REGNUM)
10177         (unspec:SI [(const_int 0)]
10178          VADCIQ_M))
10179   ]
10180   "TARGET_HAVE_MVE"
10181   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10182   [(set_attr "type" "mve_move")
10183    (set_attr "length" "8")])
10184
10185 ;;
10186 ;; [vadciq_u, vadciq_s])
10187 ;;
10188 (define_insn "mve_vadciq_<supf>v4si"
10189   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10190         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10191                       (match_operand:V4SI 2 "s_register_operand" "w")]
10192          VADCIQ))
10193    (set (reg:SI VFPCC_REGNUM)
10194         (unspec:SI [(const_int 0)]
10195          VADCIQ))
10196   ]
10197   "TARGET_HAVE_MVE"
10198   "vadci.i32\t%q0, %q1, %q2"
10199   [(set_attr "type" "mve_move")
10200    (set_attr "length" "4")])
10201
10202 ;;
10203 ;; [vadcq_m_s, vadcq_m_u])
10204 ;;
10205 (define_insn "mve_vadcq_m_<supf>v4si"
10206   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10207         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10208                       (match_operand:V4SI 2 "s_register_operand" "w")
10209                       (match_operand:V4SI 3 "s_register_operand" "w")
10210                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10211          VADCQ_M))
10212    (set (reg:SI VFPCC_REGNUM)
10213         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10214          VADCQ_M))
10215   ]
10216   "TARGET_HAVE_MVE"
10217   "vpst\;vadct.i32\t%q0, %q2, %q3"
10218   [(set_attr "type" "mve_move")
10219    (set_attr "length" "8")])
10220
10221 ;;
10222 ;; [vadcq_u, vadcq_s])
10223 ;;
10224 (define_insn "mve_vadcq_<supf>v4si"
10225   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10226         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10227                        (match_operand:V4SI 2 "s_register_operand" "w")]
10228          VADCQ))
10229    (set (reg:SI VFPCC_REGNUM)
10230         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10231          VADCQ))
10232   ]
10233   "TARGET_HAVE_MVE"
10234   "vadc.i32\t%q0, %q1, %q2"
10235   [(set_attr "type" "mve_move")
10236    (set_attr "length" "4")
10237    (set_attr "conds" "set")])
10238
10239 ;;
10240 ;; [vsbciq_m_u, vsbciq_m_s])
10241 ;;
10242 (define_insn "mve_vsbciq_m_<supf>v4si"
10243   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10244         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10245                       (match_operand:V4SI 2 "s_register_operand" "w")
10246                       (match_operand:V4SI 3 "s_register_operand" "w")
10247                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10248          VSBCIQ_M))
10249    (set (reg:SI VFPCC_REGNUM)
10250         (unspec:SI [(const_int 0)]
10251          VSBCIQ_M))
10252   ]
10253   "TARGET_HAVE_MVE"
10254   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10255   [(set_attr "type" "mve_move")
10256    (set_attr "length" "8")])
10257
10258 ;;
10259 ;; [vsbciq_s, vsbciq_u])
10260 ;;
10261 (define_insn "mve_vsbciq_<supf>v4si"
10262   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10263         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10264                       (match_operand:V4SI 2 "s_register_operand" "w")]
10265          VSBCIQ))
10266    (set (reg:SI VFPCC_REGNUM)
10267         (unspec:SI [(const_int 0)]
10268          VSBCIQ))
10269   ]
10270   "TARGET_HAVE_MVE"
10271   "vsbci.i32\t%q0, %q1, %q2"
10272   [(set_attr "type" "mve_move")
10273    (set_attr "length" "4")])
10274
10275 ;;
10276 ;; [vsbcq_m_u, vsbcq_m_s])
10277 ;;
10278 (define_insn "mve_vsbcq_m_<supf>v4si"
10279   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10280         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10281                       (match_operand:V4SI 2 "s_register_operand" "w")
10282                       (match_operand:V4SI 3 "s_register_operand" "w")
10283                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10284          VSBCQ_M))
10285    (set (reg:SI VFPCC_REGNUM)
10286         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10287          VSBCQ_M))
10288   ]
10289   "TARGET_HAVE_MVE"
10290   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10291   [(set_attr "type" "mve_move")
10292    (set_attr "length" "8")])
10293
10294 ;;
10295 ;; [vsbcq_s, vsbcq_u])
10296 ;;
10297 (define_insn "mve_vsbcq_<supf>v4si"
10298   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10299         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10300                       (match_operand:V4SI 2 "s_register_operand" "w")]
10301          VSBCQ))
10302    (set (reg:SI VFPCC_REGNUM)
10303         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10304          VSBCQ))
10305   ]
10306   "TARGET_HAVE_MVE"
10307   "vsbc.i32\t%q0, %q1, %q2"
10308   [(set_attr "type" "mve_move")
10309    (set_attr "length" "4")])
10310
10311 ;;
10312 ;; [vst2q])
10313 ;;
10314 (define_insn "mve_vst2q<mode>"
10315   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10316         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10317                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10318          VST2Q))
10319   ]
10320   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10321    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10322 {
10323    rtx ops[4];
10324    int regno = REGNO (operands[1]);
10325    ops[0] = gen_rtx_REG (TImode, regno);
10326    ops[1] = gen_rtx_REG (TImode, regno + 4);
10327    rtx reg  = operands[0];
10328    while (reg && !REG_P (reg))
10329     reg = XEXP (reg, 0);
10330    gcc_assert (REG_P (reg));
10331    ops[2] = reg;
10332    ops[3] = operands[0];
10333    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10334                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10335    return "";
10336 }
10337   [(set_attr "length" "8")])
10338
10339 ;;
10340 ;; [vld2q])
10341 ;;
10342 (define_insn "mve_vld2q<mode>"
10343   [(set (match_operand:OI 0 "s_register_operand" "=w")
10344         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10345                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10346          VLD2Q))
10347   ]
10348   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10349    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10350 {
10351    rtx ops[4];
10352    int regno = REGNO (operands[0]);
10353    ops[0] = gen_rtx_REG (TImode, regno);
10354    ops[1] = gen_rtx_REG (TImode, regno + 4);
10355    rtx reg  = operands[1];
10356    while (reg && !REG_P (reg))
10357     reg = XEXP (reg, 0);
10358    gcc_assert (REG_P (reg));
10359    ops[2] = reg;
10360    ops[3] = operands[1];
10361    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10362                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10363    return "";
10364 }
10365   [(set_attr "length" "8")])
10366
10367 ;;
10368 ;; [vld4q])
10369 ;;
10370 (define_insn "mve_vld4q<mode>"
10371   [(set (match_operand:XI 0 "s_register_operand" "=w")
10372         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10373                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10374          VLD4Q))
10375   ]
10376   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10377    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10378 {
10379    rtx ops[6];
10380    int regno = REGNO (operands[0]);
10381    ops[0] = gen_rtx_REG (TImode, regno);
10382    ops[1] = gen_rtx_REG (TImode, regno+4);
10383    ops[2] = gen_rtx_REG (TImode, regno+8);
10384    ops[3] = gen_rtx_REG (TImode, regno + 12);
10385    rtx reg  = operands[1];
10386    while (reg && !REG_P (reg))
10387     reg = XEXP (reg, 0);
10388    gcc_assert (REG_P (reg));
10389    ops[4] = reg;
10390    ops[5] = operands[1];
10391    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10392                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10393                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10394                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10395    return "";
10396 }
10397   [(set_attr "length" "16")])
10398 ;;
10399 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10400 ;;
10401 (define_insn "mve_vec_extract<mode><V_elem_l>"
10402  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10403    (vec_select:<V_elem>
10404     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10405     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10406   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10407    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10408 {
10409   if (BYTES_BIG_ENDIAN)
10410     {
10411       int elt = INTVAL (operands[2]);
10412       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10413       operands[2] = GEN_INT (elt);
10414     }
10415   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10416 }
10417  [(set_attr "type" "mve_move")])
10418
10419 (define_insn "mve_vec_extractv2didi"
10420  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10421    (vec_select:DI
10422     (match_operand:V2DI 1 "s_register_operand" "w")
10423     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10424   "TARGET_HAVE_MVE"
10425 {
10426   int elt = INTVAL (operands[2]);
10427   if (BYTES_BIG_ENDIAN)
10428     elt = 1 - elt;
10429
10430   if (elt == 0)
10431    return "vmov\t%Q0, %R0, %e1";
10432   else
10433    return "vmov\t%Q0, %R0, %f1";
10434 }
10435  [(set_attr "type" "mve_move")])
10436
10437 (define_insn "*mve_vec_extract_sext_internal<mode>"
10438  [(set (match_operand:SI 0 "s_register_operand" "=r")
10439    (sign_extend:SI
10440     (vec_select:<V_elem>
10441      (match_operand:MVE_2 1 "s_register_operand" "w")
10442      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10443   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10444    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10445 {
10446   if (BYTES_BIG_ENDIAN)
10447     {
10448       int elt = INTVAL (operands[2]);
10449       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10450       operands[2] = GEN_INT (elt);
10451     }
10452   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10453 }
10454  [(set_attr "type" "mve_move")])
10455
10456 (define_insn "*mve_vec_extract_zext_internal<mode>"
10457  [(set (match_operand:SI 0 "s_register_operand" "=r")
10458    (zero_extend:SI
10459     (vec_select:<V_elem>
10460      (match_operand:MVE_2 1 "s_register_operand" "w")
10461      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10462   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10463    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10464 {
10465   if (BYTES_BIG_ENDIAN)
10466     {
10467       int elt = INTVAL (operands[2]);
10468       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10469       operands[2] = GEN_INT (elt);
10470     }
10471   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10472 }
10473  [(set_attr "type" "mve_move")])
10474
10475 ;;
10476 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10477 ;;
10478 (define_insn "mve_vec_set<mode>_internal"
10479  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10480        (vec_merge:VQ2
10481         (vec_duplicate:VQ2
10482           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10483         (match_operand:VQ2 3 "s_register_operand" "0")
10484         (match_operand:SI 2 "immediate_operand" "i")))]
10485   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10486    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10487 {
10488   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10489   if (BYTES_BIG_ENDIAN)
10490     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10491   operands[2] = GEN_INT (elt);
10492
10493   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10494 }
10495  [(set_attr "type" "mve_move")])
10496
10497 (define_insn "mve_vec_setv2di_internal"
10498  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10499        (vec_merge:V2DI
10500         (vec_duplicate:V2DI
10501           (match_operand:DI 1 "nonimmediate_operand" "r"))
10502         (match_operand:V2DI 3 "s_register_operand" "0")
10503         (match_operand:SI 2 "immediate_operand" "i")))]
10504  "TARGET_HAVE_MVE"
10505 {
10506   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10507   if (BYTES_BIG_ENDIAN)
10508     elt = 1 - elt;
10509
10510   if (elt == 0)
10511    return "vmov\t%e0, %Q1, %R1";
10512   else
10513    return "vmov\t%f0, %J1, %K1";
10514 }
10515  [(set_attr "type" "mve_move")])
10516
10517 ;;
10518 ;; [uqrshll_di]
10519 ;;
10520 (define_insn "mve_uqrshll_sat<supf>_di"
10521   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10522         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10523                     (match_operand:SI 2 "register_operand" "r")]
10524          UQRSHLLQ))]
10525   "TARGET_HAVE_MVE"
10526   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10527   [(set_attr "predicable" "yes")])
10528
10529 ;;
10530 ;; [sqrshrl_di]
10531 ;;
10532 (define_insn "mve_sqrshrl_sat<supf>_di"
10533   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10534         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10535                     (match_operand:SI 2 "register_operand" "r")]
10536          SQRSHRLQ))]
10537   "TARGET_HAVE_MVE"
10538   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10539   [(set_attr "predicable" "yes")])
10540
10541 ;;
10542 ;; [uqrshl_si]
10543 ;;
10544 (define_insn "mve_uqrshl_si"
10545   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10546         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10547                     (match_operand:SI 2 "register_operand" "r")]
10548          UQRSHL))]
10549   "TARGET_HAVE_MVE"
10550   "uqrshl%?\\t%1, %2"
10551   [(set_attr "predicable" "yes")])
10552
10553 ;;
10554 ;; [sqrshr_si]
10555 ;;
10556 (define_insn "mve_sqrshr_si"
10557   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10558         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10559                     (match_operand:SI 2 "register_operand" "r")]
10560          SQRSHR))]
10561   "TARGET_HAVE_MVE"
10562   "sqrshr%?\\t%1, %2"
10563   [(set_attr "predicable" "yes")])
10564
10565 ;;
10566 ;; [uqshll_di]
10567 ;;
10568 (define_insn "mve_uqshll_di"
10569   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10570         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10571                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10572   "TARGET_HAVE_MVE"
10573   "uqshll%?\\t%Q1, %R1, %2"
10574   [(set_attr "predicable" "yes")])
10575
10576 ;;
10577 ;; [urshrl_di]
10578 ;;
10579 (define_insn "mve_urshrl_di"
10580   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10581         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10582                     (match_operand:SI 2 "immediate_operand" "Pg")]
10583          URSHRL))]
10584   "TARGET_HAVE_MVE"
10585   "urshrl%?\\t%Q1, %R1, %2"
10586   [(set_attr "predicable" "yes")])
10587
10588 ;;
10589 ;; [uqshl_si]
10590 ;;
10591 (define_insn "mve_uqshl_si"
10592   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10593         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10594                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10595   "TARGET_HAVE_MVE"
10596   "uqshl%?\\t%1, %2"
10597   [(set_attr "predicable" "yes")])
10598
10599 ;;
10600 ;; [urshr_si]
10601 ;;
10602 (define_insn "mve_urshr_si"
10603   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10604         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10605                     (match_operand:SI 2 "immediate_operand" "Pg")]
10606          URSHR))]
10607   "TARGET_HAVE_MVE"
10608   "urshr%?\\t%1, %2"
10609   [(set_attr "predicable" "yes")])
10610
10611 ;;
10612 ;; [sqshl_si]
10613 ;;
10614 (define_insn "mve_sqshl_si"
10615   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10616         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10617                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10618   "TARGET_HAVE_MVE"
10619   "sqshl%?\\t%1, %2"
10620   [(set_attr "predicable" "yes")])
10621
10622 ;;
10623 ;; [srshr_si]
10624 ;;
10625 (define_insn "mve_srshr_si"
10626   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10627         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10628                     (match_operand:SI 2 "immediate_operand" "Pg")]
10629          SRSHR))]
10630   "TARGET_HAVE_MVE"
10631   "srshr%?\\t%1, %2"
10632   [(set_attr "predicable" "yes")])
10633
10634 ;;
10635 ;; [srshrl_di]
10636 ;;
10637 (define_insn "mve_srshrl_di"
10638   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10639         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10640                     (match_operand:SI 2 "immediate_operand" "Pg")]
10641          SRSHRL))]
10642   "TARGET_HAVE_MVE"
10643   "srshrl%?\\t%Q1, %R1, %2"
10644   [(set_attr "predicable" "yes")])
10645
10646 ;;
10647 ;; [sqshll_di]
10648 ;;
10649 (define_insn "mve_sqshll_di"
10650   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10651         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10652                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10653   "TARGET_HAVE_MVE"
10654   "sqshll%?\\t%Q1, %R1, %2"
10655   [(set_attr "predicable" "yes")])
10656
10657 ;;
10658 ;; [vshlcq_m_u vshlcq_m_s]
10659 ;;
10660 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10661  [(match_operand:MVE_2 0 "s_register_operand")
10662   (match_operand:MVE_2 1 "s_register_operand")
10663   (match_operand:SI 2 "s_register_operand")
10664   (match_operand:SI 3 "mve_imm_32")
10665   (match_operand:HI 4 "vpr_register_operand")
10666   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10667  "TARGET_HAVE_MVE"
10668 {
10669   rtx ignore_wb = gen_reg_rtx (SImode);
10670   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10671                                             operands[2], operands[3],
10672                                             operands[4]));
10673   DONE;
10674 })
10675
10676 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10677  [(match_operand:SI 0 "s_register_operand")
10678   (match_operand:MVE_2 1 "s_register_operand")
10679   (match_operand:SI 2 "s_register_operand")
10680   (match_operand:SI 3 "mve_imm_32")
10681   (match_operand:HI 4 "vpr_register_operand")
10682   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10683  "TARGET_HAVE_MVE"
10684 {
10685   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10686   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10687                                             operands[1], operands[2],
10688                                             operands[3], operands[4]));
10689   DONE;
10690 })
10691
10692 (define_insn "mve_vshlcq_m_<supf><mode>"
10693  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10694        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10695                       (match_operand:SI 3 "s_register_operand" "1")
10696                       (match_operand:SI 4 "mve_imm_32" "Rf")
10697                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10698         VSHLCQ_M))
10699   (set (match_operand:SI  1 "s_register_operand" "=r")
10700        (unspec:SI [(match_dup 2)
10701                    (match_dup 3)
10702                    (match_dup 4)
10703                    (match_dup 5)]
10704         VSHLCQ_M))
10705  ]
10706  "TARGET_HAVE_MVE"
10707  "vpst\;vshlct\t%q0, %1, %4"
10708  [(set_attr "type" "mve_move")
10709   (set_attr "length" "8")])
10710
10711 ;; CDE instructions on MVE registers.
10712
10713 (define_insn "arm_vcx1qv16qi"
10714   [(set (match_operand:V16QI 0 "register_operand" "=t")
10715         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10716                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10717          UNSPEC_VCDE))]
10718   "TARGET_CDE && TARGET_HAVE_MVE"
10719   "vcx1\\tp%c1, %q0, #%c2"
10720   [(set_attr "type" "coproc")]
10721 )
10722
10723 (define_insn "arm_vcx1qav16qi"
10724   [(set (match_operand:V16QI 0 "register_operand" "=t")
10725         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10726                             (match_operand:V16QI 2 "register_operand" "0")
10727                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10728          UNSPEC_VCDEA))]
10729   "TARGET_CDE && TARGET_HAVE_MVE"
10730   "vcx1a\\tp%c1, %q0, #%c3"
10731   [(set_attr "type" "coproc")]
10732 )
10733
10734 (define_insn "arm_vcx2qv16qi"
10735   [(set (match_operand:V16QI 0 "register_operand" "=t")
10736         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10737                           (match_operand:V16QI 2 "register_operand" "t")
10738                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10739          UNSPEC_VCDE))]
10740   "TARGET_CDE && TARGET_HAVE_MVE"
10741   "vcx2\\tp%c1, %q0, %q2, #%c3"
10742   [(set_attr "type" "coproc")]
10743 )
10744
10745 (define_insn "arm_vcx2qav16qi"
10746   [(set (match_operand:V16QI 0 "register_operand" "=t")
10747         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10748                           (match_operand:V16QI 2 "register_operand" "0")
10749                           (match_operand:V16QI 3 "register_operand" "t")
10750                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10751          UNSPEC_VCDEA))]
10752   "TARGET_CDE && TARGET_HAVE_MVE"
10753   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10754   [(set_attr "type" "coproc")]
10755 )
10756
10757 (define_insn "arm_vcx3qv16qi"
10758   [(set (match_operand:V16QI 0 "register_operand" "=t")
10759         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10760                           (match_operand:V16QI 2 "register_operand" "t")
10761                           (match_operand:V16QI 3 "register_operand" "t")
10762                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10763          UNSPEC_VCDE))]
10764   "TARGET_CDE && TARGET_HAVE_MVE"
10765   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10766   [(set_attr "type" "coproc")]
10767 )
10768
10769 (define_insn "arm_vcx3qav16qi"
10770   [(set (match_operand:V16QI 0 "register_operand" "=t")
10771         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10772                           (match_operand:V16QI 2 "register_operand" "0")
10773                           (match_operand:V16QI 3 "register_operand" "t")
10774                           (match_operand:V16QI 4 "register_operand" "t")
10775                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10776          UNSPEC_VCDEA))]
10777   "TARGET_CDE && TARGET_HAVE_MVE"
10778   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10779   [(set_attr "type" "coproc")]
10780 )
10781
10782 (define_insn "arm_vcx1q<a>_p_v16qi"
10783   [(set (match_operand:V16QI 0 "register_operand" "=t")
10784         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10785                            (match_operand:V16QI 2 "register_operand" "0")
10786                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10787                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10788          CDE_VCX))]
10789   "TARGET_CDE && TARGET_HAVE_MVE"
10790   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10791   [(set_attr "type" "coproc")
10792    (set_attr "length" "8")]
10793 )
10794
10795 (define_insn "arm_vcx2q<a>_p_v16qi"
10796   [(set (match_operand:V16QI 0 "register_operand" "=t")
10797         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10798                           (match_operand:V16QI 2 "register_operand" "0")
10799                           (match_operand:V16QI 3 "register_operand" "t")
10800                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10801                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10802          CDE_VCX))]
10803   "TARGET_CDE && TARGET_HAVE_MVE"
10804   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10805   [(set_attr "type" "coproc")
10806    (set_attr "length" "8")]
10807 )
10808
10809 (define_insn "arm_vcx3q<a>_p_v16qi"
10810   [(set (match_operand:V16QI 0 "register_operand" "=t")
10811         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10812                           (match_operand:V16QI 2 "register_operand" "0")
10813                           (match_operand:V16QI 3 "register_operand" "t")
10814                           (match_operand:V16QI 4 "register_operand" "t")
10815                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10816                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10817          CDE_VCX))]
10818   "TARGET_CDE && TARGET_HAVE_MVE"
10819   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10820   [(set_attr "type" "coproc")
10821    (set_attr "length" "8")]
10822 )
10823
10824 (define_insn "*movmisalign<mode>_mve_store"
10825   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Ux")
10826         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10827          UNSPEC_MISALIGNED_ACCESS))]
10828   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10829     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10830    && !BYTES_BIG_ENDIAN && unaligned_access"
10831   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10832   [(set_attr "type" "mve_store")]
10833 )
10834
10835
10836 (define_insn "*movmisalign<mode>_mve_load"
10837   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10838         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Ux")]
10839          UNSPEC_MISALIGNED_ACCESS))]
10840   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10841     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10842    && !BYTES_BIG_ENDIAN && unaligned_access"
10843   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10844   [(set_attr "type" "mve_load")]
10845 )