gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_mode_attr V_sz_elem2 [(V16QI "s8") (V8HI "u16") (V4SI "u32")
  21                               (V2DI "u64")])
  22 (define_mode_iterator MVE_types [V16QI V8HI V4SI V2DI TI V8HF V4SF V2DF])
  23 (define_mode_iterator MVE_VLD_ST [V16QI V8HI V4SI V8HF V4SF])
  24 (define_mode_iterator MVE_0 [V8HF V4SF])
  25 (define_mode_iterator MVE_1 [V16QI V8HI V4SI V2DI])
  26 (define_mode_iterator MVE_3 [V16QI V8HI])
  27 (define_mode_iterator MVE_2 [V16QI V8HI V4SI])
  28 (define_mode_iterator MVE_5 [V8HI V4SI])
  29 (define_mode_iterator MVE_6 [V8HI V4SI])
  30
  31 (define_c_enum "unspec" [VST4Q VRNDXQ_F VRNDQ_F VRNDPQ_F VRNDNQ_F VRNDMQ_F
  32                          VRNDAQ_F VREV64Q_F VNEGQ_F VDUPQ_N_F VABSQ_F VREV32Q_F
  33                          VCVTTQ_F32_F16 VCVTBQ_F32_F16 VCVTQ_TO_F_S VQNEGQ_S
  34                          VCVTQ_TO_F_U VREV16Q_S VREV16Q_U VADDLVQ_S VMVNQ_N_S
  35                          VMVNQ_N_U VCVTAQ_S VCVTAQ_U VREV64Q_S VREV64Q_U
  36                          VQABSQ_S VNEGQ_S VMVNQ_S VMVNQ_U VDUPQ_N_U VDUPQ_N_S
  37                          VCLZQ_U VCLZQ_S VCLSQ_S VADDVQ_S VADDVQ_U VABSQ_S
  38                          VREV32Q_U VREV32Q_S VMOVLTQ_U VMOVLTQ_S VMOVLBQ_S
  39                          VMOVLBQ_U VCVTQ_FROM_F_S VCVTQ_FROM_F_U VCVTPQ_S
  40                          VCVTPQ_U VCVTNQ_S VCVTNQ_U VCVTMQ_S VCVTMQ_U
  41                          VADDLVQ_U VCTP8Q VCTP16Q VCTP32Q VCTP64Q VPNOT
  42                          VCREATEQ_F VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U VBRSRQ_N_F
  43                          VSUBQ_N_F VCREATEQ_U VCREATEQ_S VSHRQ_N_S VSHRQ_N_U
  44                          VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U VADDLVQ_P_S
  45                          VADDLVQ_P_U VCMPNEQ_U VCMPNEQ_S VSHLQ_S VSHLQ_U VABDQ_S
  46                          VADDQ_N_S VADDVAQ_S VADDVQ_P_S VANDQ_S VBICQ_S
  47                          VBRSRQ_N_S VCADDQ_ROT270_S VCADDQ_ROT90_S VCMPEQQ_S
  48                          VCMPEQQ_N_S VCMPNEQ_N_S VEORQ_S VHADDQ_S VHADDQ_N_S
  49                          VHSUBQ_S VHSUBQ_N_S VMAXQ_S VMAXVQ_S VMINQ_S VMINVQ_S
  50                          VMLADAVQ_S VMULHQ_S VMULLBQ_INT_S VMULLTQ_INT_S VMULQ_S
  51                          VMULQ_N_S VORNQ_S VORRQ_S VQADDQ_S VQADDQ_N_S VQRSHLQ_S
  52                          VQRSHLQ_N_S VQSHLQ_S VQSHLQ_N_S VQSHLQ_R_S VQSUBQ_S
  53                          VQSUBQ_N_S VRHADDQ_S VRMULHQ_S VRSHLQ_S VRSHLQ_N_S
  54                          VRSHRQ_N_S VSHLQ_N_S VSHLQ_R_S VSUBQ_S VSUBQ_N_S
  55                          VABDQ_U VADDQ_N_U VADDVAQ_U VADDVQ_P_U VANDQ_U VBICQ_U
  56                          VBRSRQ_N_U VCADDQ_ROT270_U VCADDQ_ROT90_U VCMPEQQ_U
  57                          VCMPEQQ_N_U VCMPNEQ_N_U VEORQ_U VHADDQ_U VHADDQ_N_U
  58                          VHSUBQ_U VHSUBQ_N_U VMAXQ_U VMAXVQ_U VMINQ_U VMINVQ_U
  59                          VMLADAVQ_U VMULHQ_U VMULLBQ_INT_U VMULLTQ_INT_U VMULQ_U
  60                          VMULQ_N_U VORNQ_U VORRQ_U VQADDQ_U VQADDQ_N_U VQRSHLQ_U
  61                          VQRSHLQ_N_U VQSHLQ_U VQSHLQ_N_U VQSHLQ_R_U VQSUBQ_U
  62                          VQSUBQ_N_U VRHADDQ_U VRMULHQ_U VRSHLQ_U VRSHLQ_N_U
  63                          VRSHRQ_N_U VSHLQ_N_U VSHLQ_R_U VSUBQ_U VSUBQ_N_U
  64                          VCMPGEQ_N_S VCMPGEQ_S VCMPGTQ_N_S VCMPGTQ_S VCMPLEQ_N_S
  65                          VCMPLEQ_S VCMPLTQ_N_S VCMPLTQ_S VHCADDQ_ROT270_S
  66                          VHCADDQ_ROT90_S VMAXAQ_S VMAXAVQ_S VMINAQ_S VMINAVQ_S
  67                          VMLADAVXQ_S VMLSDAVQ_S VMLSDAVXQ_S VQDMULHQ_N_S
  68                          VQDMULHQ_S VQRDMULHQ_N_S VQRDMULHQ_S VQSHLUQ_N_S
  69                          VCMPCSQ_N_U VCMPCSQ_U VCMPHIQ_N_U VCMPHIQ_U VABDQ_M_S
  70                          VABDQ_M_U VABDQ_F VADDQ_N_F VANDQ_F VBICQ_F
  71                          VCADDQ_ROT270_F VCADDQ_ROT90_F VCMPEQQ_F VCMPEQQ_N_F
  72                          VCMPGEQ_F VCMPGEQ_N_F VCMPGTQ_F VCMPGTQ_N_F VCMPLEQ_F
  73                          VCMPLEQ_N_F VCMPLTQ_F VCMPLTQ_N_F VCMPNEQ_F VCMPNEQ_N_F
  74                          VCMULQ_F VCMULQ_ROT180_F VCMULQ_ROT270_F VCMULQ_ROT90_F
  75                          VEORQ_F VMAXNMAQ_F VMAXNMAVQ_F VMAXNMQ_F VMAXNMVQ_F
  76                          VMINNMAQ_F VMINNMAVQ_F VMINNMQ_F VMINNMVQ_F VMULQ_F
  77                          VMULQ_N_F VORNQ_F VORRQ_F VSUBQ_F VADDLVAQ_U
  78                          VADDLVAQ_S VBICQ_N_U VBICQ_N_S VCTP8Q_M VCTP16Q_M
  79                          VCTP32Q_M VCTP64Q_M VCVTBQ_F16_F32 VCVTTQ_F16_F32
  80                          VMLALDAVQ_U VMLALDAVXQ_U VMLALDAVXQ_S VMLALDAVQ_S
  81                          VMLSLDAVQ_S VMLSLDAVXQ_S VMOVNBQ_U VMOVNBQ_S
  82                          VMOVNTQ_U VMOVNTQ_S VORRQ_N_S VORRQ_N_U VQDMULLBQ_N_S
  83                          VQDMULLBQ_S VQDMULLTQ_N_S VQDMULLTQ_S VQMOVNBQ_U
  84                          VQMOVNBQ_S VQMOVUNBQ_S VQMOVUNTQ_S VRMLALDAVHXQ_S
  85                          VRMLSLDAVHQ_S VRMLSLDAVHXQ_S VSHLLBQ_S
  86                          VSHLLBQ_U VSHLLTQ_U VSHLLTQ_S VQMOVNTQ_U VQMOVNTQ_S
  87                          VSHLLBQ_N_S VSHLLBQ_N_U VSHLLTQ_N_U VSHLLTQ_N_S
  88                          VRMLALDAVHQ_U VRMLALDAVHQ_S VMULLTQ_POLY_P
  89                          VMULLBQ_POLY_P VBICQ_M_N_S VBICQ_M_N_U VCMPEQQ_M_F
  90                          VCVTAQ_M_S VCVTAQ_M_U VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U
  91                          VQRSHRNBQ_N_U VQRSHRNBQ_N_S VQRSHRUNBQ_N_S
  92                          VRMLALDAVHAQ_S VABAVQ_S VABAVQ_U VSHLCQ_S VSHLCQ_U
  93                          VRMLALDAVHAQ_U VABSQ_M_S VADDVAQ_P_S VADDVAQ_P_U
  94                          VCLSQ_M_S VCLZQ_M_S VCLZQ_M_U VCMPCSQ_M_N_U
  95                          VCMPCSQ_M_U VCMPEQQ_M_N_S VCMPEQQ_M_N_U VCMPEQQ_M_S
  96                          VCMPEQQ_M_U VCMPGEQ_M_N_S VCMPGEQ_M_S VCMPGTQ_M_N_S
  97                          VCMPGTQ_M_S VCMPHIQ_M_N_U VCMPHIQ_M_U VCMPLEQ_M_N_S
  98                          VCMPLEQ_M_S VCMPLTQ_M_N_S VCMPLTQ_M_S VCMPNEQ_M_N_S
  99                          VCMPNEQ_M_N_U VCMPNEQ_M_S VCMPNEQ_M_U VDUPQ_M_N_S
 100                          VDUPQ_M_N_U VDWDUPQ_N_U VDWDUPQ_WB_U VIWDUPQ_N_U
 101                          VIWDUPQ_WB_U VMAXAQ_M_S VMAXAVQ_P_S VMAXVQ_P_S
 102                          VMAXVQ_P_U VMINAQ_M_S VMINAVQ_P_S VMINVQ_P_S VMINVQ_P_U
 103                          VMLADAVAQ_S VMLADAVAQ_U VMLADAVQ_P_S VMLADAVQ_P_U
 104                          VMLADAVXQ_P_S VMLAQ_N_S VMLAQ_N_U VMLASQ_N_S VMLASQ_N_U
 105                          VMLSDAVQ_P_S VMLSDAVXQ_P_S VMVNQ_M_S VMVNQ_M_U
 106                          VNEGQ_M_S VPSELQ_S VPSELQ_U VQABSQ_M_S VQDMLAHQ_N_S
 107                          VQDMLAHQ_N_U VQNEGQ_M_S VQRDMLADHQ_S VQRDMLADHXQ_S
 108                          VQRDMLAHQ_N_S VQRDMLAHQ_N_U VQRDMLASHQ_N_S
 109                          VQRDMLASHQ_N_U VQRDMLSDHQ_S VQRDMLSDHXQ_S VQRSHLQ_M_N_S
 110                          VQRSHLQ_M_N_U VQSHLQ_M_R_S VQSHLQ_M_R_U VREV64Q_M_S
 111                          VREV64Q_M_U VRSHLQ_M_N_S VRSHLQ_M_N_U VSHLQ_M_R_S
 112                          VSHLQ_M_R_U VSLIQ_N_S VSLIQ_N_U VSRIQ_N_S VSRIQ_N_U
 113                          VQDMLSDHXQ_S VQDMLSDHQ_S VQDMLADHXQ_S VQDMLADHQ_S
 114                          VMLSDAVAXQ_S VMLSDAVAQ_S VMLADAVAXQ_S
 115                          VCMPGEQ_M_F VCMPGTQ_M_N_F VMLSLDAVQ_P_S VRMLALDAVHAXQ_S
 116                          VMLSLDAVXQ_P_S VFMAQ_F VMLSLDAVAQ_S VQSHRUNBQ_N_S
 117                          VQRSHRUNTQ_N_S VCMLAQ_F VMINNMAQ_M_F VFMASQ_N_F
 118                          VDUPQ_M_N_F VCMPGTQ_M_F VCMPLTQ_M_F VRMLSLDAVHQ_P_S
 119                          VQSHRUNTQ_N_S VABSQ_M_F VMAXNMAVQ_P_F VFMAQ_N_F
 120                          VRMLSLDAVHXQ_P_S VREV32Q_M_F VRMLSLDAVHAQ_S
 121                          VRMLSLDAVHAXQ_S VCMPLTQ_M_N_F VCMPNEQ_M_F VRNDAQ_M_F
 122                          VRNDPQ_M_F VADDLVAQ_P_S VQMOVUNBQ_M_S VCMPLEQ_M_F
 123                          VCMLAQ_ROT180_F VMLSLDAVAXQ_S VRNDXQ_M_F VFMSQ_F
 124                          VMINNMVQ_P_F VMAXNMVQ_P_F VPSELQ_F VCMLAQ_ROT90_F
 125                          VQMOVUNTQ_M_S VREV64Q_M_F VNEGQ_M_F VRNDMQ_M_F
 126                          VCMPLEQ_M_N_F VCMPGEQ_M_N_F VRNDNQ_M_F VMINNMAVQ_P_F
 127                          VCMPNEQ_M_N_F VRMLALDAVHQ_P_S VRMLALDAVHXQ_P_S
 128                          VCMPEQQ_M_N_F VCMLAQ_ROT270_F VMAXNMAQ_M_F VRNDQ_M_F
 129                          VMLALDAVQ_P_U VMLALDAVQ_P_S VQMOVNBQ_M_S VQMOVNBQ_M_U
 130                          VMOVLTQ_M_U VMOVLTQ_M_S VMOVNBQ_M_U VMOVNBQ_M_S
 131                          VRSHRNTQ_N_U VRSHRNTQ_N_S VORRQ_M_N_S VORRQ_M_N_U
 132                          VREV32Q_M_S VREV32Q_M_U VQRSHRNTQ_N_U VQRSHRNTQ_N_S
 133                          VMOVNTQ_M_U VMOVNTQ_M_S VMOVLBQ_M_U VMOVLBQ_M_S
 134                          VMLALDAVAQ_S VMLALDAVAQ_U VQSHRNBQ_N_U VQSHRNBQ_N_S
 135                          VSHRNBQ_N_U VSHRNBQ_N_S VRSHRNBQ_N_S VRSHRNBQ_N_U
 136                          VMLALDAVXQ_P_U VMLALDAVXQ_P_S VQMOVNTQ_M_U VQMOVNTQ_M_S
 137                          VMVNQ_M_N_U VMVNQ_M_N_S VQSHRNTQ_N_U VQSHRNTQ_N_S
 138                          VMLALDAVAXQ_S VMLALDAVAXQ_U VSHRNTQ_N_S VSHRNTQ_N_U
 139                          VCVTBQ_M_F16_F32 VCVTBQ_M_F32_F16 VCVTTQ_M_F16_F32
 140                          VCVTTQ_M_F32_F16 VCVTMQ_M_S VCVTMQ_M_U VCVTNQ_M_S
 141                          VCVTPQ_M_S VCVTPQ_M_U VCVTQ_M_N_FROM_F_S VCVTNQ_M_U
 142                          VREV16Q_M_S VREV16Q_M_U VREV32Q_M VCVTQ_M_FROM_F_U
 143                          VCVTQ_M_FROM_F_S VRMLALDAVHQ_P_U VADDLVAQ_P_U
 144                          VCVTQ_M_N_FROM_F_U VQSHLUQ_M_N_S VABAVQ_P_S
 145                          VABAVQ_P_U VSHLQ_M_S VSHLQ_M_U VSRIQ_M_N_S
 146                          VSRIQ_M_N_U VSUBQ_M_U VSUBQ_M_S VCVTQ_M_N_TO_F_U
 147                          VCVTQ_M_N_TO_F_S VQADDQ_M_U VQADDQ_M_S
 148                          VRSHRQ_M_N_S VSUBQ_M_N_S VSUBQ_M_N_U VBRSRQ_M_N_S
 149                          VSUBQ_M_N_F VBICQ_M_F VHADDQ_M_U VBICQ_M_U VBICQ_M_S
 150                          VMULQ_M_N_U VHADDQ_M_S VORNQ_M_F VMLAQ_M_N_S VQSUBQ_M_U
 151                          VQSUBQ_M_S VMLAQ_M_N_U VQSUBQ_M_N_U VQSUBQ_M_N_S
 152                          VMULLTQ_INT_M_S VMULLTQ_INT_M_U VMULQ_M_N_S VMULQ_M_N_F
 153                          VMLASQ_M_N_U VMLASQ_M_N_S VMAXQ_M_U VQRDMLAHQ_M_N_U
 154                          VCADDQ_ROT270_M_F VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S
 155                          VQRSHLQ_M_S VMULQ_M_F VRHADDQ_M_U VSHRQ_M_N_U
 156                          VRHADDQ_M_S VMULQ_M_S VMULQ_M_U VQRDMLASHQ_M_N_S
 157                          VRSHLQ_M_S VRSHLQ_M_U VRSHRQ_M_N_U VADDQ_M_N_F
 158                          VADDQ_M_N_S VADDQ_M_N_U VQRDMLASHQ_M_N_U VMAXQ_M_S
 159                          VQRDMLAHQ_M_N_S VORRQ_M_S VORRQ_M_U VORRQ_M_F
 160                          VQRSHLQ_M_U VRMULHQ_M_U VRMULHQ_M_S VMINQ_M_S VMINQ_M_U
 161                          VANDQ_M_F VANDQ_M_U VANDQ_M_S VHSUBQ_M_N_S VHSUBQ_M_N_U
 162                          VMULHQ_M_S VMULHQ_M_U VMULLBQ_INT_M_U
 163                          VMULLBQ_INT_M_S VCADDQ_ROT90_M_F
 164                          VSHRQ_M_N_S VADDQ_M_U VSLIQ_M_N_U
 165                          VQADDQ_M_N_S VBRSRQ_M_N_F VABDQ_M_F VBRSRQ_M_N_U
 166                          VEORQ_M_F VSHLQ_M_N_S VQDMLAHQ_M_N_U VQDMLAHQ_M_N_S
 167                          VSHLQ_M_N_U VMLADAVAQ_P_U VMLADAVAQ_P_S VSLIQ_M_N_S
 168                          VQSHLQ_M_U VQSHLQ_M_S VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S
 169                          VORNQ_M_U VORNQ_M_S VQSHLQ_M_N_S VQSHLQ_M_N_U VADDQ_M_S
 170                          VHADDQ_M_N_S VADDQ_M_F VQADDQ_M_N_U VEORQ_M_S VEORQ_M_U
 171                          VHSUBQ_M_S VHSUBQ_M_U VHADDQ_M_N_U VHCADDQ_ROT90_M_S
 172                          VQRDMLSDHQ_M_S VQRDMLSDHXQ_M_S VQRDMLADHXQ_M_S
 173                          VQDMULHQ_M_S VMLADAVAXQ_P_S VQDMLADHXQ_M_S
 174                          VQRDMULHQ_M_S VMLSDAVAXQ_P_S VQDMULHQ_M_N_S
 175                          VHCADDQ_ROT270_M_S VQDMLSDHQ_M_S VQDMLSDHXQ_M_S
 176                          VMLSDAVAQ_P_S VQRDMLADHQ_M_S VQDMLADHQ_M_S
 177                          VMLALDAVAQ_P_U VMLALDAVAQ_P_S VMLALDAVAXQ_P_U
 178                          VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S VQRSHRNTQ_M_N_S
 179                          VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S VQSHRNTQ_M_N_S
 180                          VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S VRSHRNTQ_M_N_U
 181                          VSHLLBQ_M_N_U VSHLLBQ_M_N_S VSHLLTQ_M_N_U VSHLLTQ_M_N_S
 182                          VSHRNBQ_M_N_S VSHRNBQ_M_N_U VSHRNTQ_M_N_S VSHRNTQ_M_N_U
 183                          VMLALDAVAXQ_P_S VQRSHRNTQ_M_N_U VQSHRNTQ_M_N_U
 184                          VRSHRNTQ_M_N_S VQRDMULHQ_M_N_S VRMLALDAVHAQ_P_S
 185                          VMLSLDAVAQ_P_S VMLSLDAVAXQ_P_S VMULLBQ_POLY_M_P
 186                          VMULLTQ_POLY_M_P VQDMULLBQ_M_N_S VQDMULLBQ_M_S
 187                          VQDMULLTQ_M_N_S VQDMULLTQ_M_S VQRSHRUNBQ_M_N_S
 188                          VQRSHRUNTQ_M_N_SVQSHRUNBQ_M_N_S VQSHRUNTQ_M_N_S
 189                          VRMLALDAVHAQ_P_U VRMLALDAVHAXQ_P_S VRMLSLDAVHAQ_P_S
 190                          VRMLSLDAVHAXQ_P_S VQRSHRUNTQ_M_N_S VQSHRUNBQ_M_N_S
 191                          VCMLAQ_M_F VCMLAQ_ROT180_M_F VCMLAQ_ROT270_M_F
 192                          VCMLAQ_ROT90_M_F VCMULQ_M_F VCMULQ_ROT180_M_F
 193                          VCMULQ_ROT270_M_F VCMULQ_ROT90_M_F VFMAQ_M_F
 194                          VFMAQ_M_N_F VFMASQ_M_N_F VFMSQ_M_F VMAXNMQ_M_F
 195                          VMINNMQ_M_F VSUBQ_M_F VSTRWQSB_S VSTRWQSB_U
 196                          VSTRBQSO_S VSTRBQSO_U VSTRBQ_S VSTRBQ_U VLDRBQGO_S
 197                          VLDRBQGO_U VLDRBQ_S VLDRBQ_U VLDRWQGB_S VLDRWQGB_U
 198                          VLD1Q_F VLD1Q_S VLD1Q_U VLDRHQ_F VLDRHQGO_S
 199                          VLDRHQGO_U VLDRHQGSO_S VLDRHQGSO_U VLDRHQ_S VLDRHQ_U
 200                          VLDRWQ_F VLDRWQ_S VLDRWQ_U VLDRDQGB_S VLDRDQGB_U
 201                          VLDRDQGO_S VLDRDQGO_U VLDRDQGSO_S VLDRDQGSO_U
 202                          VLDRHQGO_F VLDRHQGSO_F VLDRWQGB_F VLDRWQGO_F
 203                          VLDRWQGO_S VLDRWQGO_U VLDRWQGSO_F VLDRWQGSO_S
 204                          VLDRWQGSO_U VSTRHQ_F VST1Q_S VST1Q_U VSTRHQSO_S
 205                          VSTRHQSO_U VSTRHQSSO_S VSTRHQSSO_U VSTRHQ_S
 206                          VSTRHQ_U VSTRWQ_S VSTRWQ_U VSTRWQ_F VST1Q_F VSTRDQSB_S
 207                          VSTRDQSB_U VSTRDQSO_S VSTRDQSO_U VSTRDQSSO_S
 208                          VSTRDQSSO_U VSTRWQSO_S VSTRWQSO_U VSTRWQSSO_S
 209                          VSTRWQSSO_U VSTRHQSO_F VSTRHQSSO_F VSTRWQSB_F
 210                          VSTRWQSO_F VSTRWQSSO_F VDDUPQ VDDUPQ_M VDWDUPQ
 211                          VDWDUPQ_M VIDUPQ VIDUPQ_M VIWDUPQ VIWDUPQ_M
 212                          VSTRWQSBWB_S VSTRWQSBWB_U VLDRWQGBWB_S VLDRWQGBWB_U
 213                          VSTRWQSBWB_F VLDRWQGBWB_F VSTRDQSBWB_S VSTRDQSBWB_U
 214                          VLDRDQGBWB_S VLDRDQGBWB_U VADCQ_U VADCQ_M_U VADCQ_S
 215                          VADCQ_M_S VSBCIQ_U VSBCIQ_S VSBCIQ_M_U VSBCIQ_M_S
 216                          VSBCQ_U VSBCQ_S VSBCQ_M_U VSBCQ_M_S VADCIQ_U VADCIQ_M_U
 217                          VADCIQ_S VADCIQ_M_S VLD2Q VLD4Q VST2Q SRSHRL SRSHR
 218                          URSHR URSHRL SQRSHR UQRSHL UQRSHLL_64
 219                          UQRSHLL_48 SQRSHRL_64 SQRSHRL_48])
 220
 221 (define_mode_attr MVE_CNVT [(V8HI "V8HF") (V4SI "V4SF") (V8HF "V8HI")
 222                             (V4SF "V4SI")])
 223
 224 (define_int_attr supf [(VCVTQ_TO_F_S "s") (VCVTQ_TO_F_U "u") (VREV16Q_S "s")
 225                        (VREV16Q_U "u") (VMVNQ_N_S "s") (VMVNQ_N_U "u")
 226                        (VCVTAQ_U "u") (VCVTAQ_S "s") (VREV64Q_S "s")
 227                        (VREV64Q_U "u") (VMVNQ_S "s") (VMVNQ_U "u")
 228                        (VDUPQ_N_U "u") (VDUPQ_N_S"s") (VADDVQ_S "s")
 229                        (VADDVQ_U "u") (VADDVQ_S "s") (VADDVQ_U "u")
 230                        (VMOVLTQ_U "u") (VMOVLTQ_S "s") (VMOVLBQ_S "s")
 231                        (VMOVLBQ_U "u") (VCVTQ_FROM_F_S "s") (VCVTQ_FROM_F_U "u")
 232                        (VCVTPQ_S "s") (VCVTPQ_U "u") (VCVTNQ_S "s")
 233                        (VCVTNQ_U "u") (VCVTMQ_S "s") (VCVTMQ_U "u")
 234                        (VCLZQ_U "u") (VCLZQ_S "s") (VREV32Q_U "u")
 235                        (VREV32Q_S "s") (VADDLVQ_U "u") (VADDLVQ_S "s")
 236                        (VCVTQ_N_TO_F_S "s") (VCVTQ_N_TO_F_U "u")
 237                        (VCREATEQ_U "u") (VCREATEQ_S "s") (VSHRQ_N_S "s")
 238                        (VSHRQ_N_U "u") (VCVTQ_N_FROM_F_S "s") (VSHLQ_U "u")
 239                        (VCVTQ_N_FROM_F_U "u") (VADDLVQ_P_S "s") (VSHLQ_S "s")
 240                        (VADDLVQ_P_U "u") (VCMPNEQ_U "u") (VCMPNEQ_S "s")
 241                        (VABDQ_M_S "s") (VABDQ_M_U "u") (VABDQ_S "s")
 242                        (VABDQ_U "u") (VADDQ_N_S "s") (VADDQ_N_U "u")
 243                        (VADDVQ_P_S "s") (VADDVQ_P_U "u") (VANDQ_S "s")
 244                        (VANDQ_U "u") (VBICQ_S "s") (VBICQ_U "u")
 245                        (VBRSRQ_N_S "s") (VBRSRQ_N_U "u") (VCADDQ_ROT270_S "s")
 246                        (VCADDQ_ROT270_U "u") (VCADDQ_ROT90_S "s")
 247                        (VCMPEQQ_S "s") (VCMPEQQ_U "u") (VCADDQ_ROT90_U "u")
 248                        (VCMPEQQ_N_S "s") (VCMPEQQ_N_U "u") (VCMPNEQ_N_S "s")
 249                        (VCMPNEQ_N_U "u") (VEORQ_S "s") (VEORQ_U "u")
 250                        (VHADDQ_N_S "s") (VHADDQ_N_U "u") (VHADDQ_S "s")
 251                        (VHADDQ_U "u") (VHSUBQ_N_S "s")  (VHSUBQ_N_U "u")
 252                        (VHSUBQ_S "s") (VMAXQ_S "s") (VMAXQ_U "u") (VHSUBQ_U "u")
 253                        (VMAXVQ_S "s") (VMAXVQ_U "u") (VMINQ_S "s") (VMINQ_U "u")
 254                        (VMINVQ_S "s") (VMINVQ_U "u") (VMLADAVQ_S "s")
 255                        (VMLADAVQ_U "u") (VMULHQ_S "s") (VMULHQ_U "u")
 256                        (VMULLBQ_INT_S "s") (VMULLBQ_INT_U "u") (VQADDQ_S "s")
 257                        (VMULLTQ_INT_S "s") (VMULLTQ_INT_U "u") (VQADDQ_U "u")
 258                        (VMULQ_N_S "s") (VMULQ_N_U "u") (VMULQ_S "s")
 259                        (VMULQ_U "u") (VORNQ_S "s") (VORNQ_U "u") (VORRQ_S "s")
 260                        (VORRQ_U "u") (VQADDQ_N_S "s") (VQADDQ_N_U "u")
 261                        (VQRSHLQ_N_S "s") (VQRSHLQ_N_U "u") (VQRSHLQ_S "s")
 262                        (VQRSHLQ_U "u") (VQSHLQ_N_S "s") (VQSHLQ_N_U "u")
 263                        (VQSHLQ_R_S "s") (VQSHLQ_R_U "u") (VQSHLQ_S "s")
 264                        (VQSHLQ_U "u") (VQSUBQ_N_S "s") (VQSUBQ_N_U "u")
 265                        (VQSUBQ_S "s") (VQSUBQ_U "u") (VRHADDQ_S "s")
 266                        (VRHADDQ_U "u") (VRMULHQ_S "s") (VRMULHQ_U "u")
 267                        (VRSHLQ_N_S "s") (VRSHLQ_N_U "u") (VRSHLQ_S "s")
 268                        (VRSHLQ_U "u") (VRSHRQ_N_S "s") (VRSHRQ_N_U "u")
 269                        (VSHLQ_N_S "s") (VSHLQ_N_U "u") (VSHLQ_R_S "s")
 270                        (VSHLQ_R_U "u") (VSUBQ_N_S "s") (VSUBQ_N_U "u")
 271                        (VSUBQ_S "s") (VSUBQ_U "u") (VADDVAQ_S "s")
 272                        (VADDVAQ_U "u") (VADDLVAQ_S "s") (VADDLVAQ_U "u")
 273                        (VBICQ_N_S "s") (VBICQ_N_U "u") (VMLALDAVQ_U "u")
 274                        (VMLALDAVQ_S "s") (VMLALDAVXQ_U "u") (VMLALDAVXQ_S "s")
 275                        (VMOVNBQ_U "u") (VMOVNBQ_S "s") (VMOVNTQ_U "u")
 276                        (VMOVNTQ_S "s") (VORRQ_N_S "s") (VORRQ_N_U "u")
 277                        (VQMOVNBQ_U "u") (VQMOVNBQ_S "s") (VQMOVNTQ_S "s")
 278                        (VQMOVNTQ_U "u") (VSHLLBQ_N_U "u") (VSHLLBQ_N_S "s")
 279                        (VSHLLTQ_N_U "u") (VSHLLTQ_N_S "s") (VRMLALDAVHQ_U "u")
 280                        (VRMLALDAVHQ_S "s") (VBICQ_M_N_S "s") (VBICQ_M_N_U "u")
 281                        (VCVTAQ_M_S "s") (VCVTAQ_M_U "u") (VCVTQ_M_TO_F_S "s")
 282                        (VCVTQ_M_TO_F_U "u") (VQRSHRNBQ_N_S "s")
 283                        (VQRSHRNBQ_N_U "u") (VABAVQ_S "s") (VABAVQ_U "u")
 284                        (VRMLALDAVHAQ_U "u") (VRMLALDAVHAQ_S "s") (VSHLCQ_S "s")
 285                        (VSHLCQ_U "u") (VADDVAQ_P_S "s") (VADDVAQ_P_U "u")
 286                        (VCLZQ_M_S "s") (VCLZQ_M_U "u") (VCMPEQQ_M_N_S "s")
 287                        (VCMPEQQ_M_N_U "u") (VCMPEQQ_M_S "s") (VCMPEQQ_M_U "u")
 288                        (VCMPNEQ_M_N_S "s") (VCMPNEQ_M_N_U "u") (VCMPNEQ_M_S "s")
 289                        (VCMPNEQ_M_U "u") (VDUPQ_M_N_S "s") (VDUPQ_M_N_U "u")
 290                        (VMAXVQ_P_S "s") (VMAXVQ_P_U "u") (VMINVQ_P_S "s")
 291                        (VMINVQ_P_U "u") (VMLADAVAQ_S "s") (VMLADAVAQ_U "u")
 292                        (VMLADAVQ_P_S "s") (VMLADAVQ_P_U "u") (VMLAQ_N_S "s")
 293                        (VMLAQ_N_U "u") (VMLASQ_N_S "s") (VMLASQ_N_U "u")
 294                        (VMVNQ_M_S "s") (VMVNQ_M_U "u") (VPSELQ_S "s")
 295                        (VPSELQ_U "u") (VQDMLAHQ_N_S "s") (VQDMLAHQ_N_U "u")
 296                        (VQRDMLAHQ_N_S "s") (VQRDMLAHQ_N_U "u")
 297                        (VQRDMLASHQ_N_S "s") (VQRDMLASHQ_N_U "u")
 298                        (VQRSHLQ_M_N_S "s") (VQRSHLQ_M_N_U "u")
 299                        (VQSHLQ_M_R_S "s") (VQSHLQ_M_R_U "u") (VSRIQ_N_S "s")
 300                        (VREV64Q_M_S "s") (VREV64Q_M_U "u") (VSRIQ_N_U "u")
 301                        (VRSHLQ_M_N_S "s") (VRSHLQ_M_N_U "u") (VSHLQ_M_R_S "s")
 302                        (VSHLQ_M_R_U "u") (VSLIQ_N_S "s") (VSLIQ_N_U "u")
 303                        (VMLALDAVQ_P_S "s") (VQMOVNBQ_M_S "s") (VMOVLTQ_M_S "s")
 304                        (VMOVNBQ_M_S "s") (VRSHRNTQ_N_S "s") (VORRQ_M_N_S "s")
 305                        (VREV32Q_M_S "s") (VQRSHRNTQ_N_S "s") (VMOVNTQ_M_S "s")
 306                        (VMOVLBQ_M_S "s") (VMLALDAVAQ_S "s") (VQSHRNBQ_N_S "s")
 307                        (VSHRNBQ_N_S "s") (VRSHRNBQ_N_S "s") (VMLALDAVXQ_P_S "s")
 308                        (VQMOVNTQ_M_S "s") (VMVNQ_M_N_S "s") (VQSHRNTQ_N_S "s")
 309                        (VMLALDAVAXQ_S "s") (VSHRNTQ_N_S "s") (VMLALDAVQ_P_U "u")
 310                        (VQMOVNBQ_M_U "u") (VMOVLTQ_M_U "u") (VMOVNBQ_M_U "u")
 311                        (VRSHRNTQ_N_U "u") (VORRQ_M_N_U "u") (VREV32Q_M_U "u")
 312                        (VREV16Q_M_S "s") (VREV16Q_M_U "u")
 313                        (VQRSHRNTQ_N_U "u") (VMOVNTQ_M_U "u") (VMOVLBQ_M_U "u")
 314                        (VMLALDAVAQ_U "u") (VQSHRNBQ_N_U "u") (VSHRNBQ_N_U "u")
 315                        (VRSHRNBQ_N_U "u") (VMLALDAVXQ_P_U "u")
 316                        (VMVNQ_M_N_U "u") (VQSHRNTQ_N_U "u") (VMLALDAVAXQ_U "u")
 317                        (VQMOVNTQ_M_U "u") (VSHRNTQ_N_U "u") (VCVTMQ_M_S "s")
 318                        (VCVTMQ_M_U "u") (VCVTNQ_M_S "s") (VCVTNQ_M_U "u")
 319                        (VCVTPQ_M_S "s") (VCVTPQ_M_U "u") (VADDLVAQ_P_S "s")
 320                        (VCVTQ_M_N_FROM_F_U "u") (VCVTQ_M_FROM_F_S "s")
 321                        (VCVTQ_M_FROM_F_U "u") (VRMLALDAVHQ_P_U "u")
 322                        (VRMLALDAVHQ_P_S "s") (VADDLVAQ_P_U "u")
 323                        (VCVTQ_M_N_FROM_F_S "s") (VABAVQ_P_U "u")
 324                        (VABAVQ_P_S "s") (VSHLQ_M_S "s") (VSHLQ_M_U "u")
 325                        (VSRIQ_M_N_S "s") (VSRIQ_M_N_U "u") (VSUBQ_M_S "s")
 326                        (VSUBQ_M_U "u") (VCVTQ_M_N_TO_F_S "s")
 327                        (VCVTQ_M_N_TO_F_U "u") (VADDQ_M_N_U "u")
 328                        (VSHLQ_M_N_S "s") (VMAXQ_M_U "u") (VHSUBQ_M_N_U "u")
 329                        (VMULQ_M_N_S "s") (VQSHLQ_M_U "u") (VRHADDQ_M_S "s")
 330                        (VEORQ_M_U "u") (VSHRQ_M_N_U "u") (VCADDQ_ROT90_M_U "u")
 331                        (VMLADAVAQ_P_U "u") (VEORQ_M_S "s") (VBRSRQ_M_N_S "s")
 332                        (VMULQ_M_U "u") (VQRDMLAHQ_M_N_S "s") (VHSUBQ_M_N_S "s")
 333                        (VQRSHLQ_M_S "s") (VMULQ_M_N_U "u")
 334                        (VMULQ_M_S "s") (VQSHLQ_M_N_U "u") (VSLIQ_M_N_U "u")
 335                        (VMLADAVAQ_P_S "s") (VQRSHLQ_M_U "u")
 336                        (VMULLBQ_INT_M_U "u") (VSHLQ_M_N_U "u") (VQSUBQ_M_U "u")
 337                        (VQRDMLASHQ_M_N_U "u") (VRSHRQ_M_N_S "s")
 338                        (VORNQ_M_S "s") (VCADDQ_ROT270_M_S "s") (VRHADDQ_M_U "u")
 339                        (VRSHRQ_M_N_U "u") (VMLASQ_M_N_U "u") (VHSUBQ_M_U "u")
 340                        (VQSUBQ_M_N_S "s") (VMULLTQ_INT_M_S "s")
 341                        (VORRQ_M_S "s") (VQDMLAHQ_M_N_U "u") (VRSHLQ_M_S "s")
 342                        (VHADDQ_M_U "u") (VHADDQ_M_N_S "s") (VMULLTQ_INT_M_U "u")
 343                        (VORRQ_M_U "u") (VHADDQ_M_S "s") (VHADDQ_M_N_U "u")
 344                        (VQDMLAHQ_M_N_S "s") (VMAXQ_M_S "s") (VORNQ_M_U "u")
 345                        (VCADDQ_ROT270_M_U "u") (VQADDQ_M_U "u")
 346                        (VQRDMLASHQ_M_N_S "s") (VBICQ_M_U "u") (VMINQ_M_U "u")
 347                        (VSUBQ_M_N_S "s") (VMULLBQ_INT_M_S "s") (VQSUBQ_M_S "s")
 348                        (VCADDQ_ROT90_M_S "s") (VRMULHQ_M_S "s") (VANDQ_M_U "u")
 349                        (VMULHQ_M_S "s") (VADDQ_M_S "s") (VQRDMLAHQ_M_N_U "u")
 350                        (VMLASQ_M_N_S "s") (VHSUBQ_M_S "s") (VRMULHQ_M_U "u")
 351                        (VQADDQ_M_N_S "s") (VSHRQ_M_N_S "s") (VANDQ_M_S "s")
 352                        (VABDQ_M_U "u") (VQSHLQ_M_S "s") (VABDQ_M_S "s")
 353                        (VSUBQ_M_N_U "u") (VMLAQ_M_N_S "s") (VBRSRQ_M_N_U "u")
 354                        (VADDQ_M_U "u") (VRSHLQ_M_U "u") (VSLIQ_M_N_S "s")
 355                        (VQADDQ_M_N_U "u") (VADDQ_M_N_S "s") (VQSUBQ_M_N_U "u")
 356                        (VMLAQ_M_N_U "u") (VMINQ_M_S "s") (VMULHQ_M_U "u")
 357                        (VQADDQ_M_S "s") (VBICQ_M_S "s") (VQSHLQ_M_N_S "s")
 358                        (VQSHRNTQ_M_N_S "s") (VQSHRNTQ_M_N_U "u")
 359                        (VSHRNTQ_M_N_U "u") (VSHRNTQ_M_N_S "s")
 360                        (VSHRNBQ_M_N_S "s") (VSHRNBQ_M_N_U "u")
 361                        (VSHLLTQ_M_N_S "s") (VSHLLTQ_M_N_U "u")
 362                        (VSHLLBQ_M_N_S "s") (VSHLLBQ_M_N_U "u")
 363                        (VRSHRNTQ_M_N_S "s") (VRSHRNTQ_M_N_U "u")
 364                        (VRSHRNBQ_M_N_U "u") (VRSHRNBQ_M_N_S "s")
 365                        (VQSHRNTQ_M_N_U "u") (VQSHRNTQ_M_N_S "s")
 366                        (VQSHRNBQ_M_N_S "s") (VQSHRNBQ_M_N_U "u")
 367                        (VQRSHRNTQ_M_N_S "s") (VQRSHRNTQ_M_N_U "u")
 368                        (VQRSHRNBQ_M_N_S "s") (VQRSHRNBQ_M_N_U "u")
 369                        (VMLALDAVAXQ_P_S "s") (VMLALDAVAXQ_P_U "u")
 370                        (VMLALDAVAQ_P_S "s") (VMLALDAVAQ_P_U "u")
 371                        (VSTRWQSB_S "s") (VSTRWQSB_U "u") (VSTRBQSO_S "s")
 372                        (VSTRBQSO_U "u") (VSTRBQ_S "s") (VSTRBQ_U "u")
 373                        (VLDRBQGO_S "s") (VLDRBQGO_U "u") (VLDRBQ_S "s")
 374                        (VLDRBQ_U "u") (VLDRWQGB_S "s") (VLDRWQGB_U "u")
 375                        (VLD1Q_S "s") (VLD1Q_U "u") (VLDRHQGO_S "s")
 376                        (VLDRHQGO_U "u") (VLDRHQGSO_S "s") (VLDRHQGSO_U "u")
 377                        (VLDRHQ_S "s") (VLDRHQ_U "u") (VLDRWQ_S "s")
 378                        (VLDRWQ_U "u") (VLDRDQGB_S "s") (VLDRDQGB_U "u")
 379                        (VLDRDQGO_S "s") (VLDRDQGO_U "u") (VLDRDQGSO_S "s")
 380                        (VLDRDQGSO_U "u") (VLDRWQGO_S "s") (VLDRWQGO_U "u")
 381                        (VLDRWQGSO_S "s") (VLDRWQGSO_U "u") (VST1Q_S "s")
 382                        (VST1Q_U "u") (VSTRHQSO_S "s") (VSTRHQSO_U "u")
 383                        (VSTRHQSSO_S "s") (VSTRHQSSO_U "u") (VSTRHQ_S "s")
 384                        (VSTRHQ_U "u") (VSTRWQ_S "s") (VSTRWQ_U "u")
 385                        (VSTRDQSB_S "s") (VSTRDQSB_U "u") (VSTRDQSO_S "s")
 386                        (VSTRDQSO_U "u") (VSTRDQSSO_S "s") (VSTRDQSSO_U "u")
 387                        (VSTRWQSO_U "u") (VSTRWQSO_S "s") (VSTRWQSSO_U "u")
 388                        (VSTRWQSSO_S "s") (VSTRWQSBWB_S "s") (VSTRWQSBWB_U "u")
 389                        (VLDRWQGBWB_S "s") (VLDRWQGBWB_U "u") (VLDRDQGBWB_S "s")
 390                        (VLDRDQGBWB_U "u") (VSTRDQSBWB_S "s") (VADCQ_M_S "s")
 391                        (VSTRDQSBWB_U "u") (VSBCQ_U "u")  (VSBCQ_M_U "u")
 392                        (VSBCQ_S "s")  (VSBCQ_M_S "s") (VSBCIQ_U "u")
 393                        (VSBCIQ_M_U "u") (VSBCIQ_S "s") (VSBCIQ_M_S "s")
 394                        (VADCQ_U "u")  (VADCQ_M_U "u") (VADCQ_S "s")
 395                        (VADCIQ_U "u") (VADCIQ_M_U "u") (VADCIQ_S "s")
 396                        (VADCIQ_M_S "s") (SQRSHRL_64 "64") (SQRSHRL_48 "48")
 397                        (UQRSHLL_64 "64") (UQRSHLL_48 "48")])
 398
 399 (define_int_attr mode1 [(VCTP8Q "8") (VCTP16Q "16") (VCTP32Q "32")
 400                         (VCTP64Q "64") (VCTP8Q_M "8") (VCTP16Q_M "16")
 401                         (VCTP32Q_M "32") (VCTP64Q_M "64")])
 402 (define_mode_attr MVE_pred2 [(V16QI "mve_imm_8") (V8HI "mve_imm_16")
 403                              (V4SI "mve_imm_32")])
 404 (define_mode_attr MVE_constraint2 [(V16QI "Rb") (V8HI "Rd") (V4SI "Rf")])
 405 (define_mode_attr MVE_LANES [(V16QI "16") (V8HI "8") (V4SI "4")])
 406 (define_mode_attr MVE_constraint [ (V16QI "Ra") (V8HI "Rc") (V4SI "Re")])
 407 (define_mode_attr MVE_pred [ (V16QI "mve_imm_7") (V8HI "mve_imm_15")
 408                                    (V4SI "mve_imm_31")])
 409 (define_mode_attr MVE_constraint3 [ (V8HI "Rb") (V4SI "Rd")])
 410 (define_mode_attr MVE_pred3 [ (V8HI "mve_imm_8") (V4SI "mve_imm_16")])
 411 (define_mode_attr MVE_constraint1 [ (V8HI "Ra") (V4SI "Rc")])
 412 (define_mode_attr MVE_pred1 [ (V8HI "mve_imm_7") (V4SI "mve_imm_15")])
 413 (define_mode_attr MVE_B_ELEM [ (V16QI "V16QI") (V8HI "V8QI") (V4SI "V4QI")])
 414 (define_mode_attr MVE_H_ELEM [ (V8HI "V8HI") (V4SI "V4HI")])
 415 (define_mode_attr V_sz_elem1 [(V16QI "b") (V8HI  "h") (V4SI "w") (V8HF "h")
 416                               (V4SF "w")])
 417 (define_mode_attr V_extr_elem [(V16QI "u8") (V8HI "u16") (V4SI "32")
 418                               (V8HF "u16") (V4SF "32")])
 419
 420 (define_mode_attr earlyclobber_32 [(V16QI "=w") (V8HI "=w") (V4SI "=&w")
 421                                                 (V8HF "=w") (V4SF "=&w")])
 422
 423 (define_int_iterator VCVTQ_TO_F [VCVTQ_TO_F_S VCVTQ_TO_F_U])
 424 (define_int_iterator VMVNQ_N [VMVNQ_N_U VMVNQ_N_S])
 425 (define_int_iterator VREV64Q [VREV64Q_S VREV64Q_U])
 426 (define_int_iterator VCVTQ_FROM_F [VCVTQ_FROM_F_S VCVTQ_FROM_F_U])
 427 (define_int_iterator VREV16Q [VREV16Q_U VREV16Q_S])
 428 (define_int_iterator VCVTAQ [VCVTAQ_U VCVTAQ_S])
 429 (define_int_iterator VMVNQ [VMVNQ_U VMVNQ_S])
 430 (define_int_iterator VDUPQ_N [VDUPQ_N_U VDUPQ_N_S])
 431 (define_int_iterator VCLZQ [VCLZQ_U VCLZQ_S])
 432 (define_int_iterator VADDVQ [VADDVQ_U VADDVQ_S])
 433 (define_int_iterator VREV32Q [VREV32Q_U VREV32Q_S])
 434 (define_int_iterator VMOVLBQ [VMOVLBQ_S VMOVLBQ_U])
 435 (define_int_iterator VMOVLTQ [VMOVLTQ_U VMOVLTQ_S])
 436 (define_int_iterator VCVTPQ [VCVTPQ_S VCVTPQ_U])
 437 (define_int_iterator VCVTNQ [VCVTNQ_S VCVTNQ_U])
 438 (define_int_iterator VCVTMQ [VCVTMQ_S VCVTMQ_U])
 439 (define_int_iterator VADDLVQ [VADDLVQ_U VADDLVQ_S])
 440 (define_int_iterator VCTPQ [VCTP8Q VCTP16Q VCTP32Q VCTP64Q])
 441 (define_int_iterator VCTPQ_M [VCTP8Q_M VCTP16Q_M VCTP32Q_M VCTP64Q_M])
 442 (define_int_iterator VCVTQ_N_TO_F [VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U])
 443 (define_int_iterator VCREATEQ [VCREATEQ_U VCREATEQ_S])
 444 (define_int_iterator VSHRQ_N [VSHRQ_N_S VSHRQ_N_U])
 445 (define_int_iterator VCVTQ_N_FROM_F [VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U])
 446 (define_int_iterator VADDLVQ_P [VADDLVQ_P_S VADDLVQ_P_U])
 447 (define_int_iterator VCMPNEQ [VCMPNEQ_U VCMPNEQ_S])
 448 (define_int_iterator VSHLQ [VSHLQ_S VSHLQ_U])
 449 (define_int_iterator VABDQ [VABDQ_S VABDQ_U])
 450 (define_int_iterator VADDQ_N [VADDQ_N_S VADDQ_N_U])
 451 (define_int_iterator VADDVAQ [VADDVAQ_S VADDVAQ_U])
 452 (define_int_iterator VADDVQ_P [VADDVQ_P_U VADDVQ_P_S])
 453 (define_int_iterator VANDQ [VANDQ_U VANDQ_S])
 454 (define_int_iterator VBICQ [VBICQ_S VBICQ_U])
 455 (define_int_iterator VBRSRQ_N [VBRSRQ_N_U VBRSRQ_N_S])
 456 (define_int_iterator VCADDQ_ROT270 [VCADDQ_ROT270_S VCADDQ_ROT270_U])
 457 (define_int_iterator VCADDQ_ROT90 [VCADDQ_ROT90_U VCADDQ_ROT90_S])
 458 (define_int_iterator VCMPEQQ [VCMPEQQ_U VCMPEQQ_S])
 459 (define_int_iterator VCMPEQQ_N [VCMPEQQ_N_S VCMPEQQ_N_U])
 460 (define_int_iterator VCMPNEQ_N [VCMPNEQ_N_U VCMPNEQ_N_S])
 461 (define_int_iterator VEORQ [VEORQ_U VEORQ_S])
 462 (define_int_iterator VHADDQ [VHADDQ_S VHADDQ_U])
 463 (define_int_iterator VHADDQ_N [VHADDQ_N_U VHADDQ_N_S])
 464 (define_int_iterator VHSUBQ [VHSUBQ_S VHSUBQ_U])
 465 (define_int_iterator VHSUBQ_N [VHSUBQ_N_U VHSUBQ_N_S])
 466 (define_int_iterator VMAXQ [VMAXQ_U VMAXQ_S])
 467 (define_int_iterator VMAXVQ [VMAXVQ_U VMAXVQ_S])
 468 (define_int_iterator VMINQ [VMINQ_S VMINQ_U])
 469 (define_int_iterator VMINVQ [VMINVQ_U VMINVQ_S])
 470 (define_int_iterator VMLADAVQ [VMLADAVQ_U VMLADAVQ_S])
 471 (define_int_iterator VMULHQ [VMULHQ_S VMULHQ_U])
 472 (define_int_iterator VMULLBQ_INT [VMULLBQ_INT_U VMULLBQ_INT_S])
 473 (define_int_iterator VMULLTQ_INT [VMULLTQ_INT_U VMULLTQ_INT_S])
 474 (define_int_iterator VMULQ [VMULQ_U VMULQ_S])
 475 (define_int_iterator VMULQ_N [VMULQ_N_U VMULQ_N_S])
 476 (define_int_iterator VORNQ [VORNQ_U VORNQ_S])
 477 (define_int_iterator VORRQ [VORRQ_S VORRQ_U])
 478 (define_int_iterator VQADDQ [VQADDQ_U VQADDQ_S])
 479 (define_int_iterator VQADDQ_N [VQADDQ_N_S VQADDQ_N_U])
 480 (define_int_iterator VQRSHLQ [VQRSHLQ_S VQRSHLQ_U])
 481 (define_int_iterator VQRSHLQ_N [VQRSHLQ_N_S VQRSHLQ_N_U])
 482 (define_int_iterator VQSHLQ [VQSHLQ_S VQSHLQ_U])
 483 (define_int_iterator VQSHLQ_N [VQSHLQ_N_S VQSHLQ_N_U])
 484 (define_int_iterator VQSHLQ_R [VQSHLQ_R_U VQSHLQ_R_S])
 485 (define_int_iterator VQSUBQ [VQSUBQ_U VQSUBQ_S])
 486 (define_int_iterator VQSUBQ_N [VQSUBQ_N_S VQSUBQ_N_U])
 487 (define_int_iterator VRHADDQ [VRHADDQ_S VRHADDQ_U])
 488 (define_int_iterator VRMULHQ [VRMULHQ_S VRMULHQ_U])
 489 (define_int_iterator VRSHLQ [VRSHLQ_S VRSHLQ_U])
 490 (define_int_iterator VRSHLQ_N [VRSHLQ_N_U VRSHLQ_N_S])
 491 (define_int_iterator VRSHRQ_N [VRSHRQ_N_S VRSHRQ_N_U])
 492 (define_int_iterator VSHLQ_N [VSHLQ_N_U VSHLQ_N_S])
 493 (define_int_iterator VSHLQ_R [VSHLQ_R_S VSHLQ_R_U])
 494 (define_int_iterator VSUBQ [VSUBQ_S VSUBQ_U])
 495 (define_int_iterator VSUBQ_N [VSUBQ_N_S VSUBQ_N_U])
 496 (define_int_iterator VADDLVAQ [VADDLVAQ_S VADDLVAQ_U])
 497 (define_int_iterator VBICQ_N [VBICQ_N_S VBICQ_N_U])
 498 (define_int_iterator VMLALDAVQ [VMLALDAVQ_U VMLALDAVQ_S])
 499 (define_int_iterator VMLALDAVXQ [VMLALDAVXQ_U VMLALDAVXQ_S])
 500 (define_int_iterator VMOVNBQ [VMOVNBQ_U VMOVNBQ_S])
 501 (define_int_iterator VMOVNTQ [VMOVNTQ_S VMOVNTQ_U])
 502 (define_int_iterator VORRQ_N [VORRQ_N_U VORRQ_N_S])
 503 (define_int_iterator VQMOVNBQ [VQMOVNBQ_U VQMOVNBQ_S])
 504 (define_int_iterator VQMOVNTQ [VQMOVNTQ_U VQMOVNTQ_S])
 505 (define_int_iterator VSHLLBQ_N [VSHLLBQ_N_S VSHLLBQ_N_U])
 506 (define_int_iterator VSHLLTQ_N [VSHLLTQ_N_U VSHLLTQ_N_S])
 507 (define_int_iterator VRMLALDAVHQ [VRMLALDAVHQ_U VRMLALDAVHQ_S])
 508 (define_int_iterator VBICQ_M_N [VBICQ_M_N_S VBICQ_M_N_U])
 509 (define_int_iterator VCVTAQ_M [VCVTAQ_M_S VCVTAQ_M_U])
 510 (define_int_iterator VCVTQ_M_TO_F [VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U])
 511 (define_int_iterator VQRSHRNBQ_N [VQRSHRNBQ_N_U VQRSHRNBQ_N_S])
 512 (define_int_iterator VABAVQ [VABAVQ_S VABAVQ_U])
 513 (define_int_iterator VSHLCQ [VSHLCQ_S VSHLCQ_U])
 514 (define_int_iterator VRMLALDAVHAQ [VRMLALDAVHAQ_S VRMLALDAVHAQ_U])
 515 (define_int_iterator VADDVAQ_P [VADDVAQ_P_S VADDVAQ_P_U])
 516 (define_int_iterator VCLZQ_M [VCLZQ_M_S VCLZQ_M_U])
 517 (define_int_iterator VCMPEQQ_M_N [VCMPEQQ_M_N_S VCMPEQQ_M_N_U])
 518 (define_int_iterator VCMPEQQ_M [VCMPEQQ_M_S VCMPEQQ_M_U])
 519 (define_int_iterator VCMPNEQ_M_N [VCMPNEQ_M_N_S VCMPNEQ_M_N_U])
 520 (define_int_iterator VCMPNEQ_M [VCMPNEQ_M_S VCMPNEQ_M_U])
 521 (define_int_iterator VDUPQ_M_N [VDUPQ_M_N_S VDUPQ_M_N_U])
 522 (define_int_iterator VMAXVQ_P [VMAXVQ_P_S VMAXVQ_P_U])
 523 (define_int_iterator VMINVQ_P [VMINVQ_P_S VMINVQ_P_U])
 524 (define_int_iterator VMLADAVAQ [VMLADAVAQ_S VMLADAVAQ_U])
 525 (define_int_iterator VMLADAVQ_P [VMLADAVQ_P_S VMLADAVQ_P_U])
 526 (define_int_iterator VMLAQ_N [VMLAQ_N_S VMLAQ_N_U])
 527 (define_int_iterator VMLASQ_N [VMLASQ_N_S VMLASQ_N_U])
 528 (define_int_iterator VMVNQ_M [VMVNQ_M_S VMVNQ_M_U])
 529 (define_int_iterator VPSELQ [VPSELQ_S VPSELQ_U])
 530 (define_int_iterator VQDMLAHQ_N [VQDMLAHQ_N_S VQDMLAHQ_N_U])
 531 (define_int_iterator VQRDMLAHQ_N [VQRDMLAHQ_N_S VQRDMLAHQ_N_U])
 532 (define_int_iterator VQRDMLASHQ_N [VQRDMLASHQ_N_S VQRDMLASHQ_N_U])
 533 (define_int_iterator VQRSHLQ_M_N [VQRSHLQ_M_N_S VQRSHLQ_M_N_U])
 534 (define_int_iterator VQSHLQ_M_R [VQSHLQ_M_R_S VQSHLQ_M_R_U])
 535 (define_int_iterator VREV64Q_M [VREV64Q_M_S VREV64Q_M_U])
 536 (define_int_iterator VRSHLQ_M_N [VRSHLQ_M_N_S VRSHLQ_M_N_U])
 537 (define_int_iterator VSHLQ_M_R [VSHLQ_M_R_S VSHLQ_M_R_U])
 538 (define_int_iterator VSLIQ_N [VSLIQ_N_S VSLIQ_N_U])
 539 (define_int_iterator VSRIQ_N [VSRIQ_N_S VSRIQ_N_U])
 540 (define_int_iterator VMLALDAVQ_P [VMLALDAVQ_P_U VMLALDAVQ_P_S])
 541 (define_int_iterator VQMOVNBQ_M [VQMOVNBQ_M_S VQMOVNBQ_M_U])
 542 (define_int_iterator VMOVLTQ_M [VMOVLTQ_M_U VMOVLTQ_M_S])
 543 (define_int_iterator VMOVNBQ_M [VMOVNBQ_M_U VMOVNBQ_M_S])
 544 (define_int_iterator VRSHRNTQ_N [VRSHRNTQ_N_U VRSHRNTQ_N_S])
 545 (define_int_iterator VORRQ_M_N [VORRQ_M_N_S VORRQ_M_N_U])
 546 (define_int_iterator VREV32Q_M [VREV32Q_M_S VREV32Q_M_U])
 547 (define_int_iterator VREV16Q_M [VREV16Q_M_S VREV16Q_M_U])
 548 (define_int_iterator VQRSHRNTQ_N [VQRSHRNTQ_N_U VQRSHRNTQ_N_S])
 549 (define_int_iterator VMOVNTQ_M [VMOVNTQ_M_U VMOVNTQ_M_S])
 550 (define_int_iterator VMOVLBQ_M [VMOVLBQ_M_U VMOVLBQ_M_S])
 551 (define_int_iterator VMLALDAVAQ [VMLALDAVAQ_S VMLALDAVAQ_U])
 552 (define_int_iterator VQSHRNBQ_N [VQSHRNBQ_N_U VQSHRNBQ_N_S])
 553 (define_int_iterator VSHRNBQ_N [VSHRNBQ_N_U VSHRNBQ_N_S])
 554 (define_int_iterator VRSHRNBQ_N [VRSHRNBQ_N_S VRSHRNBQ_N_U])
 555 (define_int_iterator VMLALDAVXQ_P [VMLALDAVXQ_P_U VMLALDAVXQ_P_S])
 556 (define_int_iterator VQMOVNTQ_M [VQMOVNTQ_M_U VQMOVNTQ_M_S])
 557 (define_int_iterator VMVNQ_M_N [VMVNQ_M_N_U VMVNQ_M_N_S])
 558 (define_int_iterator VQSHRNTQ_N [VQSHRNTQ_N_U VQSHRNTQ_N_S])
 559 (define_int_iterator VMLALDAVAXQ [VMLALDAVAXQ_S VMLALDAVAXQ_U])
 560 (define_int_iterator VSHRNTQ_N [VSHRNTQ_N_S VSHRNTQ_N_U])
 561 (define_int_iterator VCVTMQ_M [VCVTMQ_M_S VCVTMQ_M_U])
 562 (define_int_iterator VCVTNQ_M [VCVTNQ_M_S VCVTNQ_M_U])
 563 (define_int_iterator VCVTPQ_M [VCVTPQ_M_S VCVTPQ_M_U])
 564 (define_int_iterator VCVTQ_M_N_FROM_F [VCVTQ_M_N_FROM_F_S VCVTQ_M_N_FROM_F_U])
 565 (define_int_iterator VCVTQ_M_FROM_F [VCVTQ_M_FROM_F_U VCVTQ_M_FROM_F_S])
 566 (define_int_iterator VRMLALDAVHQ_P [VRMLALDAVHQ_P_S VRMLALDAVHQ_P_U])
 567 (define_int_iterator VADDLVAQ_P [VADDLVAQ_P_U VADDLVAQ_P_S])
 568 (define_int_iterator VABAVQ_P [VABAVQ_P_S VABAVQ_P_U])
 569 (define_int_iterator VSHLQ_M [VSHLQ_M_S VSHLQ_M_U])
 570 (define_int_iterator VSRIQ_M_N [VSRIQ_M_N_S VSRIQ_M_N_U])
 571 (define_int_iterator VSUBQ_M [VSUBQ_M_U VSUBQ_M_S])
 572 (define_int_iterator VCVTQ_M_N_TO_F [VCVTQ_M_N_TO_F_U VCVTQ_M_N_TO_F_S])
 573 (define_int_iterator VHSUBQ_M [VHSUBQ_M_S VHSUBQ_M_U])
 574 (define_int_iterator VSLIQ_M_N [VSLIQ_M_N_U VSLIQ_M_N_S])
 575 (define_int_iterator VRSHLQ_M [VRSHLQ_M_S VRSHLQ_M_U])
 576 (define_int_iterator VMINQ_M [VMINQ_M_S VMINQ_M_U])
 577 (define_int_iterator VMULLBQ_INT_M [VMULLBQ_INT_M_U VMULLBQ_INT_M_S])
 578 (define_int_iterator VMULHQ_M [VMULHQ_M_S VMULHQ_M_U])
 579 (define_int_iterator VMULQ_M [VMULQ_M_S VMULQ_M_U])
 580 (define_int_iterator VHSUBQ_M_N [VHSUBQ_M_N_S VHSUBQ_M_N_U])
 581 (define_int_iterator VHADDQ_M_N [VHADDQ_M_N_S VHADDQ_M_N_U])
 582 (define_int_iterator VORRQ_M [VORRQ_M_S VORRQ_M_U])
 583 (define_int_iterator VRMULHQ_M [VRMULHQ_M_U VRMULHQ_M_S])
 584 (define_int_iterator VQADDQ_M [VQADDQ_M_U VQADDQ_M_S])
 585 (define_int_iterator VRSHRQ_M_N [VRSHRQ_M_N_S VRSHRQ_M_N_U])
 586 (define_int_iterator VQSUBQ_M_N [VQSUBQ_M_N_U VQSUBQ_M_N_S])
 587 (define_int_iterator VADDQ_M [VADDQ_M_U VADDQ_M_S])
 588 (define_int_iterator VORNQ_M [VORNQ_M_U VORNQ_M_S])
 589 (define_int_iterator VRHADDQ_M [VRHADDQ_M_U VRHADDQ_M_S])
 590 (define_int_iterator VQSHLQ_M [VQSHLQ_M_U VQSHLQ_M_S])
 591 (define_int_iterator VANDQ_M [VANDQ_M_U VANDQ_M_S])
 592 (define_int_iterator VBICQ_M [VBICQ_M_U VBICQ_M_S])
 593 (define_int_iterator VSHLQ_M_N [VSHLQ_M_N_S VSHLQ_M_N_U])
 594 (define_int_iterator VCADDQ_ROT270_M [VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S])
 595 (define_int_iterator VQRSHLQ_M [VQRSHLQ_M_U VQRSHLQ_M_S])
 596 (define_int_iterator VQADDQ_M_N [VQADDQ_M_N_U VQADDQ_M_N_S])
 597 (define_int_iterator VADDQ_M_N [VADDQ_M_N_S VADDQ_M_N_U])
 598 (define_int_iterator VMAXQ_M [VMAXQ_M_S VMAXQ_M_U])
 599 (define_int_iterator VQSUBQ_M [VQSUBQ_M_U VQSUBQ_M_S])
 600 (define_int_iterator VMLASQ_M_N [VMLASQ_M_N_U VMLASQ_M_N_S])
 601 (define_int_iterator VMLADAVAQ_P [VMLADAVAQ_P_U VMLADAVAQ_P_S])
 602 (define_int_iterator VBRSRQ_M_N [VBRSRQ_M_N_U VBRSRQ_M_N_S])
 603 (define_int_iterator VMULQ_M_N [VMULQ_M_N_U VMULQ_M_N_S])
 604 (define_int_iterator VCADDQ_ROT90_M [VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S])
 605 (define_int_iterator VMULLTQ_INT_M [VMULLTQ_INT_M_S VMULLTQ_INT_M_U])
 606 (define_int_iterator VEORQ_M [VEORQ_M_S VEORQ_M_U])
 607 (define_int_iterator VSHRQ_M_N [VSHRQ_M_N_S VSHRQ_M_N_U])
 608 (define_int_iterator VSUBQ_M_N [VSUBQ_M_N_S VSUBQ_M_N_U])
 609 (define_int_iterator VHADDQ_M [VHADDQ_M_S VHADDQ_M_U])
 610 (define_int_iterator VABDQ_M [VABDQ_M_S VABDQ_M_U])
 611 (define_int_iterator VMLAQ_M_N [VMLAQ_M_N_S VMLAQ_M_N_U])
 612 (define_int_iterator VQSHLQ_M_N [VQSHLQ_M_N_S VQSHLQ_M_N_U])
 613 (define_int_iterator VMLALDAVAQ_P [VMLALDAVAQ_P_U VMLALDAVAQ_P_S])
 614 (define_int_iterator VMLALDAVAXQ_P [VMLALDAVAXQ_P_U VMLALDAVAXQ_P_S])
 615 (define_int_iterator VQRSHRNBQ_M_N [VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S])
 616 (define_int_iterator VQRSHRNTQ_M_N [VQRSHRNTQ_M_N_S VQRSHRNTQ_M_N_U])
 617 (define_int_iterator VQSHRNBQ_M_N [VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S])
 618 (define_int_iterator VQSHRNTQ_M_N [VQSHRNTQ_M_N_S VQSHRNTQ_M_N_U])
 619 (define_int_iterator VRSHRNBQ_M_N [VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S])
 620 (define_int_iterator VRSHRNTQ_M_N [VRSHRNTQ_M_N_U VRSHRNTQ_M_N_S])
 621 (define_int_iterator VSHLLBQ_M_N [VSHLLBQ_M_N_U VSHLLBQ_M_N_S])
 622 (define_int_iterator VSHLLTQ_M_N [VSHLLTQ_M_N_U VSHLLTQ_M_N_S])
 623 (define_int_iterator VSHRNBQ_M_N [VSHRNBQ_M_N_S VSHRNBQ_M_N_U])
 624 (define_int_iterator VSHRNTQ_M_N [VSHRNTQ_M_N_S VSHRNTQ_M_N_U])
 625 (define_int_iterator VSTRWSBQ [VSTRWQSB_S VSTRWQSB_U])
 626 (define_int_iterator VSTRBSOQ [VSTRBQSO_S VSTRBQSO_U])
 627 (define_int_iterator VSTRBQ [VSTRBQ_S VSTRBQ_U])
 628 (define_int_iterator VLDRBGOQ [VLDRBQGO_S VLDRBQGO_U])
 629 (define_int_iterator VLDRBQ [VLDRBQ_S VLDRBQ_U])
 630 (define_int_iterator VLDRWGBQ [VLDRWQGB_S VLDRWQGB_U])
 631 (define_int_iterator VLD1Q [VLD1Q_S VLD1Q_U])
 632 (define_int_iterator VLDRHGOQ [VLDRHQGO_S VLDRHQGO_U])
 633 (define_int_iterator VLDRHGSOQ [VLDRHQGSO_S VLDRHQGSO_U])
 634 (define_int_iterator VLDRHQ [VLDRHQ_S VLDRHQ_U])
 635 (define_int_iterator VLDRWQ [VLDRWQ_S VLDRWQ_U])
 636 (define_int_iterator VLDRDGBQ [VLDRDQGB_S VLDRDQGB_U])
 637 (define_int_iterator VLDRDGOQ [VLDRDQGO_S VLDRDQGO_U])
 638 (define_int_iterator VLDRDGSOQ [VLDRDQGSO_S VLDRDQGSO_U])
 639 (define_int_iterator VLDRWGOQ [VLDRWQGO_S VLDRWQGO_U])
 640 (define_int_iterator VLDRWGSOQ [VLDRWQGSO_S VLDRWQGSO_U])
 641 (define_int_iterator VST1Q [VST1Q_S VST1Q_U])
 642 (define_int_iterator VSTRHSOQ [VSTRHQSO_S VSTRHQSO_U])
 643 (define_int_iterator VSTRHSSOQ [VSTRHQSSO_S VSTRHQSSO_U])
 644 (define_int_iterator VSTRHQ [VSTRHQ_S VSTRHQ_U])
 645 (define_int_iterator VSTRWQ [VSTRWQ_S VSTRWQ_U])
 646 (define_int_iterator VSTRDSBQ [VSTRDQSB_S VSTRDQSB_U])
 647 (define_int_iterator VSTRDSOQ [VSTRDQSO_S VSTRDQSO_U])
 648 (define_int_iterator VSTRDSSOQ [VSTRDQSSO_S VSTRDQSSO_U])
 649 (define_int_iterator VSTRWSOQ [VSTRWQSO_S VSTRWQSO_U])
 650 (define_int_iterator VSTRWSSOQ [VSTRWQSSO_S VSTRWQSSO_U])
 651 (define_int_iterator VSTRWSBWBQ [VSTRWQSBWB_S VSTRWQSBWB_U])
 652 (define_int_iterator VLDRWGBWBQ [VLDRWQGBWB_S VLDRWQGBWB_U])
 653 (define_int_iterator VSTRDSBWBQ [VSTRDQSBWB_S VSTRDQSBWB_U])
 654 (define_int_iterator VLDRDGBWBQ [VLDRDQGBWB_S VLDRDQGBWB_U])
 655 (define_int_iterator VADCIQ [VADCIQ_U VADCIQ_S])
 656 (define_int_iterator VADCIQ_M [VADCIQ_M_U VADCIQ_M_S])
 657 (define_int_iterator VSBCQ [VSBCQ_U VSBCQ_S])
 658 (define_int_iterator VSBCQ_M [VSBCQ_M_U VSBCQ_M_S])
 659 (define_int_iterator VSBCIQ [VSBCIQ_U VSBCIQ_S])
 660 (define_int_iterator VSBCIQ_M [VSBCIQ_M_U VSBCIQ_M_S])
 661 (define_int_iterator VADCQ [VADCQ_U VADCQ_S])
 662 (define_int_iterator VADCQ_M [VADCQ_M_U VADCQ_M_S])
 663 (define_int_iterator UQRSHLLQ [UQRSHLL_64 UQRSHLL_48])
 664 (define_int_iterator SQRSHRLQ [SQRSHRL_64 SQRSHRL_48])
 665
 666 (define_insn "*mve_mov<mode>"
 667   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Us")
 668         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Usi,r,Dm,w"))]
 669   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 670 {
 671   if (which_alternative == 3 || which_alternative == 6)
 672     {
 673       int width, is_valid;
 674       static char templ[40];
 675
 676       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
 677         &operands[1], &width);
 678
 679       gcc_assert (is_valid != 0);
 680
 681       if (width == 0)
 682         return "vmov.f32\t%q0, %1  @ <mode>";
 683       else
 684         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
 685       return templ;
 686     }
 687   switch (which_alternative)
 688     {
 689     case 0:
 690       return "vmov\t%q0, %q1";
 691     case 1:
 692       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 693     case 2:
 694       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 695     case 4:
 696       if ((TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))
 697           || (MEM_P (operands[1])
 698               && GET_CODE (XEXP (operands[1], 0)) == LABEL_REF))
 699         return output_move_neon (operands);
 700       else
 701         return "vldrb.8 %q0, %E1";
 702     case 5:
 703       return output_move_quad (operands);
 704     case 7:
 705       return "vstrb.8 %q1, %E0";
 706     default:
 707       gcc_unreachable ();
 708       return "";
 709     }
 710 }
 711   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store")
 712    (set_attr "length" "4,8,8,4,8,8,4,4")
 713    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*")
 714    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*")])
 715
 716 (define_insn "*mve_mov<mode>"
 717   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 718         (vec_duplicate:MVE_types
 719           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 720   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 721 {
 722   if (which_alternative == 0)
 723     return "vdup.<V_sz_elem>\t%q0, %1";
 724   return "vmov.<V_sz_elem>\t%q0, %1";
 725 }
 726   [(set_attr "length" "4,4")
 727    (set_attr "type" "mve_move,mve_move")])
 728
 729 ;;
 730 ;; [vst4q])
 731 ;;
 732 (define_insn "mve_vst4q<mode>"
 733   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 734         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 735                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 736          VST4Q))
 737   ]
 738   "TARGET_HAVE_MVE"
 739 {
 740    rtx ops[6];
 741    int regno = REGNO (operands[1]);
 742    ops[0] = gen_rtx_REG (TImode, regno);
 743    ops[1] = gen_rtx_REG (TImode, regno+4);
 744    ops[2] = gen_rtx_REG (TImode, regno+8);
 745    ops[3] = gen_rtx_REG (TImode, regno+12);
 746    rtx reg  = operands[0];
 747    while (reg && !REG_P (reg))
 748     reg = XEXP (reg, 0);
 749    gcc_assert (REG_P (reg));
 750    ops[4] = reg;
 751    ops[5] = operands[0];
 752    /* Here in first three instructions data is stored to ops[4]'s location but
 753       in the fourth instruction data is stored to operands[0], this is to
 754       support the writeback.  */
 755    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 756                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 757                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 758                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 759    return "";
 760 }
 761   [(set_attr "length" "16")])
 762
 763 ;;
 764 ;; [vrndq_m_f])
 765 ;;
 766 (define_insn "mve_vrndq_m_f<mode>"
 767   [
 768    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 769         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 770                        (match_operand:MVE_0 2 "s_register_operand" "w")
 771                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 772          VRNDQ_M_F))
 773   ]
 774   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 775   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 776   [(set_attr "type" "mve_move")
 777    (set_attr "length""8")])
 778
 779 ;;
 780 ;; [vrndxq_f])
 781 ;;
 782 (define_insn "mve_vrndxq_f<mode>"
 783   [
 784    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 785         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 786          VRNDXQ_F))
 787   ]
 788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 789   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 790   [(set_attr "type" "mve_move")
 791 ])
 792
 793 ;;
 794 ;; [vrndq_f])
 795 ;;
 796 (define_insn "mve_vrndq_f<mode>"
 797   [
 798    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 799         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 800          VRNDQ_F))
 801   ]
 802   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 803   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 804   [(set_attr "type" "mve_move")
 805 ])
 806
 807 ;;
 808 ;; [vrndpq_f])
 809 ;;
 810 (define_insn "mve_vrndpq_f<mode>"
 811   [
 812    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 813         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 814          VRNDPQ_F))
 815   ]
 816   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 817   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 818   [(set_attr "type" "mve_move")
 819 ])
 820
 821 ;;
 822 ;; [vrndnq_f])
 823 ;;
 824 (define_insn "mve_vrndnq_f<mode>"
 825   [
 826    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 827         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 828          VRNDNQ_F))
 829   ]
 830   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 831   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 832   [(set_attr "type" "mve_move")
 833 ])
 834
 835 ;;
 836 ;; [vrndmq_f])
 837 ;;
 838 (define_insn "mve_vrndmq_f<mode>"
 839   [
 840    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 841         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 842          VRNDMQ_F))
 843   ]
 844   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 845   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 846   [(set_attr "type" "mve_move")
 847 ])
 848
 849 ;;
 850 ;; [vrndaq_f])
 851 ;;
 852 (define_insn "mve_vrndaq_f<mode>"
 853   [
 854    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 855         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 856          VRNDAQ_F))
 857   ]
 858   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 859   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 860   [(set_attr "type" "mve_move")
 861 ])
 862
 863 ;;
 864 ;; [vrev64q_f])
 865 ;;
 866 (define_insn "mve_vrev64q_f<mode>"
 867   [
 868    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 869         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 870          VREV64Q_F))
 871   ]
 872   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 873   "vrev64.%#<V_sz_elem> %q0, %q1"
 874   [(set_attr "type" "mve_move")
 875 ])
 876
 877 ;;
 878 ;; [vnegq_f])
 879 ;;
 880 (define_insn "mve_vnegq_f<mode>"
 881   [
 882    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 883         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 884          VNEGQ_F))
 885   ]
 886   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 887   "vneg.f%#<V_sz_elem>  %q0, %q1"
 888   [(set_attr "type" "mve_move")
 889 ])
 890
 891 ;;
 892 ;; [vdupq_n_f])
 893 ;;
 894 (define_insn "mve_vdupq_n_f<mode>"
 895   [
 896    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 897         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 898          VDUPQ_N_F))
 899   ]
 900   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 901   "vdup.%#<V_sz_elem>   %q0, %1"
 902   [(set_attr "type" "mve_move")
 903 ])
 904
 905 ;;
 906 ;; [vabsq_f])
 907 ;;
 908 (define_insn "mve_vabsq_f<mode>"
 909   [
 910    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 911         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 912          VABSQ_F))
 913   ]
 914   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 915   "vabs.f%#<V_sz_elem>  %q0, %q1"
 916   [(set_attr "type" "mve_move")
 917 ])
 918
 919 ;;
 920 ;; [vrev32q_f])
 921 ;;
 922 (define_insn "mve_vrev32q_fv8hf"
 923   [
 924    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 925         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 926          VREV32Q_F))
 927   ]
 928   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 929   "vrev32.16 %q0, %q1"
 930   [(set_attr "type" "mve_move")
 931 ])
 932 ;;
 933 ;; [vcvttq_f32_f16])
 934 ;;
 935 (define_insn "mve_vcvttq_f32_f16v4sf"
 936   [
 937    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 938         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 939          VCVTTQ_F32_F16))
 940   ]
 941   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 942   "vcvtt.f32.f16 %q0, %q1"
 943   [(set_attr "type" "mve_move")
 944 ])
 945
 946 ;;
 947 ;; [vcvtbq_f32_f16])
 948 ;;
 949 (define_insn "mve_vcvtbq_f32_f16v4sf"
 950   [
 951    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 952         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 953          VCVTBQ_F32_F16))
 954   ]
 955   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 956   "vcvtb.f32.f16 %q0, %q1"
 957   [(set_attr "type" "mve_move")
 958 ])
 959
 960 ;;
 961 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 962 ;;
 963 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 964   [
 965    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 966         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 967          VCVTQ_TO_F))
 968   ]
 969   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 970   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 971   [(set_attr "type" "mve_move")
 972 ])
 973
 974 ;;
 975 ;; [vrev64q_u, vrev64q_s])
 976 ;;
 977 (define_insn "mve_vrev64q_<supf><mode>"
 978   [
 979    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 980         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 981          VREV64Q))
 982   ]
 983   "TARGET_HAVE_MVE"
 984   "vrev64.%#<V_sz_elem> %q0, %q1"
 985   [(set_attr "type" "mve_move")
 986 ])
 987
 988 ;;
 989 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 990 ;;
 991 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 992   [
 993    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 994         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 995          VCVTQ_FROM_F))
 996   ]
 997   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 998   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 999   [(set_attr "type" "mve_move")
1000 ])
1001 ;; [vqnegq_s])
1002 ;;
1003 (define_insn "mve_vqnegq_s<mode>"
1004   [
1005    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1006         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1007          VQNEGQ_S))
1008   ]
1009   "TARGET_HAVE_MVE"
1010   "vqneg.s%#<V_sz_elem> %q0, %q1"
1011   [(set_attr "type" "mve_move")
1012 ])
1013
1014 ;;
1015 ;; [vqabsq_s])
1016 ;;
1017 (define_insn "mve_vqabsq_s<mode>"
1018   [
1019    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1020         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1021          VQABSQ_S))
1022   ]
1023   "TARGET_HAVE_MVE"
1024   "vqabs.s%#<V_sz_elem> %q0, %q1"
1025   [(set_attr "type" "mve_move")
1026 ])
1027
1028 ;;
1029 ;; [vnegq_s])
1030 ;;
1031 (define_insn "mve_vnegq_s<mode>"
1032   [
1033    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1034         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1035          VNEGQ_S))
1036   ]
1037   "TARGET_HAVE_MVE"
1038   "vneg.s%#<V_sz_elem>  %q0, %q1"
1039   [(set_attr "type" "mve_move")
1040 ])
1041
1042 ;;
1043 ;; [vmvnq_u, vmvnq_s])
1044 ;;
1045 (define_insn "mve_vmvnq_<supf><mode>"
1046   [
1047    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1048         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1049          VMVNQ))
1050   ]
1051   "TARGET_HAVE_MVE"
1052   "vmvn %q0, %q1"
1053   [(set_attr "type" "mve_move")
1054 ])
1055
1056 ;;
1057 ;; [vdupq_n_u, vdupq_n_s])
1058 ;;
1059 (define_insn "mve_vdupq_n_<supf><mode>"
1060   [
1061    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1062         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
1063          VDUPQ_N))
1064   ]
1065   "TARGET_HAVE_MVE"
1066   "vdup.%#<V_sz_elem>   %q0, %1"
1067   [(set_attr "type" "mve_move")
1068 ])
1069
1070 ;;
1071 ;; [vclzq_u, vclzq_s])
1072 ;;
1073 (define_insn "mve_vclzq_<supf><mode>"
1074   [
1075    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1076         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1077          VCLZQ))
1078   ]
1079   "TARGET_HAVE_MVE"
1080   "vclz.i%#<V_sz_elem>  %q0, %q1"
1081   [(set_attr "type" "mve_move")
1082 ])
1083
1084 ;;
1085 ;; [vclsq_s])
1086 ;;
1087 (define_insn "mve_vclsq_s<mode>"
1088   [
1089    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1090         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1091          VCLSQ_S))
1092   ]
1093   "TARGET_HAVE_MVE"
1094   "vcls.s%#<V_sz_elem>  %q0, %q1"
1095   [(set_attr "type" "mve_move")
1096 ])
1097
1098 ;;
1099 ;; [vaddvq_s, vaddvq_u])
1100 ;;
1101 (define_insn "mve_vaddvq_<supf><mode>"
1102   [
1103    (set (match_operand:SI 0 "s_register_operand" "=e")
1104         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
1105          VADDVQ))
1106   ]
1107   "TARGET_HAVE_MVE"
1108   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
1109   [(set_attr "type" "mve_move")
1110 ])
1111
1112 ;;
1113 ;; [vabsq_s])
1114 ;;
1115 (define_insn "mve_vabsq_s<mode>"
1116   [
1117    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1118         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1119          VABSQ_S))
1120   ]
1121   "TARGET_HAVE_MVE"
1122   "vabs.s%#<V_sz_elem>\t%q0, %q1"
1123   [(set_attr "type" "mve_move")
1124 ])
1125
1126 ;;
1127 ;; [vrev32q_u, vrev32q_s])
1128 ;;
1129 (define_insn "mve_vrev32q_<supf><mode>"
1130   [
1131    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
1132         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
1133          VREV32Q))
1134   ]
1135   "TARGET_HAVE_MVE"
1136   "vrev32.%#<V_sz_elem>\t%q0, %q1"
1137   [(set_attr "type" "mve_move")
1138 ])
1139
1140 ;;
1141 ;; [vmovltq_u, vmovltq_s])
1142 ;;
1143 (define_insn "mve_vmovltq_<supf><mode>"
1144   [
1145    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1146         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1147          VMOVLTQ))
1148   ]
1149   "TARGET_HAVE_MVE"
1150   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
1151   [(set_attr "type" "mve_move")
1152 ])
1153
1154 ;;
1155 ;; [vmovlbq_s, vmovlbq_u])
1156 ;;
1157 (define_insn "mve_vmovlbq_<supf><mode>"
1158   [
1159    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1160         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1161          VMOVLBQ))
1162   ]
1163   "TARGET_HAVE_MVE"
1164   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
1165   [(set_attr "type" "mve_move")
1166 ])
1167
1168 ;;
1169 ;; [vcvtpq_s, vcvtpq_u])
1170 ;;
1171 (define_insn "mve_vcvtpq_<supf><mode>"
1172   [
1173    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1174         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1175          VCVTPQ))
1176   ]
1177   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1178   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1179   [(set_attr "type" "mve_move")
1180 ])
1181
1182 ;;
1183 ;; [vcvtnq_s, vcvtnq_u])
1184 ;;
1185 (define_insn "mve_vcvtnq_<supf><mode>"
1186   [
1187    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1188         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1189          VCVTNQ))
1190   ]
1191   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1192   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1193   [(set_attr "type" "mve_move")
1194 ])
1195
1196 ;;
1197 ;; [vcvtmq_s, vcvtmq_u])
1198 ;;
1199 (define_insn "mve_vcvtmq_<supf><mode>"
1200   [
1201    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1202         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1203          VCVTMQ))
1204   ]
1205   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1206   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1207   [(set_attr "type" "mve_move")
1208 ])
1209
1210 ;;
1211 ;; [vcvtaq_u, vcvtaq_s])
1212 ;;
1213 (define_insn "mve_vcvtaq_<supf><mode>"
1214   [
1215    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1216         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1217          VCVTAQ))
1218   ]
1219   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1220   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1221   [(set_attr "type" "mve_move")
1222 ])
1223
1224 ;;
1225 ;; [vmvnq_n_u, vmvnq_n_s])
1226 ;;
1227 (define_insn "mve_vmvnq_n_<supf><mode>"
1228   [
1229    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1230         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
1231          VMVNQ_N))
1232   ]
1233   "TARGET_HAVE_MVE"
1234   "vmvn.i%#<V_sz_elem>  %q0, %1"
1235   [(set_attr "type" "mve_move")
1236 ])
1237
1238 ;;
1239 ;; [vrev16q_u, vrev16q_s])
1240 ;;
1241 (define_insn "mve_vrev16q_<supf>v16qi"
1242   [
1243    (set (match_operand:V16QI 0 "s_register_operand" "=w")
1244         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
1245          VREV16Q))
1246   ]
1247   "TARGET_HAVE_MVE"
1248   "vrev16.8 %q0, %q1"
1249   [(set_attr "type" "mve_move")
1250 ])
1251
1252 ;;
1253 ;; [vaddlvq_s vaddlvq_u])
1254 ;;
1255 (define_insn "mve_vaddlvq_<supf>v4si"
1256   [
1257    (set (match_operand:DI 0 "s_register_operand" "=r")
1258         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
1259          VADDLVQ))
1260   ]
1261   "TARGET_HAVE_MVE"
1262   "vaddlv.<supf>32 %Q0, %R0, %q1"
1263   [(set_attr "type" "mve_move")
1264 ])
1265
1266 ;;
1267 ;; [vctp8q vctp16q vctp32q vctp64q])
1268 ;;
1269 (define_insn "mve_vctp<mode1>qhi"
1270   [
1271    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1272         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
1273         VCTPQ))
1274   ]
1275   "TARGET_HAVE_MVE"
1276   "vctp.<mode1> %1"
1277   [(set_attr "type" "mve_move")
1278 ])
1279
1280 ;;
1281 ;; [vpnot])
1282 ;;
1283 (define_insn "mve_vpnothi"
1284   [
1285    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1286         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
1287          VPNOT))
1288   ]
1289   "TARGET_HAVE_MVE"
1290   "vpnot"
1291   [(set_attr "type" "mve_move")
1292 ])
1293
1294 ;;
1295 ;; [vsubq_n_f])
1296 ;;
1297 (define_insn "mve_vsubq_n_f<mode>"
1298   [
1299    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1300         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1301                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1302          VSUBQ_N_F))
1303   ]
1304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1305   "vsub.f<V_sz_elem>  %q0, %q1, %2"
1306   [(set_attr "type" "mve_move")
1307 ])
1308
1309 ;;
1310 ;; [vbrsrq_n_f])
1311 ;;
1312 (define_insn "mve_vbrsrq_n_f<mode>"
1313   [
1314    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1315         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1316                        (match_operand:SI 2 "s_register_operand" "r")]
1317          VBRSRQ_N_F))
1318   ]
1319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1320   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
1321   [(set_attr "type" "mve_move")
1322 ])
1323
1324 ;;
1325 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
1326 ;;
1327 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
1328   [
1329    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1330         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1331                        (match_operand:SI 2 "mve_imm_16" "Rd")]
1332          VCVTQ_N_TO_F))
1333   ]
1334   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1335   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
1336   [(set_attr "type" "mve_move")
1337 ])
1338
1339 ;; [vcreateq_f])
1340 ;;
1341 (define_insn "mve_vcreateq_f<mode>"
1342   [
1343    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1344         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
1345                        (match_operand:DI 2 "s_register_operand" "r")]
1346          VCREATEQ_F))
1347   ]
1348   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1349   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1350   [(set_attr "type" "mve_move")
1351    (set_attr "length""8")])
1352
1353 ;;
1354 ;; [vcreateq_u, vcreateq_s])
1355 ;;
1356 (define_insn "mve_vcreateq_<supf><mode>"
1357   [
1358    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
1359         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
1360                        (match_operand:DI 2 "s_register_operand" "r")]
1361          VCREATEQ))
1362   ]
1363   "TARGET_HAVE_MVE"
1364   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1365   [(set_attr "type" "mve_move")
1366    (set_attr "length""8")])
1367
1368 ;;
1369 ;; [vshrq_n_s, vshrq_n_u])
1370 ;;
1371 (define_insn "mve_vshrq_n_<supf><mode>"
1372   [
1373    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1374         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1375                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1376          VSHRQ_N))
1377   ]
1378   "TARGET_HAVE_MVE"
1379   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
1380   [(set_attr "type" "mve_move")
1381 ])
1382
1383 ;;
1384 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
1385 ;;
1386 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
1387   [
1388    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1389         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1390                        (match_operand:SI 2 "mve_imm_16" "Rd")]
1391          VCVTQ_N_FROM_F))
1392   ]
1393   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1394   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
1395   [(set_attr "type" "mve_move")
1396 ])
1397
1398 ;;
1399 ;; [vaddlvq_p_s])
1400 ;;
1401 (define_insn "mve_vaddlvq_p_<supf>v4si"
1402   [
1403    (set (match_operand:DI 0 "s_register_operand" "=r")
1404         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
1405                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1406          VADDLVQ_P))
1407   ]
1408   "TARGET_HAVE_MVE"
1409   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
1410   [(set_attr "type" "mve_move")
1411    (set_attr "length""8")])
1412
1413 ;;
1414 ;; [vcmpneq_u, vcmpneq_s])
1415 ;;
1416 (define_insn "mve_vcmpneq_<supf><mode>"
1417   [
1418    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1419         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1420                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1421          VCMPNEQ))
1422   ]
1423   "TARGET_HAVE_MVE"
1424   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
1425   [(set_attr "type" "mve_move")
1426 ])
1427
1428 ;;
1429 ;; [vshlq_s, vshlq_u])
1430 ;;
1431 (define_insn "mve_vshlq_<supf><mode>"
1432   [
1433    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1434         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1435                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1436          VSHLQ))
1437   ]
1438   "TARGET_HAVE_MVE"
1439   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1440   [(set_attr "type" "mve_move")
1441 ])
1442
1443 ;;
1444 ;; [vabdq_s, vabdq_u])
1445 ;;
1446 (define_insn "mve_vabdq_<supf><mode>"
1447   [
1448    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1449         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1450                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1451          VABDQ))
1452   ]
1453   "TARGET_HAVE_MVE"
1454   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
1455   [(set_attr "type" "mve_move")
1456 ])
1457
1458 ;;
1459 ;; [vaddq_n_s, vaddq_n_u])
1460 ;;
1461 (define_insn "mve_vaddq_n_<supf><mode>"
1462   [
1463    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1464         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1465                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1466          VADDQ_N))
1467   ]
1468   "TARGET_HAVE_MVE"
1469   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
1470   [(set_attr "type" "mve_move")
1471 ])
1472
1473 ;;
1474 ;; [vaddvaq_s, vaddvaq_u])
1475 ;;
1476 (define_insn "mve_vaddvaq_<supf><mode>"
1477   [
1478    (set (match_operand:SI 0 "s_register_operand" "=e")
1479         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1480                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1481          VADDVAQ))
1482   ]
1483   "TARGET_HAVE_MVE"
1484   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
1485   [(set_attr "type" "mve_move")
1486 ])
1487
1488 ;;
1489 ;; [vaddvq_p_u, vaddvq_p_s])
1490 ;;
1491 (define_insn "mve_vaddvq_p_<supf><mode>"
1492   [
1493    (set (match_operand:SI 0 "s_register_operand" "=e")
1494         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1495                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1496          VADDVQ_P))
1497   ]
1498   "TARGET_HAVE_MVE"
1499   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
1500   [(set_attr "type" "mve_move")
1501    (set_attr "length""8")])
1502
1503 ;;
1504 ;; [vandq_u, vandq_s])
1505 ;;
1506 (define_insn "mve_vandq_<supf><mode>"
1507   [
1508    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1509         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1510                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1511          VANDQ))
1512   ]
1513   "TARGET_HAVE_MVE"
1514   "vand %q0, %q1, %q2"
1515   [(set_attr "type" "mve_move")
1516 ])
1517
1518 ;;
1519 ;; [vbicq_s, vbicq_u])
1520 ;;
1521 (define_insn "mve_vbicq_<supf><mode>"
1522   [
1523    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1524         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1525                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1526          VBICQ))
1527   ]
1528   "TARGET_HAVE_MVE"
1529   "vbic %q0, %q1, %q2"
1530   [(set_attr "type" "mve_move")
1531 ])
1532
1533 ;;
1534 ;; [vbrsrq_n_u, vbrsrq_n_s])
1535 ;;
1536 (define_insn "mve_vbrsrq_n_<supf><mode>"
1537   [
1538    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1539         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1540                        (match_operand:SI 2 "s_register_operand" "r")]
1541          VBRSRQ_N))
1542   ]
1543   "TARGET_HAVE_MVE"
1544   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
1545   [(set_attr "type" "mve_move")
1546 ])
1547
1548 ;;
1549 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
1550 ;;
1551 (define_insn "mve_vcaddq_rot270_<supf><mode>"
1552   [
1553    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1554         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1555                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1556          VCADDQ_ROT270))
1557   ]
1558   "TARGET_HAVE_MVE"
1559   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
1560   [(set_attr "type" "mve_move")
1561 ])
1562
1563 ;;
1564 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
1565 ;;
1566 (define_insn "mve_vcaddq_rot90_<supf><mode>"
1567   [
1568    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1569         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1570                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1571          VCADDQ_ROT90))
1572   ]
1573   "TARGET_HAVE_MVE"
1574   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
1575   [(set_attr "type" "mve_move")
1576 ])
1577
1578 ;;
1579 ;; [vcmpcsq_n_u])
1580 ;;
1581 (define_insn "mve_vcmpcsq_n_u<mode>"
1582   [
1583    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1584         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1585                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1586          VCMPCSQ_N_U))
1587   ]
1588   "TARGET_HAVE_MVE"
1589   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1590   [(set_attr "type" "mve_move")
1591 ])
1592
1593 ;;
1594 ;; [vcmpcsq_u])
1595 ;;
1596 (define_insn "mve_vcmpcsq_u<mode>"
1597   [
1598    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1599         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1600                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1601          VCMPCSQ_U))
1602   ]
1603   "TARGET_HAVE_MVE"
1604   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1605   [(set_attr "type" "mve_move")
1606 ])
1607
1608 ;;
1609 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1610 ;;
1611 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1612   [
1613    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1614         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1615                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1616          VCMPEQQ_N))
1617   ]
1618   "TARGET_HAVE_MVE"
1619   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1620   [(set_attr "type" "mve_move")
1621 ])
1622
1623 ;;
1624 ;; [vcmpeqq_u, vcmpeqq_s])
1625 ;;
1626 (define_insn "mve_vcmpeqq_<supf><mode>"
1627   [
1628    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1629         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1630                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1631          VCMPEQQ))
1632   ]
1633   "TARGET_HAVE_MVE"
1634   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1635   [(set_attr "type" "mve_move")
1636 ])
1637
1638 ;;
1639 ;; [vcmpgeq_n_s])
1640 ;;
1641 (define_insn "mve_vcmpgeq_n_s<mode>"
1642   [
1643    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1644         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1645                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1646          VCMPGEQ_N_S))
1647   ]
1648   "TARGET_HAVE_MVE"
1649   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1650   [(set_attr "type" "mve_move")
1651 ])
1652
1653 ;;
1654 ;; [vcmpgeq_s])
1655 ;;
1656 (define_insn "mve_vcmpgeq_s<mode>"
1657   [
1658    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1659         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1660                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1661          VCMPGEQ_S))
1662   ]
1663   "TARGET_HAVE_MVE"
1664   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1665   [(set_attr "type" "mve_move")
1666 ])
1667
1668 ;;
1669 ;; [vcmpgtq_n_s])
1670 ;;
1671 (define_insn "mve_vcmpgtq_n_s<mode>"
1672   [
1673    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1674         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1675                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1676          VCMPGTQ_N_S))
1677   ]
1678   "TARGET_HAVE_MVE"
1679   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1680   [(set_attr "type" "mve_move")
1681 ])
1682
1683 ;;
1684 ;; [vcmpgtq_s])
1685 ;;
1686 (define_insn "mve_vcmpgtq_s<mode>"
1687   [
1688    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1689         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1690                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1691          VCMPGTQ_S))
1692   ]
1693   "TARGET_HAVE_MVE"
1694   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1695   [(set_attr "type" "mve_move")
1696 ])
1697
1698 ;;
1699 ;; [vcmphiq_n_u])
1700 ;;
1701 (define_insn "mve_vcmphiq_n_u<mode>"
1702   [
1703    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1704         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1705                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1706          VCMPHIQ_N_U))
1707   ]
1708   "TARGET_HAVE_MVE"
1709   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1710   [(set_attr "type" "mve_move")
1711 ])
1712
1713 ;;
1714 ;; [vcmphiq_u])
1715 ;;
1716 (define_insn "mve_vcmphiq_u<mode>"
1717   [
1718    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1719         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1720                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1721          VCMPHIQ_U))
1722   ]
1723   "TARGET_HAVE_MVE"
1724   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1725   [(set_attr "type" "mve_move")
1726 ])
1727
1728 ;;
1729 ;; [vcmpleq_n_s])
1730 ;;
1731 (define_insn "mve_vcmpleq_n_s<mode>"
1732   [
1733    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1734         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1735                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1736          VCMPLEQ_N_S))
1737   ]
1738   "TARGET_HAVE_MVE"
1739   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1740   [(set_attr "type" "mve_move")
1741 ])
1742
1743 ;;
1744 ;; [vcmpleq_s])
1745 ;;
1746 (define_insn "mve_vcmpleq_s<mode>"
1747   [
1748    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1749         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1750                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1751          VCMPLEQ_S))
1752   ]
1753   "TARGET_HAVE_MVE"
1754   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1755   [(set_attr "type" "mve_move")
1756 ])
1757
1758 ;;
1759 ;; [vcmpltq_n_s])
1760 ;;
1761 (define_insn "mve_vcmpltq_n_s<mode>"
1762   [
1763    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1764         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1765                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1766          VCMPLTQ_N_S))
1767   ]
1768   "TARGET_HAVE_MVE"
1769   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1770   [(set_attr "type" "mve_move")
1771 ])
1772
1773 ;;
1774 ;; [vcmpltq_s])
1775 ;;
1776 (define_insn "mve_vcmpltq_s<mode>"
1777   [
1778    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1779         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1780                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1781          VCMPLTQ_S))
1782   ]
1783   "TARGET_HAVE_MVE"
1784   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1785   [(set_attr "type" "mve_move")
1786 ])
1787
1788 ;;
1789 ;; [vcmpneq_n_u, vcmpneq_n_s])
1790 ;;
1791 (define_insn "mve_vcmpneq_n_<supf><mode>"
1792   [
1793    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1794         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1795                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1796          VCMPNEQ_N))
1797   ]
1798   "TARGET_HAVE_MVE"
1799   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1800   [(set_attr "type" "mve_move")
1801 ])
1802
1803 ;;
1804 ;; [veorq_u, veorq_s])
1805 ;;
1806 (define_insn "mve_veorq_<supf><mode>"
1807   [
1808    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1809         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1810                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1811          VEORQ))
1812   ]
1813   "TARGET_HAVE_MVE"
1814   "veor %q0, %q1, %q2"
1815   [(set_attr "type" "mve_move")
1816 ])
1817
1818 ;;
1819 ;; [vhaddq_n_u, vhaddq_n_s])
1820 ;;
1821 (define_insn "mve_vhaddq_n_<supf><mode>"
1822   [
1823    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1824         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1825                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1826          VHADDQ_N))
1827   ]
1828   "TARGET_HAVE_MVE"
1829   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1830   [(set_attr "type" "mve_move")
1831 ])
1832
1833 ;;
1834 ;; [vhaddq_s, vhaddq_u])
1835 ;;
1836 (define_insn "mve_vhaddq_<supf><mode>"
1837   [
1838    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1839         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1840                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1841          VHADDQ))
1842   ]
1843   "TARGET_HAVE_MVE"
1844   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1845   [(set_attr "type" "mve_move")
1846 ])
1847
1848 ;;
1849 ;; [vhcaddq_rot270_s])
1850 ;;
1851 (define_insn "mve_vhcaddq_rot270_s<mode>"
1852   [
1853    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1854         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1855                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1856          VHCADDQ_ROT270_S))
1857   ]
1858   "TARGET_HAVE_MVE"
1859   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1860   [(set_attr "type" "mve_move")
1861 ])
1862
1863 ;;
1864 ;; [vhcaddq_rot90_s])
1865 ;;
1866 (define_insn "mve_vhcaddq_rot90_s<mode>"
1867   [
1868    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1869         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1870                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1871          VHCADDQ_ROT90_S))
1872   ]
1873   "TARGET_HAVE_MVE"
1874   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1875   [(set_attr "type" "mve_move")
1876 ])
1877
1878 ;;
1879 ;; [vhsubq_n_u, vhsubq_n_s])
1880 ;;
1881 (define_insn "mve_vhsubq_n_<supf><mode>"
1882   [
1883    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1884         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1885                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1886          VHSUBQ_N))
1887   ]
1888   "TARGET_HAVE_MVE"
1889   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1890   [(set_attr "type" "mve_move")
1891 ])
1892
1893 ;;
1894 ;; [vhsubq_s, vhsubq_u])
1895 ;;
1896 (define_insn "mve_vhsubq_<supf><mode>"
1897   [
1898    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1899         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1900                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1901          VHSUBQ))
1902   ]
1903   "TARGET_HAVE_MVE"
1904   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1905   [(set_attr "type" "mve_move")
1906 ])
1907
1908 ;;
1909 ;; [vmaxaq_s])
1910 ;;
1911 (define_insn "mve_vmaxaq_s<mode>"
1912   [
1913    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1914         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1915                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1916          VMAXAQ_S))
1917   ]
1918   "TARGET_HAVE_MVE"
1919   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1920   [(set_attr "type" "mve_move")
1921 ])
1922
1923 ;;
1924 ;; [vmaxavq_s])
1925 ;;
1926 (define_insn "mve_vmaxavq_s<mode>"
1927   [
1928    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1929         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1930                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1931          VMAXAVQ_S))
1932   ]
1933   "TARGET_HAVE_MVE"
1934   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1935   [(set_attr "type" "mve_move")
1936 ])
1937
1938 ;;
1939 ;; [vmaxq_u, vmaxq_s])
1940 ;;
1941 (define_insn "mve_vmaxq_<supf><mode>"
1942   [
1943    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1944         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1945                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1946          VMAXQ))
1947   ]
1948   "TARGET_HAVE_MVE"
1949   "vmax.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1950   [(set_attr "type" "mve_move")
1951 ])
1952
1953 ;;
1954 ;; [vmaxvq_u, vmaxvq_s])
1955 ;;
1956 (define_insn "mve_vmaxvq_<supf><mode>"
1957   [
1958    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1959         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1960                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1961          VMAXVQ))
1962   ]
1963   "TARGET_HAVE_MVE"
1964   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1965   [(set_attr "type" "mve_move")
1966 ])
1967
1968 ;;
1969 ;; [vminaq_s])
1970 ;;
1971 (define_insn "mve_vminaq_s<mode>"
1972   [
1973    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1974         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1975                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1976          VMINAQ_S))
1977   ]
1978   "TARGET_HAVE_MVE"
1979   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1980   [(set_attr "type" "mve_move")
1981 ])
1982
1983 ;;
1984 ;; [vminavq_s])
1985 ;;
1986 (define_insn "mve_vminavq_s<mode>"
1987   [
1988    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1989         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1990                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1991          VMINAVQ_S))
1992   ]
1993   "TARGET_HAVE_MVE"
1994   "vminav.s%#<V_sz_elem>\t%0, %q2"
1995   [(set_attr "type" "mve_move")
1996 ])
1997
1998 ;;
1999 ;; [vminq_s, vminq_u])
2000 ;;
2001 (define_insn "mve_vminq_<supf><mode>"
2002   [
2003    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2004         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2005                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2006          VMINQ))
2007   ]
2008   "TARGET_HAVE_MVE"
2009   "vmin.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2010   [(set_attr "type" "mve_move")
2011 ])
2012
2013 ;;
2014 ;; [vminvq_u, vminvq_s])
2015 ;;
2016 (define_insn "mve_vminvq_<supf><mode>"
2017   [
2018    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2019         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2020                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2021          VMINVQ))
2022   ]
2023   "TARGET_HAVE_MVE"
2024   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
2025   [(set_attr "type" "mve_move")
2026 ])
2027
2028 ;;
2029 ;; [vmladavq_u, vmladavq_s])
2030 ;;
2031 (define_insn "mve_vmladavq_<supf><mode>"
2032   [
2033    (set (match_operand:SI 0 "s_register_operand" "=e")
2034         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2035                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2036          VMLADAVQ))
2037   ]
2038   "TARGET_HAVE_MVE"
2039   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
2040   [(set_attr "type" "mve_move")
2041 ])
2042
2043 ;;
2044 ;; [vmladavxq_s])
2045 ;;
2046 (define_insn "mve_vmladavxq_s<mode>"
2047   [
2048    (set (match_operand:SI 0 "s_register_operand" "=e")
2049         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2050                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2051          VMLADAVXQ_S))
2052   ]
2053   "TARGET_HAVE_MVE"
2054   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2055   [(set_attr "type" "mve_move")
2056 ])
2057
2058 ;;
2059 ;; [vmlsdavq_s])
2060 ;;
2061 (define_insn "mve_vmlsdavq_s<mode>"
2062   [
2063    (set (match_operand:SI 0 "s_register_operand" "=e")
2064         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2065                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2066          VMLSDAVQ_S))
2067   ]
2068   "TARGET_HAVE_MVE"
2069   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
2070   [(set_attr "type" "mve_move")
2071 ])
2072
2073 ;;
2074 ;; [vmlsdavxq_s])
2075 ;;
2076 (define_insn "mve_vmlsdavxq_s<mode>"
2077   [
2078    (set (match_operand:SI 0 "s_register_operand" "=e")
2079         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2080                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2081          VMLSDAVXQ_S))
2082   ]
2083   "TARGET_HAVE_MVE"
2084   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2085   [(set_attr "type" "mve_move")
2086 ])
2087
2088 ;;
2089 ;; [vmulhq_s, vmulhq_u])
2090 ;;
2091 (define_insn "mve_vmulhq_<supf><mode>"
2092   [
2093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2095                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2096          VMULHQ))
2097   ]
2098   "TARGET_HAVE_MVE"
2099   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2100   [(set_attr "type" "mve_move")
2101 ])
2102
2103 ;;
2104 ;; [vmullbq_int_u, vmullbq_int_s])
2105 ;;
2106 (define_insn "mve_vmullbq_int_<supf><mode>"
2107   [
2108    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2109         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2110                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2111          VMULLBQ_INT))
2112   ]
2113   "TARGET_HAVE_MVE"
2114   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2115   [(set_attr "type" "mve_move")
2116 ])
2117
2118 ;;
2119 ;; [vmulltq_int_u, vmulltq_int_s])
2120 ;;
2121 (define_insn "mve_vmulltq_int_<supf><mode>"
2122   [
2123    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2124         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2125                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2126          VMULLTQ_INT))
2127   ]
2128   "TARGET_HAVE_MVE"
2129   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2130   [(set_attr "type" "mve_move")
2131 ])
2132
2133 ;;
2134 ;; [vmulq_n_u, vmulq_n_s])
2135 ;;
2136 (define_insn "mve_vmulq_n_<supf><mode>"
2137   [
2138    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2139         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2140                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2141          VMULQ_N))
2142   ]
2143   "TARGET_HAVE_MVE"
2144   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
2145   [(set_attr "type" "mve_move")
2146 ])
2147
2148 ;;
2149 ;; [vmulq_u, vmulq_s])
2150 ;;
2151 (define_insn "mve_vmulq_<supf><mode>"
2152   [
2153    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2154         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2155                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2156          VMULQ))
2157   ]
2158   "TARGET_HAVE_MVE"
2159   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
2160   [(set_attr "type" "mve_move")
2161 ])
2162
2163 ;;
2164 ;; [vornq_u, vornq_s])
2165 ;;
2166 (define_insn "mve_vornq_<supf><mode>"
2167   [
2168    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2169         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2170                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2171          VORNQ))
2172   ]
2173   "TARGET_HAVE_MVE"
2174   "vorn %q0, %q1, %q2"
2175   [(set_attr "type" "mve_move")
2176 ])
2177
2178 ;;
2179 ;; [vorrq_s, vorrq_u])
2180 ;;
2181 (define_insn "mve_vorrq_<supf><mode>"
2182   [
2183    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2184         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2185                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2186          VORRQ))
2187   ]
2188   "TARGET_HAVE_MVE"
2189   "vorr %q0, %q1, %q2"
2190   [(set_attr "type" "mve_move")
2191 ])
2192
2193 ;;
2194 ;; [vqaddq_n_s, vqaddq_n_u])
2195 ;;
2196 (define_insn "mve_vqaddq_n_<supf><mode>"
2197   [
2198    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2199         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2200                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2201          VQADDQ_N))
2202   ]
2203   "TARGET_HAVE_MVE"
2204   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2205   [(set_attr "type" "mve_move")
2206 ])
2207
2208 ;;
2209 ;; [vqaddq_u, vqaddq_s])
2210 ;;
2211 (define_insn "mve_vqaddq_<supf><mode>"
2212   [
2213    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2214         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2215                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2216          VQADDQ))
2217   ]
2218   "TARGET_HAVE_MVE"
2219   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2220   [(set_attr "type" "mve_move")
2221 ])
2222
2223 ;;
2224 ;; [vqdmulhq_n_s])
2225 ;;
2226 (define_insn "mve_vqdmulhq_n_s<mode>"
2227   [
2228    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2229         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2230                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2231          VQDMULHQ_N_S))
2232   ]
2233   "TARGET_HAVE_MVE"
2234   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2235   [(set_attr "type" "mve_move")
2236 ])
2237
2238 ;;
2239 ;; [vqdmulhq_s])
2240 ;;
2241 (define_insn "mve_vqdmulhq_s<mode>"
2242   [
2243    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2244         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2245                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2246          VQDMULHQ_S))
2247   ]
2248   "TARGET_HAVE_MVE"
2249   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2250   [(set_attr "type" "mve_move")
2251 ])
2252
2253 ;;
2254 ;; [vqrdmulhq_n_s])
2255 ;;
2256 (define_insn "mve_vqrdmulhq_n_s<mode>"
2257   [
2258    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2259         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2260                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2261          VQRDMULHQ_N_S))
2262   ]
2263   "TARGET_HAVE_MVE"
2264   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2265   [(set_attr "type" "mve_move")
2266 ])
2267
2268 ;;
2269 ;; [vqrdmulhq_s])
2270 ;;
2271 (define_insn "mve_vqrdmulhq_s<mode>"
2272   [
2273    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2274         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2275                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2276          VQRDMULHQ_S))
2277   ]
2278   "TARGET_HAVE_MVE"
2279   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2280   [(set_attr "type" "mve_move")
2281 ])
2282
2283 ;;
2284 ;; [vqrshlq_n_s, vqrshlq_n_u])
2285 ;;
2286 (define_insn "mve_vqrshlq_n_<supf><mode>"
2287   [
2288    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2289         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2290                        (match_operand:SI 2 "s_register_operand" "r")]
2291          VQRSHLQ_N))
2292   ]
2293   "TARGET_HAVE_MVE"
2294   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2295   [(set_attr "type" "mve_move")
2296 ])
2297
2298 ;;
2299 ;; [vqrshlq_s, vqrshlq_u])
2300 ;;
2301 (define_insn "mve_vqrshlq_<supf><mode>"
2302   [
2303    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2304         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2305                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2306          VQRSHLQ))
2307   ]
2308   "TARGET_HAVE_MVE"
2309   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2310   [(set_attr "type" "mve_move")
2311 ])
2312
2313 ;;
2314 ;; [vqshlq_n_s, vqshlq_n_u])
2315 ;;
2316 (define_insn "mve_vqshlq_n_<supf><mode>"
2317   [
2318    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2319         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2320                        (match_operand:SI 2 "immediate_operand" "i")]
2321          VQSHLQ_N))
2322   ]
2323   "TARGET_HAVE_MVE"
2324   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2325   [(set_attr "type" "mve_move")
2326 ])
2327
2328 ;;
2329 ;; [vqshlq_r_u, vqshlq_r_s])
2330 ;;
2331 (define_insn "mve_vqshlq_r_<supf><mode>"
2332   [
2333    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2334         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2335                        (match_operand:SI 2 "s_register_operand" "r")]
2336          VQSHLQ_R))
2337   ]
2338   "TARGET_HAVE_MVE"
2339   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
2340   [(set_attr "type" "mve_move")
2341 ])
2342
2343 ;;
2344 ;; [vqshlq_s, vqshlq_u])
2345 ;;
2346 (define_insn "mve_vqshlq_<supf><mode>"
2347   [
2348    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2349         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2350                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2351          VQSHLQ))
2352   ]
2353   "TARGET_HAVE_MVE"
2354   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2355   [(set_attr "type" "mve_move")
2356 ])
2357
2358 ;;
2359 ;; [vqshluq_n_s])
2360 ;;
2361 (define_insn "mve_vqshluq_n_s<mode>"
2362   [
2363    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2364         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2365                        (match_operand:SI 2 "mve_imm_7" "Ra")]
2366          VQSHLUQ_N_S))
2367   ]
2368   "TARGET_HAVE_MVE"
2369   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
2370   [(set_attr "type" "mve_move")
2371 ])
2372
2373 ;;
2374 ;; [vqsubq_n_s, vqsubq_n_u])
2375 ;;
2376 (define_insn "mve_vqsubq_n_<supf><mode>"
2377   [
2378    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2379         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2380                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2381          VQSUBQ_N))
2382   ]
2383   "TARGET_HAVE_MVE"
2384   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2385   [(set_attr "type" "mve_move")
2386 ])
2387
2388 ;;
2389 ;; [vqsubq_u, vqsubq_s])
2390 ;;
2391 (define_insn "mve_vqsubq_<supf><mode>"
2392   [
2393    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2394         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2395                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2396          VQSUBQ))
2397   ]
2398   "TARGET_HAVE_MVE"
2399   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2400   [(set_attr "type" "mve_move")
2401 ])
2402
2403 ;;
2404 ;; [vrhaddq_s, vrhaddq_u])
2405 ;;
2406 (define_insn "mve_vrhaddq_<supf><mode>"
2407   [
2408    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2409         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2410                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2411          VRHADDQ))
2412   ]
2413   "TARGET_HAVE_MVE"
2414   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2415   [(set_attr "type" "mve_move")
2416 ])
2417
2418 ;;
2419 ;; [vrmulhq_s, vrmulhq_u])
2420 ;;
2421 (define_insn "mve_vrmulhq_<supf><mode>"
2422   [
2423    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2424         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2425                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2426          VRMULHQ))
2427   ]
2428   "TARGET_HAVE_MVE"
2429   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2430   [(set_attr "type" "mve_move")
2431 ])
2432
2433 ;;
2434 ;; [vrshlq_n_u, vrshlq_n_s])
2435 ;;
2436 (define_insn "mve_vrshlq_n_<supf><mode>"
2437   [
2438    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2439         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2440                        (match_operand:SI 2 "s_register_operand" "r")]
2441          VRSHLQ_N))
2442   ]
2443   "TARGET_HAVE_MVE"
2444   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2445   [(set_attr "type" "mve_move")
2446 ])
2447
2448 ;;
2449 ;; [vrshlq_s, vrshlq_u])
2450 ;;
2451 (define_insn "mve_vrshlq_<supf><mode>"
2452   [
2453    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2454         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2455                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2456          VRSHLQ))
2457   ]
2458   "TARGET_HAVE_MVE"
2459   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2460   [(set_attr "type" "mve_move")
2461 ])
2462
2463 ;;
2464 ;; [vrshrq_n_s, vrshrq_n_u])
2465 ;;
2466 (define_insn "mve_vrshrq_n_<supf><mode>"
2467   [
2468    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2469         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2470                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
2471          VRSHRQ_N))
2472   ]
2473   "TARGET_HAVE_MVE"
2474   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2475   [(set_attr "type" "mve_move")
2476 ])
2477
2478 ;;
2479 ;; [vshlq_n_u, vshlq_n_s])
2480 ;;
2481 (define_insn "mve_vshlq_n_<supf><mode>"
2482   [
2483    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2484         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2485                        (match_operand:SI 2 "immediate_operand" "i")]
2486          VSHLQ_N))
2487   ]
2488   "TARGET_HAVE_MVE"
2489   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2490   [(set_attr "type" "mve_move")
2491 ])
2492
2493 ;;
2494 ;; [vshlq_r_s, vshlq_r_u])
2495 ;;
2496 (define_insn "mve_vshlq_r_<supf><mode>"
2497   [
2498    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2499         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2500                        (match_operand:SI 2 "s_register_operand" "r")]
2501          VSHLQ_R))
2502   ]
2503   "TARGET_HAVE_MVE"
2504   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
2505   [(set_attr "type" "mve_move")
2506 ])
2507
2508 ;;
2509 ;; [vsubq_n_s, vsubq_n_u])
2510 ;;
2511 (define_insn "mve_vsubq_n_<supf><mode>"
2512   [
2513    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2514         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2515                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2516          VSUBQ_N))
2517   ]
2518   "TARGET_HAVE_MVE"
2519   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
2520   [(set_attr "type" "mve_move")
2521 ])
2522
2523 ;;
2524 ;; [vsubq_s, vsubq_u])
2525 ;;
2526 (define_insn "mve_vsubq_<supf><mode>"
2527   [
2528    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2529         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2530                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2531          VSUBQ))
2532   ]
2533   "TARGET_HAVE_MVE"
2534   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2535   [(set_attr "type" "mve_move")
2536 ])
2537
2538 ;;
2539 ;; [vabdq_f])
2540 ;;
2541 (define_insn "mve_vabdq_f<mode>"
2542   [
2543    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2544         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2545                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2546          VABDQ_F))
2547   ]
2548   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2549   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2550   [(set_attr "type" "mve_move")
2551 ])
2552
2553 ;;
2554 ;; [vaddlvaq_s vaddlvaq_u])
2555 ;;
2556 (define_insn "mve_vaddlvaq_<supf>v4si"
2557   [
2558    (set (match_operand:DI 0 "s_register_operand" "=r")
2559         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2560                     (match_operand:V4SI 2 "s_register_operand" "w")]
2561          VADDLVAQ))
2562   ]
2563   "TARGET_HAVE_MVE"
2564   "vaddlva.<supf>32 %Q0, %R0, %q2"
2565   [(set_attr "type" "mve_move")
2566 ])
2567
2568 ;;
2569 ;; [vaddq_n_f])
2570 ;;
2571 (define_insn "mve_vaddq_n_f<mode>"
2572   [
2573    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2574         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2575                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2576          VADDQ_N_F))
2577   ]
2578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2579   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2580   [(set_attr "type" "mve_move")
2581 ])
2582
2583 ;;
2584 ;; [vandq_f])
2585 ;;
2586 (define_insn "mve_vandq_f<mode>"
2587   [
2588    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2589         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2590                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2591          VANDQ_F))
2592   ]
2593   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2594   "vand %q0, %q1, %q2"
2595   [(set_attr "type" "mve_move")
2596 ])
2597
2598 ;;
2599 ;; [vbicq_f])
2600 ;;
2601 (define_insn "mve_vbicq_f<mode>"
2602   [
2603    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2604         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2605                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2606          VBICQ_F))
2607   ]
2608   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2609   "vbic %q0, %q1, %q2"
2610   [(set_attr "type" "mve_move")
2611 ])
2612
2613 ;;
2614 ;; [vbicq_n_s, vbicq_n_u])
2615 ;;
2616 (define_insn "mve_vbicq_n_<supf><mode>"
2617   [
2618    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2619         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2620                        (match_operand:SI 2 "immediate_operand" "i")]
2621          VBICQ_N))
2622   ]
2623   "TARGET_HAVE_MVE"
2624   "vbic.i%#<V_sz_elem>  %q0, %2"
2625   [(set_attr "type" "mve_move")
2626 ])
2627
2628 ;;
2629 ;; [vcaddq_rot270_f])
2630 ;;
2631 (define_insn "mve_vcaddq_rot270_f<mode>"
2632   [
2633    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2634         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2635                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2636          VCADDQ_ROT270_F))
2637   ]
2638   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2639   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2640   [(set_attr "type" "mve_move")
2641 ])
2642
2643 ;;
2644 ;; [vcaddq_rot90_f])
2645 ;;
2646 (define_insn "mve_vcaddq_rot90_f<mode>"
2647   [
2648    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2649         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2650                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2651          VCADDQ_ROT90_F))
2652   ]
2653   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2654   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2655   [(set_attr "type" "mve_move")
2656 ])
2657
2658 ;;
2659 ;; [vcmpeqq_f])
2660 ;;
2661 (define_insn "mve_vcmpeqq_f<mode>"
2662   [
2663    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2664         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2665                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2666          VCMPEQQ_F))
2667   ]
2668   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2669   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2670   [(set_attr "type" "mve_move")
2671 ])
2672
2673 ;;
2674 ;; [vcmpeqq_n_f])
2675 ;;
2676 (define_insn "mve_vcmpeqq_n_f<mode>"
2677   [
2678    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2679         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2680                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2681          VCMPEQQ_N_F))
2682   ]
2683   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2684   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2685   [(set_attr "type" "mve_move")
2686 ])
2687
2688 ;;
2689 ;; [vcmpgeq_f])
2690 ;;
2691 (define_insn "mve_vcmpgeq_f<mode>"
2692   [
2693    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2694         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2695                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2696          VCMPGEQ_F))
2697   ]
2698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2699   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2700   [(set_attr "type" "mve_move")
2701 ])
2702
2703 ;;
2704 ;; [vcmpgeq_n_f])
2705 ;;
2706 (define_insn "mve_vcmpgeq_n_f<mode>"
2707   [
2708    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2709         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2710                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2711          VCMPGEQ_N_F))
2712   ]
2713   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2714   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2715   [(set_attr "type" "mve_move")
2716 ])
2717
2718 ;;
2719 ;; [vcmpgtq_f])
2720 ;;
2721 (define_insn "mve_vcmpgtq_f<mode>"
2722   [
2723    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2724         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2725                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2726          VCMPGTQ_F))
2727   ]
2728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2729   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2730   [(set_attr "type" "mve_move")
2731 ])
2732
2733 ;;
2734 ;; [vcmpgtq_n_f])
2735 ;;
2736 (define_insn "mve_vcmpgtq_n_f<mode>"
2737   [
2738    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2739         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2740                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2741          VCMPGTQ_N_F))
2742   ]
2743   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2744   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2745   [(set_attr "type" "mve_move")
2746 ])
2747
2748 ;;
2749 ;; [vcmpleq_f])
2750 ;;
2751 (define_insn "mve_vcmpleq_f<mode>"
2752   [
2753    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2754         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2755                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2756          VCMPLEQ_F))
2757   ]
2758   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2759   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2760   [(set_attr "type" "mve_move")
2761 ])
2762
2763 ;;
2764 ;; [vcmpleq_n_f])
2765 ;;
2766 (define_insn "mve_vcmpleq_n_f<mode>"
2767   [
2768    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2769         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2770                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2771          VCMPLEQ_N_F))
2772   ]
2773   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2774   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2775   [(set_attr "type" "mve_move")
2776 ])
2777
2778 ;;
2779 ;; [vcmpltq_f])
2780 ;;
2781 (define_insn "mve_vcmpltq_f<mode>"
2782   [
2783    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2784         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2785                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2786          VCMPLTQ_F))
2787   ]
2788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2789   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2790   [(set_attr "type" "mve_move")
2791 ])
2792
2793 ;;
2794 ;; [vcmpltq_n_f])
2795 ;;
2796 (define_insn "mve_vcmpltq_n_f<mode>"
2797   [
2798    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2799         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2800                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2801          VCMPLTQ_N_F))
2802   ]
2803   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2804   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2805   [(set_attr "type" "mve_move")
2806 ])
2807
2808 ;;
2809 ;; [vcmpneq_f])
2810 ;;
2811 (define_insn "mve_vcmpneq_f<mode>"
2812   [
2813    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2814         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2815                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2816          VCMPNEQ_F))
2817   ]
2818   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2819   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2820   [(set_attr "type" "mve_move")
2821 ])
2822
2823 ;;
2824 ;; [vcmpneq_n_f])
2825 ;;
2826 (define_insn "mve_vcmpneq_n_f<mode>"
2827   [
2828    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2829         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2830                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2831          VCMPNEQ_N_F))
2832   ]
2833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2834   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2835   [(set_attr "type" "mve_move")
2836 ])
2837
2838 ;;
2839 ;; [vcmulq_f])
2840 ;;
2841 (define_insn "mve_vcmulq_f<mode>"
2842   [
2843    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2844         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2845                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2846          VCMULQ_F))
2847   ]
2848   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2849   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2850   [(set_attr "type" "mve_move")
2851 ])
2852
2853 ;;
2854 ;; [vcmulq_rot180_f])
2855 ;;
2856 (define_insn "mve_vcmulq_rot180_f<mode>"
2857   [
2858    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2859         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2860                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2861          VCMULQ_ROT180_F))
2862   ]
2863   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2864   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2865   [(set_attr "type" "mve_move")
2866 ])
2867
2868 ;;
2869 ;; [vcmulq_rot270_f])
2870 ;;
2871 (define_insn "mve_vcmulq_rot270_f<mode>"
2872   [
2873    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2874         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2875                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2876          VCMULQ_ROT270_F))
2877   ]
2878   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2879   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2880   [(set_attr "type" "mve_move")
2881 ])
2882
2883 ;;
2884 ;; [vcmulq_rot90_f])
2885 ;;
2886 (define_insn "mve_vcmulq_rot90_f<mode>"
2887   [
2888    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2889         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2890                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2891          VCMULQ_ROT90_F))
2892   ]
2893   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2894   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2895   [(set_attr "type" "mve_move")
2896 ])
2897
2898 ;;
2899 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2900 ;;
2901 (define_insn "mve_vctp<mode1>q_mhi"
2902   [
2903    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2904         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2905                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2906          VCTPQ_M))
2907   ]
2908   "TARGET_HAVE_MVE"
2909   "vpst\;vctpt.<mode1> %1"
2910   [(set_attr "type" "mve_move")
2911    (set_attr "length""8")])
2912
2913 ;;
2914 ;; [vcvtbq_f16_f32])
2915 ;;
2916 (define_insn "mve_vcvtbq_f16_f32v8hf"
2917   [
2918    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2919         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2920                       (match_operand:V4SF 2 "s_register_operand" "w")]
2921          VCVTBQ_F16_F32))
2922   ]
2923   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2924   "vcvtb.f16.f32 %q0, %q2"
2925   [(set_attr "type" "mve_move")
2926 ])
2927
2928 ;;
2929 ;; [vcvttq_f16_f32])
2930 ;;
2931 (define_insn "mve_vcvttq_f16_f32v8hf"
2932   [
2933    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2934         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2935                       (match_operand:V4SF 2 "s_register_operand" "w")]
2936          VCVTTQ_F16_F32))
2937   ]
2938   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2939   "vcvtt.f16.f32 %q0, %q2"
2940   [(set_attr "type" "mve_move")
2941 ])
2942
2943 ;;
2944 ;; [veorq_f])
2945 ;;
2946 (define_insn "mve_veorq_f<mode>"
2947   [
2948    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2949         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2950                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2951          VEORQ_F))
2952   ]
2953   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2954   "veor %q0, %q1, %q2"
2955   [(set_attr "type" "mve_move")
2956 ])
2957
2958 ;;
2959 ;; [vmaxnmaq_f])
2960 ;;
2961 (define_insn "mve_vmaxnmaq_f<mode>"
2962   [
2963    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2964         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2965                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2966          VMAXNMAQ_F))
2967   ]
2968   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2969   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2970   [(set_attr "type" "mve_move")
2971 ])
2972
2973 ;;
2974 ;; [vmaxnmavq_f])
2975 ;;
2976 (define_insn "mve_vmaxnmavq_f<mode>"
2977   [
2978    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2979         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2980                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2981          VMAXNMAVQ_F))
2982   ]
2983   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2984   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2985   [(set_attr "type" "mve_move")
2986 ])
2987
2988 ;;
2989 ;; [vmaxnmq_f])
2990 ;;
2991 (define_insn "mve_vmaxnmq_f<mode>"
2992   [
2993    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2994         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2995                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2996          VMAXNMQ_F))
2997   ]
2998   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2999   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3000   [(set_attr "type" "mve_move")
3001 ])
3002
3003 ;;
3004 ;; [vmaxnmvq_f])
3005 ;;
3006 (define_insn "mve_vmaxnmvq_f<mode>"
3007   [
3008    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3009         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3010                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3011          VMAXNMVQ_F))
3012   ]
3013   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3014   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
3015   [(set_attr "type" "mve_move")
3016 ])
3017
3018 ;;
3019 ;; [vminnmaq_f])
3020 ;;
3021 (define_insn "mve_vminnmaq_f<mode>"
3022   [
3023    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3024         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3025                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3026          VMINNMAQ_F))
3027   ]
3028   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3029   "vminnma.f%#<V_sz_elem>       %q0, %q2"
3030   [(set_attr "type" "mve_move")
3031 ])
3032
3033 ;;
3034 ;; [vminnmavq_f])
3035 ;;
3036 (define_insn "mve_vminnmavq_f<mode>"
3037   [
3038    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3039         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3040                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3041          VMINNMAVQ_F))
3042   ]
3043   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3044   "vminnmav.f%#<V_sz_elem>      %0, %q2"
3045   [(set_attr "type" "mve_move")
3046 ])
3047
3048 ;;
3049 ;; [vminnmq_f])
3050 ;;
3051 (define_insn "mve_vminnmq_f<mode>"
3052   [
3053    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3054         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3055                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3056          VMINNMQ_F))
3057   ]
3058   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3059   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3060   [(set_attr "type" "mve_move")
3061 ])
3062
3063 ;;
3064 ;; [vminnmvq_f])
3065 ;;
3066 (define_insn "mve_vminnmvq_f<mode>"
3067   [
3068    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3069         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3070                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3071          VMINNMVQ_F))
3072   ]
3073   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3074   "vminnmv.f%#<V_sz_elem>       %0, %q2"
3075   [(set_attr "type" "mve_move")
3076 ])
3077
3078 ;;
3079 ;; [vmlaldavq_u, vmlaldavq_s])
3080 ;;
3081 (define_insn "mve_vmlaldavq_<supf><mode>"
3082   [
3083    (set (match_operand:DI 0 "s_register_operand" "=r")
3084         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3085                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3086          VMLALDAVQ))
3087   ]
3088   "TARGET_HAVE_MVE"
3089   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3090   [(set_attr "type" "mve_move")
3091 ])
3092
3093 ;;
3094 ;; [vmlaldavxq_s])
3095 ;;
3096 (define_insn "mve_vmlaldavxq_s<mode>"
3097   [
3098    (set (match_operand:DI 0 "s_register_operand" "=r")
3099         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3100                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3101          VMLALDAVXQ_S))
3102   ]
3103   "TARGET_HAVE_MVE"
3104   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3105   [(set_attr "type" "mve_move")
3106 ])
3107
3108 ;;
3109 ;; [vmlsldavq_s])
3110 ;;
3111 (define_insn "mve_vmlsldavq_s<mode>"
3112   [
3113    (set (match_operand:DI 0 "s_register_operand" "=r")
3114         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3115                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3116          VMLSLDAVQ_S))
3117   ]
3118   "TARGET_HAVE_MVE"
3119   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3120   [(set_attr "type" "mve_move")
3121 ])
3122
3123 ;;
3124 ;; [vmlsldavxq_s])
3125 ;;
3126 (define_insn "mve_vmlsldavxq_s<mode>"
3127   [
3128    (set (match_operand:DI 0 "s_register_operand" "=r")
3129         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3130                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3131          VMLSLDAVXQ_S))
3132   ]
3133   "TARGET_HAVE_MVE"
3134   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3135   [(set_attr "type" "mve_move")
3136 ])
3137
3138 ;;
3139 ;; [vmovnbq_u, vmovnbq_s])
3140 ;;
3141 (define_insn "mve_vmovnbq_<supf><mode>"
3142   [
3143    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3144         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3145                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3146          VMOVNBQ))
3147   ]
3148   "TARGET_HAVE_MVE"
3149   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
3150   [(set_attr "type" "mve_move")
3151 ])
3152
3153 ;;
3154 ;; [vmovntq_s, vmovntq_u])
3155 ;;
3156 (define_insn "mve_vmovntq_<supf><mode>"
3157   [
3158    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3159         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3160                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3161          VMOVNTQ))
3162   ]
3163   "TARGET_HAVE_MVE"
3164   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
3165   [(set_attr "type" "mve_move")
3166 ])
3167
3168 ;;
3169 ;; [vmulq_f])
3170 ;;
3171 (define_insn "mve_vmulq_f<mode>"
3172   [
3173    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3174         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3175                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3176          VMULQ_F))
3177   ]
3178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3179   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
3180   [(set_attr "type" "mve_move")
3181 ])
3182
3183 ;;
3184 ;; [vmulq_n_f])
3185 ;;
3186 (define_insn "mve_vmulq_n_f<mode>"
3187   [
3188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3190                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
3191          VMULQ_N_F))
3192   ]
3193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3194   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
3195   [(set_attr "type" "mve_move")
3196 ])
3197
3198 ;;
3199 ;; [vornq_f])
3200 ;;
3201 (define_insn "mve_vornq_f<mode>"
3202   [
3203    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3204         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3205                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3206          VORNQ_F))
3207   ]
3208   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3209   "vorn %q0, %q1, %q2"
3210   [(set_attr "type" "mve_move")
3211 ])
3212
3213 ;;
3214 ;; [vorrq_f])
3215 ;;
3216 (define_insn "mve_vorrq_f<mode>"
3217   [
3218    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3219         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3220                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3221          VORRQ_F))
3222   ]
3223   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3224   "vorr %q0, %q1, %q2"
3225   [(set_attr "type" "mve_move")
3226 ])
3227
3228 ;;
3229 ;; [vorrq_n_u, vorrq_n_s])
3230 ;;
3231 (define_insn "mve_vorrq_n_<supf><mode>"
3232   [
3233    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3234         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3235                        (match_operand:SI 2 "immediate_operand" "i")]
3236          VORRQ_N))
3237   ]
3238   "TARGET_HAVE_MVE"
3239   "vorr.i%#<V_sz_elem>  %q0, %2"
3240   [(set_attr "type" "mve_move")
3241 ])
3242
3243 ;;
3244 ;; [vqdmullbq_n_s])
3245 ;;
3246 (define_insn "mve_vqdmullbq_n_s<mode>"
3247   [
3248    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3249         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3250                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3251          VQDMULLBQ_N_S))
3252   ]
3253   "TARGET_HAVE_MVE"
3254   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
3255   [(set_attr "type" "mve_move")
3256 ])
3257
3258 ;;
3259 ;; [vqdmullbq_s])
3260 ;;
3261 (define_insn "mve_vqdmullbq_s<mode>"
3262   [
3263    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3264         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3265                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3266          VQDMULLBQ_S))
3267   ]
3268   "TARGET_HAVE_MVE"
3269   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
3270   [(set_attr "type" "mve_move")
3271 ])
3272
3273 ;;
3274 ;; [vqdmulltq_n_s])
3275 ;;
3276 (define_insn "mve_vqdmulltq_n_s<mode>"
3277   [
3278    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3279         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3280                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3281          VQDMULLTQ_N_S))
3282   ]
3283   "TARGET_HAVE_MVE"
3284   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
3285   [(set_attr "type" "mve_move")
3286 ])
3287
3288 ;;
3289 ;; [vqdmulltq_s])
3290 ;;
3291 (define_insn "mve_vqdmulltq_s<mode>"
3292   [
3293    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3294         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3295                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3296          VQDMULLTQ_S))
3297   ]
3298   "TARGET_HAVE_MVE"
3299   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
3300   [(set_attr "type" "mve_move")
3301 ])
3302
3303 ;;
3304 ;; [vqmovnbq_u, vqmovnbq_s])
3305 ;;
3306 (define_insn "mve_vqmovnbq_<supf><mode>"
3307   [
3308    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3309         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3310                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3311          VQMOVNBQ))
3312   ]
3313   "TARGET_HAVE_MVE"
3314   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
3315   [(set_attr "type" "mve_move")
3316 ])
3317
3318 ;;
3319 ;; [vqmovntq_u, vqmovntq_s])
3320 ;;
3321 (define_insn "mve_vqmovntq_<supf><mode>"
3322   [
3323    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3324         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3325                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3326          VQMOVNTQ))
3327   ]
3328   "TARGET_HAVE_MVE"
3329   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
3330   [(set_attr "type" "mve_move")
3331 ])
3332
3333 ;;
3334 ;; [vqmovunbq_s])
3335 ;;
3336 (define_insn "mve_vqmovunbq_s<mode>"
3337   [
3338    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3339         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3340                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3341          VQMOVUNBQ_S))
3342   ]
3343   "TARGET_HAVE_MVE"
3344   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
3345   [(set_attr "type" "mve_move")
3346 ])
3347
3348 ;;
3349 ;; [vqmovuntq_s])
3350 ;;
3351 (define_insn "mve_vqmovuntq_s<mode>"
3352   [
3353    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3354         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3355                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3356          VQMOVUNTQ_S))
3357   ]
3358   "TARGET_HAVE_MVE"
3359   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
3360   [(set_attr "type" "mve_move")
3361 ])
3362
3363 ;;
3364 ;; [vrmlaldavhxq_s])
3365 ;;
3366 (define_insn "mve_vrmlaldavhxq_sv4si"
3367   [
3368    (set (match_operand:DI 0 "s_register_operand" "=r")
3369         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3370                     (match_operand:V4SI 2 "s_register_operand" "w")]
3371          VRMLALDAVHXQ_S))
3372   ]
3373   "TARGET_HAVE_MVE"
3374   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
3375   [(set_attr "type" "mve_move")
3376 ])
3377
3378 ;;
3379 ;; [vrmlsldavhq_s])
3380 ;;
3381 (define_insn "mve_vrmlsldavhq_sv4si"
3382   [
3383    (set (match_operand:DI 0 "s_register_operand" "=r")
3384         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3385                     (match_operand:V4SI 2 "s_register_operand" "w")]
3386          VRMLSLDAVHQ_S))
3387   ]
3388   "TARGET_HAVE_MVE"
3389   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
3390   [(set_attr "type" "mve_move")
3391 ])
3392
3393 ;;
3394 ;; [vrmlsldavhxq_s])
3395 ;;
3396 (define_insn "mve_vrmlsldavhxq_sv4si"
3397   [
3398    (set (match_operand:DI 0 "s_register_operand" "=r")
3399         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3400                     (match_operand:V4SI 2 "s_register_operand" "w")]
3401          VRMLSLDAVHXQ_S))
3402   ]
3403   "TARGET_HAVE_MVE"
3404   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
3405   [(set_attr "type" "mve_move")
3406 ])
3407
3408 ;;
3409 ;; [vshllbq_n_s, vshllbq_n_u])
3410 ;;
3411 (define_insn "mve_vshllbq_n_<supf><mode>"
3412   [
3413    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3414         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3415                                   (match_operand:SI 2 "immediate_operand" "i")]
3416          VSHLLBQ_N))
3417   ]
3418   "TARGET_HAVE_MVE"
3419   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3420   [(set_attr "type" "mve_move")
3421 ])
3422
3423 ;;
3424 ;; [vshlltq_n_u, vshlltq_n_s])
3425 ;;
3426 (define_insn "mve_vshlltq_n_<supf><mode>"
3427   [
3428    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3429         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3430                                   (match_operand:SI 2 "immediate_operand" "i")]
3431          VSHLLTQ_N))
3432   ]
3433   "TARGET_HAVE_MVE"
3434   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3435   [(set_attr "type" "mve_move")
3436 ])
3437
3438 ;;
3439 ;; [vsubq_f])
3440 ;;
3441 (define_insn "mve_vsubq_f<mode>"
3442   [
3443    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3444         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3445                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3446          VSUBQ_F))
3447   ]
3448   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3449   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
3450   [(set_attr "type" "mve_move")
3451 ])
3452
3453 ;;
3454 ;; [vmulltq_poly_p])
3455 ;;
3456 (define_insn "mve_vmulltq_poly_p<mode>"
3457   [
3458    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3459         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3460                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3461          VMULLTQ_POLY_P))
3462   ]
3463   "TARGET_HAVE_MVE"
3464   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
3465   [(set_attr "type" "mve_move")
3466 ])
3467
3468 ;;
3469 ;; [vmullbq_poly_p])
3470 ;;
3471 (define_insn "mve_vmullbq_poly_p<mode>"
3472   [
3473    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3474         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3475                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3476          VMULLBQ_POLY_P))
3477   ]
3478   "TARGET_HAVE_MVE"
3479   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
3480   [(set_attr "type" "mve_move")
3481 ])
3482
3483 ;;
3484 ;; [vrmlaldavhq_u vrmlaldavhq_s])
3485 ;;
3486 (define_insn "mve_vrmlaldavhq_<supf>v4si"
3487   [
3488    (set (match_operand:DI 0 "s_register_operand" "=r")
3489         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3490                     (match_operand:V4SI 2 "s_register_operand" "w")]
3491          VRMLALDAVHQ))
3492   ]
3493   "TARGET_HAVE_MVE"
3494   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
3495   [(set_attr "type" "mve_move")
3496 ])
3497
3498 ;;
3499 ;; [vbicq_m_n_s, vbicq_m_n_u])
3500 ;;
3501 (define_insn "mve_vbicq_m_n_<supf><mode>"
3502   [
3503    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3504         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3505                        (match_operand:SI 2 "immediate_operand" "i")
3506                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3507          VBICQ_M_N))
3508   ]
3509   "TARGET_HAVE_MVE"
3510   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
3511   [(set_attr "type" "mve_move")
3512    (set_attr "length""8")])
3513 ;;
3514 ;; [vcmpeqq_m_f])
3515 ;;
3516 (define_insn "mve_vcmpeqq_m_f<mode>"
3517   [
3518    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3519         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3520                     (match_operand:MVE_0 2 "s_register_operand" "w")
3521                     (match_operand:HI 3 "vpr_register_operand" "Up")]
3522          VCMPEQQ_M_F))
3523   ]
3524   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3525   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
3526   [(set_attr "type" "mve_move")
3527    (set_attr "length""8")])
3528 ;;
3529 ;; [vcvtaq_m_u, vcvtaq_m_s])
3530 ;;
3531 (define_insn "mve_vcvtaq_m_<supf><mode>"
3532   [
3533    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3534         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3535                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3536                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3537          VCVTAQ_M))
3538   ]
3539   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3540   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3541   [(set_attr "type" "mve_move")
3542    (set_attr "length""8")])
3543 ;;
3544 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3545 ;;
3546 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3547   [
3548    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3549         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3550                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3551                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3552          VCVTQ_M_TO_F))
3553   ]
3554   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3555   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3556   [(set_attr "type" "mve_move")
3557    (set_attr "length""8")])
3558 ;;
3559 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3560 ;;
3561 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3562   [
3563    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3564         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3565                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3566                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3567          VQRSHRNBQ_N))
3568   ]
3569   "TARGET_HAVE_MVE"
3570   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3571   [(set_attr "type" "mve_move")
3572 ])
3573 ;;
3574 ;; [vqrshrunbq_n_s])
3575 ;;
3576 (define_insn "mve_vqrshrunbq_n_s<mode>"
3577   [
3578    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3579         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3580                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3581                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3582          VQRSHRUNBQ_N_S))
3583   ]
3584   "TARGET_HAVE_MVE"
3585   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3586   [(set_attr "type" "mve_move")
3587 ])
3588 ;;
3589 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3590 ;;
3591 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3592   [
3593    (set (match_operand:DI 0 "s_register_operand" "=r")
3594         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3595                     (match_operand:V4SI 2 "s_register_operand" "w")
3596                     (match_operand:V4SI 3 "s_register_operand" "w")]
3597          VRMLALDAVHAQ))
3598   ]
3599   "TARGET_HAVE_MVE"
3600   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3601   [(set_attr "type" "mve_move")
3602 ])
3603
3604 ;;
3605 ;; [vabavq_s, vabavq_u])
3606 ;;
3607 (define_insn "mve_vabavq_<supf><mode>"
3608   [
3609    (set (match_operand:SI 0 "s_register_operand" "=r")
3610         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3611                     (match_operand:MVE_2 2 "s_register_operand" "w")
3612                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3613          VABAVQ))
3614   ]
3615   "TARGET_HAVE_MVE"
3616   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3617   [(set_attr "type" "mve_move")
3618 ])
3619
3620 ;;
3621 ;; [vshlcq_u vshlcq_s]
3622 ;;
3623 (define_expand "mve_vshlcq_vec_<supf><mode>"
3624  [(match_operand:MVE_2 0 "s_register_operand")
3625   (match_operand:MVE_2 1 "s_register_operand")
3626   (match_operand:SI 2 "s_register_operand")
3627   (match_operand:SI 3 "mve_imm_32")
3628   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3629  "TARGET_HAVE_MVE"
3630 {
3631   rtx ignore_wb = gen_reg_rtx (SImode);
3632   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3633                                       operands[2], operands[3]));
3634   DONE;
3635 })
3636
3637 (define_expand "mve_vshlcq_carry_<supf><mode>"
3638  [(match_operand:SI 0 "s_register_operand")
3639   (match_operand:MVE_2 1 "s_register_operand")
3640   (match_operand:SI 2 "s_register_operand")
3641   (match_operand:SI 3 "mve_imm_32")
3642   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3643  "TARGET_HAVE_MVE"
3644 {
3645   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3646   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3647                                       operands[2], operands[3]));
3648   DONE;
3649 })
3650
3651 (define_insn "mve_vshlcq_<supf><mode>"
3652  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3653        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3654                       (match_operand:SI 3 "s_register_operand" "1")
3655                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3656         VSHLCQ))
3657   (set (match_operand:SI  1 "s_register_operand" "=r")
3658        (unspec:SI [(match_dup 2)
3659                    (match_dup 3)
3660                    (match_dup 4)]
3661         VSHLCQ))]
3662  "TARGET_HAVE_MVE"
3663  "vshlc %q0, %1, %4")
3664
3665 ;;
3666 ;; [vabsq_m_s])
3667 ;;
3668 (define_insn "mve_vabsq_m_s<mode>"
3669   [
3670    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3671         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3672                        (match_operand:MVE_2 2 "s_register_operand" "w")
3673                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3674          VABSQ_M_S))
3675   ]
3676   "TARGET_HAVE_MVE"
3677   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3678   [(set_attr "type" "mve_move")
3679    (set_attr "length""8")])
3680
3681 ;;
3682 ;; [vaddvaq_p_u, vaddvaq_p_s])
3683 ;;
3684 (define_insn "mve_vaddvaq_p_<supf><mode>"
3685   [
3686    (set (match_operand:SI 0 "s_register_operand" "=e")
3687         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3688                        (match_operand:MVE_2 2 "s_register_operand" "w")
3689                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3690          VADDVAQ_P))
3691   ]
3692   "TARGET_HAVE_MVE"
3693   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3694   [(set_attr "type" "mve_move")
3695    (set_attr "length""8")])
3696
3697 ;;
3698 ;; [vclsq_m_s])
3699 ;;
3700 (define_insn "mve_vclsq_m_s<mode>"
3701   [
3702    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3703         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3704                        (match_operand:MVE_2 2 "s_register_operand" "w")
3705                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3706          VCLSQ_M_S))
3707   ]
3708   "TARGET_HAVE_MVE"
3709   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3710   [(set_attr "type" "mve_move")
3711    (set_attr "length""8")])
3712
3713 ;;
3714 ;; [vclzq_m_s, vclzq_m_u])
3715 ;;
3716 (define_insn "mve_vclzq_m_<supf><mode>"
3717   [
3718    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3719         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3720                        (match_operand:MVE_2 2 "s_register_operand" "w")
3721                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3722          VCLZQ_M))
3723   ]
3724   "TARGET_HAVE_MVE"
3725   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3726   [(set_attr "type" "mve_move")
3727    (set_attr "length""8")])
3728
3729 ;;
3730 ;; [vcmpcsq_m_n_u])
3731 ;;
3732 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3733   [
3734    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3735         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3736                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3737                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3738          VCMPCSQ_M_N_U))
3739   ]
3740   "TARGET_HAVE_MVE"
3741   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3742   [(set_attr "type" "mve_move")
3743    (set_attr "length""8")])
3744
3745 ;;
3746 ;; [vcmpcsq_m_u])
3747 ;;
3748 (define_insn "mve_vcmpcsq_m_u<mode>"
3749   [
3750    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3751         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3752                        (match_operand:MVE_2 2 "s_register_operand" "w")
3753                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3754          VCMPCSQ_M_U))
3755   ]
3756   "TARGET_HAVE_MVE"
3757   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3758   [(set_attr "type" "mve_move")
3759    (set_attr "length""8")])
3760
3761 ;;
3762 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3763 ;;
3764 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3765   [
3766    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3767         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3768                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3769                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3770          VCMPEQQ_M_N))
3771   ]
3772   "TARGET_HAVE_MVE"
3773   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3774   [(set_attr "type" "mve_move")
3775    (set_attr "length""8")])
3776
3777 ;;
3778 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3779 ;;
3780 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3781   [
3782    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3783         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3784                        (match_operand:MVE_2 2 "s_register_operand" "w")
3785                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3786          VCMPEQQ_M))
3787   ]
3788   "TARGET_HAVE_MVE"
3789   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3790   [(set_attr "type" "mve_move")
3791    (set_attr "length""8")])
3792
3793 ;;
3794 ;; [vcmpgeq_m_n_s])
3795 ;;
3796 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3797   [
3798    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3799         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3800                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3801                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3802          VCMPGEQ_M_N_S))
3803   ]
3804   "TARGET_HAVE_MVE"
3805   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3806   [(set_attr "type" "mve_move")
3807    (set_attr "length""8")])
3808
3809 ;;
3810 ;; [vcmpgeq_m_s])
3811 ;;
3812 (define_insn "mve_vcmpgeq_m_s<mode>"
3813   [
3814    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3815         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3816                        (match_operand:MVE_2 2 "s_register_operand" "w")
3817                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3818          VCMPGEQ_M_S))
3819   ]
3820   "TARGET_HAVE_MVE"
3821   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3822   [(set_attr "type" "mve_move")
3823    (set_attr "length""8")])
3824
3825 ;;
3826 ;; [vcmpgtq_m_n_s])
3827 ;;
3828 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3829   [
3830    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3831         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3832                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3833                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3834          VCMPGTQ_M_N_S))
3835   ]
3836   "TARGET_HAVE_MVE"
3837   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3838   [(set_attr "type" "mve_move")
3839    (set_attr "length""8")])
3840
3841 ;;
3842 ;; [vcmpgtq_m_s])
3843 ;;
3844 (define_insn "mve_vcmpgtq_m_s<mode>"
3845   [
3846    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3847         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3848                        (match_operand:MVE_2 2 "s_register_operand" "w")
3849                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3850          VCMPGTQ_M_S))
3851   ]
3852   "TARGET_HAVE_MVE"
3853   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3854   [(set_attr "type" "mve_move")
3855    (set_attr "length""8")])
3856
3857 ;;
3858 ;; [vcmphiq_m_n_u])
3859 ;;
3860 (define_insn "mve_vcmphiq_m_n_u<mode>"
3861   [
3862    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3863         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3864                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3865                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3866          VCMPHIQ_M_N_U))
3867   ]
3868   "TARGET_HAVE_MVE"
3869   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3870   [(set_attr "type" "mve_move")
3871    (set_attr "length""8")])
3872
3873 ;;
3874 ;; [vcmphiq_m_u])
3875 ;;
3876 (define_insn "mve_vcmphiq_m_u<mode>"
3877   [
3878    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3879         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3880                        (match_operand:MVE_2 2 "s_register_operand" "w")
3881                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3882          VCMPHIQ_M_U))
3883   ]
3884   "TARGET_HAVE_MVE"
3885   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3886   [(set_attr "type" "mve_move")
3887    (set_attr "length""8")])
3888
3889 ;;
3890 ;; [vcmpleq_m_n_s])
3891 ;;
3892 (define_insn "mve_vcmpleq_m_n_s<mode>"
3893   [
3894    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3895         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3896                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3897                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3898          VCMPLEQ_M_N_S))
3899   ]
3900   "TARGET_HAVE_MVE"
3901   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3902   [(set_attr "type" "mve_move")
3903    (set_attr "length""8")])
3904
3905 ;;
3906 ;; [vcmpleq_m_s])
3907 ;;
3908 (define_insn "mve_vcmpleq_m_s<mode>"
3909   [
3910    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3911         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3912                        (match_operand:MVE_2 2 "s_register_operand" "w")
3913                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3914          VCMPLEQ_M_S))
3915   ]
3916   "TARGET_HAVE_MVE"
3917   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3918   [(set_attr "type" "mve_move")
3919    (set_attr "length""8")])
3920
3921 ;;
3922 ;; [vcmpltq_m_n_s])
3923 ;;
3924 (define_insn "mve_vcmpltq_m_n_s<mode>"
3925   [
3926    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3927         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3928                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3929                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3930          VCMPLTQ_M_N_S))
3931   ]
3932   "TARGET_HAVE_MVE"
3933   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3934   [(set_attr "type" "mve_move")
3935    (set_attr "length""8")])
3936
3937 ;;
3938 ;; [vcmpltq_m_s])
3939 ;;
3940 (define_insn "mve_vcmpltq_m_s<mode>"
3941   [
3942    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3943         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3944                        (match_operand:MVE_2 2 "s_register_operand" "w")
3945                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3946          VCMPLTQ_M_S))
3947   ]
3948   "TARGET_HAVE_MVE"
3949   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3950   [(set_attr "type" "mve_move")
3951    (set_attr "length""8")])
3952
3953 ;;
3954 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3955 ;;
3956 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3957   [
3958    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3959         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3960                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3961                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3962          VCMPNEQ_M_N))
3963   ]
3964   "TARGET_HAVE_MVE"
3965   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3966   [(set_attr "type" "mve_move")
3967    (set_attr "length""8")])
3968
3969 ;;
3970 ;; [vcmpneq_m_s, vcmpneq_m_u])
3971 ;;
3972 (define_insn "mve_vcmpneq_m_<supf><mode>"
3973   [
3974    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3975         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3976                        (match_operand:MVE_2 2 "s_register_operand" "w")
3977                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3978          VCMPNEQ_M))
3979   ]
3980   "TARGET_HAVE_MVE"
3981   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3982   [(set_attr "type" "mve_move")
3983    (set_attr "length""8")])
3984
3985 ;;
3986 ;; [vdupq_m_n_s, vdupq_m_n_u])
3987 ;;
3988 (define_insn "mve_vdupq_m_n_<supf><mode>"
3989   [
3990    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3991         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3992                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3993                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3994          VDUPQ_M_N))
3995   ]
3996   "TARGET_HAVE_MVE"
3997   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3998   [(set_attr "type" "mve_move")
3999    (set_attr "length""8")])
4000
4001 ;;
4002 ;; [vmaxaq_m_s])
4003 ;;
4004 (define_insn "mve_vmaxaq_m_s<mode>"
4005   [
4006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4008                        (match_operand:MVE_2 2 "s_register_operand" "w")
4009                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4010          VMAXAQ_M_S))
4011   ]
4012   "TARGET_HAVE_MVE"
4013   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
4014   [(set_attr "type" "mve_move")
4015    (set_attr "length""8")])
4016
4017 ;;
4018 ;; [vmaxavq_p_s])
4019 ;;
4020 (define_insn "mve_vmaxavq_p_s<mode>"
4021   [
4022    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4023         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4024                        (match_operand:MVE_2 2 "s_register_operand" "w")
4025                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4026          VMAXAVQ_P_S))
4027   ]
4028   "TARGET_HAVE_MVE"
4029   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
4030   [(set_attr "type" "mve_move")
4031    (set_attr "length""8")])
4032
4033 ;;
4034 ;; [vmaxvq_p_u, vmaxvq_p_s])
4035 ;;
4036 (define_insn "mve_vmaxvq_p_<supf><mode>"
4037   [
4038    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4039         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4040                        (match_operand:MVE_2 2 "s_register_operand" "w")
4041                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4042          VMAXVQ_P))
4043   ]
4044   "TARGET_HAVE_MVE"
4045   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
4046   [(set_attr "type" "mve_move")
4047    (set_attr "length""8")])
4048
4049 ;;
4050 ;; [vminaq_m_s])
4051 ;;
4052 (define_insn "mve_vminaq_m_s<mode>"
4053   [
4054    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4055         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4056                        (match_operand:MVE_2 2 "s_register_operand" "w")
4057                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4058          VMINAQ_M_S))
4059   ]
4060   "TARGET_HAVE_MVE"
4061   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
4062   [(set_attr "type" "mve_move")
4063    (set_attr "length""8")])
4064
4065 ;;
4066 ;; [vminavq_p_s])
4067 ;;
4068 (define_insn "mve_vminavq_p_s<mode>"
4069   [
4070    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4071         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4072                        (match_operand:MVE_2 2 "s_register_operand" "w")
4073                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4074          VMINAVQ_P_S))
4075   ]
4076   "TARGET_HAVE_MVE"
4077   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
4078   [(set_attr "type" "mve_move")
4079    (set_attr "length""8")])
4080
4081 ;;
4082 ;; [vminvq_p_s, vminvq_p_u])
4083 ;;
4084 (define_insn "mve_vminvq_p_<supf><mode>"
4085   [
4086    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4087         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4088                        (match_operand:MVE_2 2 "s_register_operand" "w")
4089                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4090          VMINVQ_P))
4091   ]
4092   "TARGET_HAVE_MVE"
4093   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
4094   [(set_attr "type" "mve_move")
4095    (set_attr "length""8")])
4096
4097 ;;
4098 ;; [vmladavaq_u, vmladavaq_s])
4099 ;;
4100 (define_insn "mve_vmladavaq_<supf><mode>"
4101   [
4102    (set (match_operand:SI 0 "s_register_operand" "=e")
4103         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4104                        (match_operand:MVE_2 2 "s_register_operand" "w")
4105                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4106          VMLADAVAQ))
4107   ]
4108   "TARGET_HAVE_MVE"
4109   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
4110   [(set_attr "type" "mve_move")
4111 ])
4112
4113 ;;
4114 ;; [vmladavq_p_u, vmladavq_p_s])
4115 ;;
4116 (define_insn "mve_vmladavq_p_<supf><mode>"
4117   [
4118    (set (match_operand:SI 0 "s_register_operand" "=e")
4119         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4120                        (match_operand:MVE_2 2 "s_register_operand" "w")
4121                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4122          VMLADAVQ_P))
4123   ]
4124   "TARGET_HAVE_MVE"
4125   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
4126   [(set_attr "type" "mve_move")
4127    (set_attr "length""8")])
4128
4129 ;;
4130 ;; [vmladavxq_p_s])
4131 ;;
4132 (define_insn "mve_vmladavxq_p_s<mode>"
4133   [
4134    (set (match_operand:SI 0 "s_register_operand" "=e")
4135         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4136                        (match_operand:MVE_2 2 "s_register_operand" "w")
4137                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4138          VMLADAVXQ_P_S))
4139   ]
4140   "TARGET_HAVE_MVE"
4141   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
4142   [(set_attr "type" "mve_move")
4143    (set_attr "length""8")])
4144
4145 ;;
4146 ;; [vmlaq_n_u, vmlaq_n_s])
4147 ;;
4148 (define_insn "mve_vmlaq_n_<supf><mode>"
4149   [
4150    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4151         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4152                        (match_operand:MVE_2 2 "s_register_operand" "w")
4153                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4154          VMLAQ_N))
4155   ]
4156   "TARGET_HAVE_MVE"
4157   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4158   [(set_attr "type" "mve_move")
4159 ])
4160
4161 ;;
4162 ;; [vmlasq_n_u, vmlasq_n_s])
4163 ;;
4164 (define_insn "mve_vmlasq_n_<supf><mode>"
4165   [
4166    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4167         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4168                        (match_operand:MVE_2 2 "s_register_operand" "w")
4169                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4170          VMLASQ_N))
4171   ]
4172   "TARGET_HAVE_MVE"
4173   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
4174   [(set_attr "type" "mve_move")
4175 ])
4176
4177 ;;
4178 ;; [vmlsdavq_p_s])
4179 ;;
4180 (define_insn "mve_vmlsdavq_p_s<mode>"
4181   [
4182    (set (match_operand:SI 0 "s_register_operand" "=e")
4183         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4184                        (match_operand:MVE_2 2 "s_register_operand" "w")
4185                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4186          VMLSDAVQ_P_S))
4187   ]
4188   "TARGET_HAVE_MVE"
4189   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
4190   [(set_attr "type" "mve_move")
4191    (set_attr "length""8")])
4192
4193 ;;
4194 ;; [vmlsdavxq_p_s])
4195 ;;
4196 (define_insn "mve_vmlsdavxq_p_s<mode>"
4197   [
4198    (set (match_operand:SI 0 "s_register_operand" "=e")
4199         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4200                        (match_operand:MVE_2 2 "s_register_operand" "w")
4201                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4202          VMLSDAVXQ_P_S))
4203   ]
4204   "TARGET_HAVE_MVE"
4205   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
4206   [(set_attr "type" "mve_move")
4207    (set_attr "length""8")])
4208
4209 ;;
4210 ;; [vmvnq_m_s, vmvnq_m_u])
4211 ;;
4212 (define_insn "mve_vmvnq_m_<supf><mode>"
4213   [
4214    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4215         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4216                        (match_operand:MVE_2 2 "s_register_operand" "w")
4217                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4218          VMVNQ_M))
4219   ]
4220   "TARGET_HAVE_MVE"
4221   "vpst\;vmvnt %q0, %q2"
4222   [(set_attr "type" "mve_move")
4223    (set_attr "length""8")])
4224
4225 ;;
4226 ;; [vnegq_m_s])
4227 ;;
4228 (define_insn "mve_vnegq_m_s<mode>"
4229   [
4230    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4231         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4232                        (match_operand:MVE_2 2 "s_register_operand" "w")
4233                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4234          VNEGQ_M_S))
4235   ]
4236   "TARGET_HAVE_MVE"
4237   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
4238   [(set_attr "type" "mve_move")
4239    (set_attr "length""8")])
4240
4241 ;;
4242 ;; [vpselq_u, vpselq_s])
4243 ;;
4244 (define_insn "mve_vpselq_<supf><mode>"
4245   [
4246    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
4247         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
4248                        (match_operand:MVE_1 2 "s_register_operand" "w")
4249                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4250          VPSELQ))
4251   ]
4252   "TARGET_HAVE_MVE"
4253   "vpsel %q0, %q1, %q2"
4254   [(set_attr "type" "mve_move")
4255 ])
4256
4257 ;;
4258 ;; [vqabsq_m_s])
4259 ;;
4260 (define_insn "mve_vqabsq_m_s<mode>"
4261   [
4262    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4263         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4264                        (match_operand:MVE_2 2 "s_register_operand" "w")
4265                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4266          VQABSQ_M_S))
4267   ]
4268   "TARGET_HAVE_MVE"
4269   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
4270   [(set_attr "type" "mve_move")
4271    (set_attr "length""8")])
4272
4273 ;;
4274 ;; [vqdmlahq_n_s, vqdmlahq_n_u])
4275 ;;
4276 (define_insn "mve_vqdmlahq_n_<supf><mode>"
4277   [
4278    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4279         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4280                        (match_operand:MVE_2 2 "s_register_operand" "w")
4281                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4282          VQDMLAHQ_N))
4283   ]
4284   "TARGET_HAVE_MVE"
4285   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4286   [(set_attr "type" "mve_move")
4287 ])
4288
4289 ;;
4290 ;; [vqnegq_m_s])
4291 ;;
4292 (define_insn "mve_vqnegq_m_s<mode>"
4293   [
4294    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4295         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4296                        (match_operand:MVE_2 2 "s_register_operand" "w")
4297                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4298          VQNEGQ_M_S))
4299   ]
4300   "TARGET_HAVE_MVE"
4301   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
4302   [(set_attr "type" "mve_move")
4303    (set_attr "length""8")])
4304
4305 ;;
4306 ;; [vqrdmladhq_s])
4307 ;;
4308 (define_insn "mve_vqrdmladhq_s<mode>"
4309   [
4310    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4311         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4312                        (match_operand:MVE_2 2 "s_register_operand" "w")
4313                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4314          VQRDMLADHQ_S))
4315   ]
4316   "TARGET_HAVE_MVE"
4317   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4318   [(set_attr "type" "mve_move")
4319 ])
4320
4321 ;;
4322 ;; [vqrdmladhxq_s])
4323 ;;
4324 (define_insn "mve_vqrdmladhxq_s<mode>"
4325   [
4326    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4327         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4328                        (match_operand:MVE_2 2 "s_register_operand" "w")
4329                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4330          VQRDMLADHXQ_S))
4331   ]
4332   "TARGET_HAVE_MVE"
4333   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4334   [(set_attr "type" "mve_move")
4335 ])
4336
4337 ;;
4338 ;; [vqrdmlahq_n_s, vqrdmlahq_n_u])
4339 ;;
4340 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
4341   [
4342    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4343         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4344                        (match_operand:MVE_2 2 "s_register_operand" "w")
4345                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4346          VQRDMLAHQ_N))
4347   ]
4348   "TARGET_HAVE_MVE"
4349   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4350   [(set_attr "type" "mve_move")
4351 ])
4352
4353 ;;
4354 ;; [vqrdmlashq_n_s, vqrdmlashq_n_u])
4355 ;;
4356 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
4357   [
4358    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4359         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4360                        (match_operand:MVE_2 2 "s_register_operand" "w")
4361                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4362          VQRDMLASHQ_N))
4363   ]
4364   "TARGET_HAVE_MVE"
4365   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
4366   [(set_attr "type" "mve_move")
4367 ])
4368
4369 ;;
4370 ;; [vqrdmlsdhq_s])
4371 ;;
4372 (define_insn "mve_vqrdmlsdhq_s<mode>"
4373   [
4374    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4375         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4376                        (match_operand:MVE_2 2 "s_register_operand" "w")
4377                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4378          VQRDMLSDHQ_S))
4379   ]
4380   "TARGET_HAVE_MVE"
4381   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4382   [(set_attr "type" "mve_move")
4383 ])
4384
4385 ;;
4386 ;; [vqrdmlsdhxq_s])
4387 ;;
4388 (define_insn "mve_vqrdmlsdhxq_s<mode>"
4389   [
4390    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4391         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4392                        (match_operand:MVE_2 2 "s_register_operand" "w")
4393                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4394          VQRDMLSDHXQ_S))
4395   ]
4396   "TARGET_HAVE_MVE"
4397   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4398   [(set_attr "type" "mve_move")
4399 ])
4400
4401 ;;
4402 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
4403 ;;
4404 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
4405   [
4406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4407         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4408                        (match_operand:SI 2 "s_register_operand" "r")
4409                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4410          VQRSHLQ_M_N))
4411   ]
4412   "TARGET_HAVE_MVE"
4413   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
4414   [(set_attr "type" "mve_move")
4415    (set_attr "length""8")])
4416
4417 ;;
4418 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
4419 ;;
4420 (define_insn "mve_vqshlq_m_r_<supf><mode>"
4421   [
4422    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4423         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4424                        (match_operand:SI 2 "s_register_operand" "r")
4425                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4426          VQSHLQ_M_R))
4427   ]
4428   "TARGET_HAVE_MVE"
4429   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4430   [(set_attr "type" "mve_move")
4431    (set_attr "length""8")])
4432
4433 ;;
4434 ;; [vrev64q_m_u, vrev64q_m_s])
4435 ;;
4436 (define_insn "mve_vrev64q_m_<supf><mode>"
4437   [
4438    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4439         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4440                        (match_operand:MVE_2 2 "s_register_operand" "w")
4441                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4442          VREV64Q_M))
4443   ]
4444   "TARGET_HAVE_MVE"
4445   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
4446   [(set_attr "type" "mve_move")
4447    (set_attr "length""8")])
4448
4449 ;;
4450 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
4451 ;;
4452 (define_insn "mve_vrshlq_m_n_<supf><mode>"
4453   [
4454    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4455         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4456                        (match_operand:SI 2 "s_register_operand" "r")
4457                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4458          VRSHLQ_M_N))
4459   ]
4460   "TARGET_HAVE_MVE"
4461   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4462   [(set_attr "type" "mve_move")
4463    (set_attr "length""8")])
4464
4465 ;;
4466 ;; [vshlq_m_r_u, vshlq_m_r_s])
4467 ;;
4468 (define_insn "mve_vshlq_m_r_<supf><mode>"
4469   [
4470    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4471         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4472                        (match_operand:SI 2 "s_register_operand" "r")
4473                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4474          VSHLQ_M_R))
4475   ]
4476   "TARGET_HAVE_MVE"
4477   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4478   [(set_attr "type" "mve_move")
4479    (set_attr "length""8")])
4480
4481 ;;
4482 ;; [vsliq_n_u, vsliq_n_s])
4483 ;;
4484 (define_insn "mve_vsliq_n_<supf><mode>"
4485   [
4486    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4487         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4488                        (match_operand:MVE_2 2 "s_register_operand" "w")
4489                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
4490          VSLIQ_N))
4491   ]
4492   "TARGET_HAVE_MVE"
4493   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
4494   [(set_attr "type" "mve_move")
4495 ])
4496
4497 ;;
4498 ;; [vsriq_n_u, vsriq_n_s])
4499 ;;
4500 (define_insn "mve_vsriq_n_<supf><mode>"
4501   [
4502    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4503         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4504                        (match_operand:MVE_2 2 "s_register_operand" "w")
4505                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
4506          VSRIQ_N))
4507   ]
4508   "TARGET_HAVE_MVE"
4509   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
4510   [(set_attr "type" "mve_move")
4511 ])
4512
4513 ;;
4514 ;; [vqdmlsdhxq_s])
4515 ;;
4516 (define_insn "mve_vqdmlsdhxq_s<mode>"
4517   [
4518    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4519         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4520                        (match_operand:MVE_2 2 "s_register_operand" "w")
4521                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4522          VQDMLSDHXQ_S))
4523   ]
4524   "TARGET_HAVE_MVE"
4525   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4526   [(set_attr "type" "mve_move")
4527 ])
4528
4529 ;;
4530 ;; [vqdmlsdhq_s])
4531 ;;
4532 (define_insn "mve_vqdmlsdhq_s<mode>"
4533   [
4534    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4535         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4536                        (match_operand:MVE_2 2 "s_register_operand" "w")
4537                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4538          VQDMLSDHQ_S))
4539   ]
4540   "TARGET_HAVE_MVE"
4541   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4542   [(set_attr "type" "mve_move")
4543 ])
4544
4545 ;;
4546 ;; [vqdmladhxq_s])
4547 ;;
4548 (define_insn "mve_vqdmladhxq_s<mode>"
4549   [
4550    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4551         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4552                        (match_operand:MVE_2 2 "s_register_operand" "w")
4553                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4554          VQDMLADHXQ_S))
4555   ]
4556   "TARGET_HAVE_MVE"
4557   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4558   [(set_attr "type" "mve_move")
4559 ])
4560
4561 ;;
4562 ;; [vqdmladhq_s])
4563 ;;
4564 (define_insn "mve_vqdmladhq_s<mode>"
4565   [
4566    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4567         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4568                        (match_operand:MVE_2 2 "s_register_operand" "w")
4569                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4570          VQDMLADHQ_S))
4571   ]
4572   "TARGET_HAVE_MVE"
4573   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4574   [(set_attr "type" "mve_move")
4575 ])
4576
4577 ;;
4578 ;; [vmlsdavaxq_s])
4579 ;;
4580 (define_insn "mve_vmlsdavaxq_s<mode>"
4581   [
4582    (set (match_operand:SI 0 "s_register_operand" "=e")
4583         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4584                     (match_operand:MVE_2 2 "s_register_operand" "w")
4585                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4586          VMLSDAVAXQ_S))
4587   ]
4588   "TARGET_HAVE_MVE"
4589   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4590   [(set_attr "type" "mve_move")
4591 ])
4592
4593 ;;
4594 ;; [vmlsdavaq_s])
4595 ;;
4596 (define_insn "mve_vmlsdavaq_s<mode>"
4597   [
4598    (set (match_operand:SI 0 "s_register_operand" "=e")
4599         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4600                     (match_operand:MVE_2 2 "s_register_operand" "w")
4601                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4602          VMLSDAVAQ_S))
4603   ]
4604   "TARGET_HAVE_MVE"
4605   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4606   [(set_attr "type" "mve_move")
4607 ])
4608
4609 ;;
4610 ;; [vmladavaxq_s])
4611 ;;
4612 (define_insn "mve_vmladavaxq_s<mode>"
4613   [
4614    (set (match_operand:SI 0 "s_register_operand" "=e")
4615         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4616                     (match_operand:MVE_2 2 "s_register_operand" "w")
4617                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4618          VMLADAVAXQ_S))
4619   ]
4620   "TARGET_HAVE_MVE"
4621   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4622   [(set_attr "type" "mve_move")
4623 ])
4624 ;;
4625 ;; [vabsq_m_f])
4626 ;;
4627 (define_insn "mve_vabsq_m_f<mode>"
4628   [
4629    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4630         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4631                        (match_operand:MVE_0 2 "s_register_operand" "w")
4632                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4633          VABSQ_M_F))
4634   ]
4635   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4636   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4637   [(set_attr "type" "mve_move")
4638    (set_attr "length""8")])
4639
4640 ;;
4641 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4642 ;;
4643 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4644   [
4645    (set (match_operand:DI 0 "s_register_operand" "=r")
4646         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4647                        (match_operand:V4SI 2 "s_register_operand" "w")
4648                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4649          VADDLVAQ_P))
4650   ]
4651   "TARGET_HAVE_MVE"
4652   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4653   [(set_attr "type" "mve_move")
4654    (set_attr "length""8")])
4655 ;;
4656 ;; [vcmlaq_f])
4657 ;;
4658 (define_insn "mve_vcmlaq_f<mode>"
4659   [
4660    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4661         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4662                        (match_operand:MVE_0 2 "s_register_operand" "w")
4663                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4664          VCMLAQ_F))
4665   ]
4666   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4667   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4668   [(set_attr "type" "mve_move")
4669 ])
4670
4671 ;;
4672 ;; [vcmlaq_rot180_f])
4673 ;;
4674 (define_insn "mve_vcmlaq_rot180_f<mode>"
4675   [
4676    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4677         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4678                        (match_operand:MVE_0 2 "s_register_operand" "w")
4679                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4680          VCMLAQ_ROT180_F))
4681   ]
4682   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4683   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4684   [(set_attr "type" "mve_move")
4685 ])
4686
4687 ;;
4688 ;; [vcmlaq_rot270_f])
4689 ;;
4690 (define_insn "mve_vcmlaq_rot270_f<mode>"
4691   [
4692    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4693         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4694                        (match_operand:MVE_0 2 "s_register_operand" "w")
4695                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4696          VCMLAQ_ROT270_F))
4697   ]
4698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4699   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4700   [(set_attr "type" "mve_move")
4701 ])
4702
4703 ;;
4704 ;; [vcmlaq_rot90_f])
4705 ;;
4706 (define_insn "mve_vcmlaq_rot90_f<mode>"
4707   [
4708    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4709         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4710                        (match_operand:MVE_0 2 "s_register_operand" "w")
4711                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4712          VCMLAQ_ROT90_F))
4713   ]
4714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4715   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4716   [(set_attr "type" "mve_move")
4717 ])
4718
4719 ;;
4720 ;; [vcmpeqq_m_n_f])
4721 ;;
4722 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4723   [
4724    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4725         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4726                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4727                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4728          VCMPEQQ_M_N_F))
4729   ]
4730   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4731   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4732   [(set_attr "type" "mve_move")
4733    (set_attr "length""8")])
4734
4735 ;;
4736 ;; [vcmpgeq_m_f])
4737 ;;
4738 (define_insn "mve_vcmpgeq_m_f<mode>"
4739   [
4740    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4741         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4742                        (match_operand:MVE_0 2 "s_register_operand" "w")
4743                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4744          VCMPGEQ_M_F))
4745   ]
4746   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4747   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4748   [(set_attr "type" "mve_move")
4749    (set_attr "length""8")])
4750
4751 ;;
4752 ;; [vcmpgeq_m_n_f])
4753 ;;
4754 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4755   [
4756    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4757         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4758                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4759                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4760          VCMPGEQ_M_N_F))
4761   ]
4762   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4763   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4764   [(set_attr "type" "mve_move")
4765    (set_attr "length""8")])
4766
4767 ;;
4768 ;; [vcmpgtq_m_f])
4769 ;;
4770 (define_insn "mve_vcmpgtq_m_f<mode>"
4771   [
4772    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4773         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4774                        (match_operand:MVE_0 2 "s_register_operand" "w")
4775                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4776          VCMPGTQ_M_F))
4777   ]
4778   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4779   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4780   [(set_attr "type" "mve_move")
4781    (set_attr "length""8")])
4782
4783 ;;
4784 ;; [vcmpgtq_m_n_f])
4785 ;;
4786 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4787   [
4788    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4789         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4790                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4791                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4792          VCMPGTQ_M_N_F))
4793   ]
4794   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4795   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4796   [(set_attr "type" "mve_move")
4797    (set_attr "length""8")])
4798
4799 ;;
4800 ;; [vcmpleq_m_f])
4801 ;;
4802 (define_insn "mve_vcmpleq_m_f<mode>"
4803   [
4804    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4805         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4806                        (match_operand:MVE_0 2 "s_register_operand" "w")
4807                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4808          VCMPLEQ_M_F))
4809   ]
4810   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4811   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4812   [(set_attr "type" "mve_move")
4813    (set_attr "length""8")])
4814
4815 ;;
4816 ;; [vcmpleq_m_n_f])
4817 ;;
4818 (define_insn "mve_vcmpleq_m_n_f<mode>"
4819   [
4820    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4821         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4822                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4823                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4824          VCMPLEQ_M_N_F))
4825   ]
4826   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4827   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4828   [(set_attr "type" "mve_move")
4829    (set_attr "length""8")])
4830
4831 ;;
4832 ;; [vcmpltq_m_f])
4833 ;;
4834 (define_insn "mve_vcmpltq_m_f<mode>"
4835   [
4836    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4837         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4838                        (match_operand:MVE_0 2 "s_register_operand" "w")
4839                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4840          VCMPLTQ_M_F))
4841   ]
4842   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4843   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4844   [(set_attr "type" "mve_move")
4845    (set_attr "length""8")])
4846
4847 ;;
4848 ;; [vcmpltq_m_n_f])
4849 ;;
4850 (define_insn "mve_vcmpltq_m_n_f<mode>"
4851   [
4852    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4853         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4854                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4855                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4856          VCMPLTQ_M_N_F))
4857   ]
4858   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4859   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4860   [(set_attr "type" "mve_move")
4861    (set_attr "length""8")])
4862
4863 ;;
4864 ;; [vcmpneq_m_f])
4865 ;;
4866 (define_insn "mve_vcmpneq_m_f<mode>"
4867   [
4868    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4869         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4870                        (match_operand:MVE_0 2 "s_register_operand" "w")
4871                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4872          VCMPNEQ_M_F))
4873   ]
4874   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4875   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4876   [(set_attr "type" "mve_move")
4877    (set_attr "length""8")])
4878
4879 ;;
4880 ;; [vcmpneq_m_n_f])
4881 ;;
4882 (define_insn "mve_vcmpneq_m_n_f<mode>"
4883   [
4884    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4885         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4886                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4887                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4888          VCMPNEQ_M_N_F))
4889   ]
4890   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4891   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4892   [(set_attr "type" "mve_move")
4893    (set_attr "length""8")])
4894
4895 ;;
4896 ;; [vcvtbq_m_f16_f32])
4897 ;;
4898 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4899   [
4900    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4901         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4902                        (match_operand:V4SF 2 "s_register_operand" "w")
4903                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4904          VCVTBQ_M_F16_F32))
4905   ]
4906   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4907   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4908   [(set_attr "type" "mve_move")
4909    (set_attr "length""8")])
4910
4911 ;;
4912 ;; [vcvtbq_m_f32_f16])
4913 ;;
4914 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4915   [
4916    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4917         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4918                        (match_operand:V8HF 2 "s_register_operand" "w")
4919                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4920          VCVTBQ_M_F32_F16))
4921   ]
4922   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4923   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4924   [(set_attr "type" "mve_move")
4925    (set_attr "length""8")])
4926
4927 ;;
4928 ;; [vcvttq_m_f16_f32])
4929 ;;
4930 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4931   [
4932    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4933         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4934                        (match_operand:V4SF 2 "s_register_operand" "w")
4935                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4936          VCVTTQ_M_F16_F32))
4937   ]
4938   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4939   "vpst\;vcvttt.f16.f32 %q0, %q2"
4940   [(set_attr "type" "mve_move")
4941    (set_attr "length""8")])
4942
4943 ;;
4944 ;; [vcvttq_m_f32_f16])
4945 ;;
4946 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4947   [
4948    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4949         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4950                        (match_operand:V8HF 2 "s_register_operand" "w")
4951                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4952          VCVTTQ_M_F32_F16))
4953   ]
4954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4955   "vpst\;vcvttt.f32.f16 %q0, %q2"
4956   [(set_attr "type" "mve_move")
4957    (set_attr "length""8")])
4958
4959 ;;
4960 ;; [vdupq_m_n_f])
4961 ;;
4962 (define_insn "mve_vdupq_m_n_f<mode>"
4963   [
4964    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4965         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4966                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4967                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4968          VDUPQ_M_N_F))
4969   ]
4970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4971   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4972   [(set_attr "type" "mve_move")
4973    (set_attr "length""8")])
4974
4975 ;;
4976 ;; [vfmaq_f])
4977 ;;
4978 (define_insn "mve_vfmaq_f<mode>"
4979   [
4980    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4981         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4982                        (match_operand:MVE_0 2 "s_register_operand" "w")
4983                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4984          VFMAQ_F))
4985   ]
4986   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4987   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4988   [(set_attr "type" "mve_move")
4989 ])
4990
4991 ;;
4992 ;; [vfmaq_n_f])
4993 ;;
4994 (define_insn "mve_vfmaq_n_f<mode>"
4995   [
4996    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4997         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4998                        (match_operand:MVE_0 2 "s_register_operand" "w")
4999                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5000          VFMAQ_N_F))
5001   ]
5002   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5003   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
5004   [(set_attr "type" "mve_move")
5005 ])
5006
5007 ;;
5008 ;; [vfmasq_n_f])
5009 ;;
5010 (define_insn "mve_vfmasq_n_f<mode>"
5011   [
5012    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5013         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5014                        (match_operand:MVE_0 2 "s_register_operand" "w")
5015                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5016          VFMASQ_N_F))
5017   ]
5018   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5019   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
5020   [(set_attr "type" "mve_move")
5021 ])
5022 ;;
5023 ;; [vfmsq_f])
5024 ;;
5025 (define_insn "mve_vfmsq_f<mode>"
5026   [
5027    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5028         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5029                        (match_operand:MVE_0 2 "s_register_operand" "w")
5030                        (match_operand:MVE_0 3 "s_register_operand" "w")]
5031          VFMSQ_F))
5032   ]
5033   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5034   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
5035   [(set_attr "type" "mve_move")
5036 ])
5037
5038 ;;
5039 ;; [vmaxnmaq_m_f])
5040 ;;
5041 (define_insn "mve_vmaxnmaq_m_f<mode>"
5042   [
5043    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5044         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5045                        (match_operand:MVE_0 2 "s_register_operand" "w")
5046                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5047          VMAXNMAQ_M_F))
5048   ]
5049   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5050   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
5051   [(set_attr "type" "mve_move")
5052    (set_attr "length""8")])
5053 ;;
5054 ;; [vmaxnmavq_p_f])
5055 ;;
5056 (define_insn "mve_vmaxnmavq_p_f<mode>"
5057   [
5058    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5059         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5060                        (match_operand:MVE_0 2 "s_register_operand" "w")
5061                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5062          VMAXNMAVQ_P_F))
5063   ]
5064   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5065   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
5066   [(set_attr "type" "mve_move")
5067    (set_attr "length""8")])
5068
5069 ;;
5070 ;; [vmaxnmvq_p_f])
5071 ;;
5072 (define_insn "mve_vmaxnmvq_p_f<mode>"
5073   [
5074    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5075         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5076                        (match_operand:MVE_0 2 "s_register_operand" "w")
5077                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5078          VMAXNMVQ_P_F))
5079   ]
5080   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5081   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
5082   [(set_attr "type" "mve_move")
5083    (set_attr "length""8")])
5084 ;;
5085 ;; [vminnmaq_m_f])
5086 ;;
5087 (define_insn "mve_vminnmaq_m_f<mode>"
5088   [
5089    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5090         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5091                        (match_operand:MVE_0 2 "s_register_operand" "w")
5092                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5093          VMINNMAQ_M_F))
5094   ]
5095   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5096   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
5097   [(set_attr "type" "mve_move")
5098    (set_attr "length""8")])
5099
5100 ;;
5101 ;; [vminnmavq_p_f])
5102 ;;
5103 (define_insn "mve_vminnmavq_p_f<mode>"
5104   [
5105    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5106         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5107                        (match_operand:MVE_0 2 "s_register_operand" "w")
5108                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5109          VMINNMAVQ_P_F))
5110   ]
5111   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5112   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
5113   [(set_attr "type" "mve_move")
5114    (set_attr "length""8")])
5115 ;;
5116 ;; [vminnmvq_p_f])
5117 ;;
5118 (define_insn "mve_vminnmvq_p_f<mode>"
5119   [
5120    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5121         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5122                        (match_operand:MVE_0 2 "s_register_operand" "w")
5123                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5124          VMINNMVQ_P_F))
5125   ]
5126   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5127   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
5128   [(set_attr "type" "mve_move")
5129    (set_attr "length""8")])
5130
5131 ;;
5132 ;; [vmlaldavaq_s, vmlaldavaq_u])
5133 ;;
5134 (define_insn "mve_vmlaldavaq_<supf><mode>"
5135   [
5136    (set (match_operand:DI 0 "s_register_operand" "=r")
5137         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5138                        (match_operand:MVE_5 2 "s_register_operand" "w")
5139                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5140          VMLALDAVAQ))
5141   ]
5142   "TARGET_HAVE_MVE"
5143   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5144   [(set_attr "type" "mve_move")
5145 ])
5146
5147 ;;
5148 ;; [vmlaldavaxq_s])
5149 ;;
5150 (define_insn "mve_vmlaldavaxq_s<mode>"
5151   [
5152    (set (match_operand:DI 0 "s_register_operand" "=r")
5153         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5154                        (match_operand:MVE_5 2 "s_register_operand" "w")
5155                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5156          VMLALDAVAXQ_S))
5157   ]
5158   "TARGET_HAVE_MVE"
5159   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5160   [(set_attr "type" "mve_move")
5161 ])
5162
5163 ;;
5164 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
5165 ;;
5166 (define_insn "mve_vmlaldavq_p_<supf><mode>"
5167   [
5168    (set (match_operand:DI 0 "s_register_operand" "=r")
5169         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5170                        (match_operand:MVE_5 2 "s_register_operand" "w")
5171                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5172          VMLALDAVQ_P))
5173   ]
5174   "TARGET_HAVE_MVE"
5175   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5176   [(set_attr "type" "mve_move")
5177    (set_attr "length""8")])
5178
5179 ;;
5180 ;; [vmlaldavxq_p_s])
5181 ;;
5182 (define_insn "mve_vmlaldavxq_p_s<mode>"
5183   [
5184    (set (match_operand:DI 0 "s_register_operand" "=r")
5185         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5186                        (match_operand:MVE_5 2 "s_register_operand" "w")
5187                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5188          VMLALDAVXQ_P_S))
5189   ]
5190   "TARGET_HAVE_MVE"
5191   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
5192   [(set_attr "type" "mve_move")
5193    (set_attr "length""8")])
5194 ;;
5195 ;; [vmlsldavaq_s])
5196 ;;
5197 (define_insn "mve_vmlsldavaq_s<mode>"
5198   [
5199    (set (match_operand:DI 0 "s_register_operand" "=r")
5200         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5201                        (match_operand:MVE_5 2 "s_register_operand" "w")
5202                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5203          VMLSLDAVAQ_S))
5204   ]
5205   "TARGET_HAVE_MVE"
5206   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5207   [(set_attr "type" "mve_move")
5208 ])
5209
5210 ;;
5211 ;; [vmlsldavaxq_s])
5212 ;;
5213 (define_insn "mve_vmlsldavaxq_s<mode>"
5214   [
5215    (set (match_operand:DI 0 "s_register_operand" "=r")
5216         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5217                        (match_operand:MVE_5 2 "s_register_operand" "w")
5218                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5219          VMLSLDAVAXQ_S))
5220   ]
5221   "TARGET_HAVE_MVE"
5222   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5223   [(set_attr "type" "mve_move")
5224 ])
5225
5226 ;;
5227 ;; [vmlsldavq_p_s])
5228 ;;
5229 (define_insn "mve_vmlsldavq_p_s<mode>"
5230   [
5231    (set (match_operand:DI 0 "s_register_operand" "=r")
5232         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5233                        (match_operand:MVE_5 2 "s_register_operand" "w")
5234                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5235          VMLSLDAVQ_P_S))
5236   ]
5237   "TARGET_HAVE_MVE"
5238   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5239   [(set_attr "type" "mve_move")
5240    (set_attr "length""8")])
5241
5242 ;;
5243 ;; [vmlsldavxq_p_s])
5244 ;;
5245 (define_insn "mve_vmlsldavxq_p_s<mode>"
5246   [
5247    (set (match_operand:DI 0 "s_register_operand" "=r")
5248         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5249                        (match_operand:MVE_5 2 "s_register_operand" "w")
5250                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5251          VMLSLDAVXQ_P_S))
5252   ]
5253   "TARGET_HAVE_MVE"
5254   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5255   [(set_attr "type" "mve_move")
5256    (set_attr "length""8")])
5257 ;;
5258 ;; [vmovlbq_m_u, vmovlbq_m_s])
5259 ;;
5260 (define_insn "mve_vmovlbq_m_<supf><mode>"
5261   [
5262    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5263         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5264                        (match_operand:MVE_3 2 "s_register_operand" "w")
5265                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5266          VMOVLBQ_M))
5267   ]
5268   "TARGET_HAVE_MVE"
5269   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
5270   [(set_attr "type" "mve_move")
5271    (set_attr "length""8")])
5272 ;;
5273 ;; [vmovltq_m_u, vmovltq_m_s])
5274 ;;
5275 (define_insn "mve_vmovltq_m_<supf><mode>"
5276   [
5277    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5278         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5279                        (match_operand:MVE_3 2 "s_register_operand" "w")
5280                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5281          VMOVLTQ_M))
5282   ]
5283   "TARGET_HAVE_MVE"
5284   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
5285   [(set_attr "type" "mve_move")
5286    (set_attr "length""8")])
5287 ;;
5288 ;; [vmovnbq_m_u, vmovnbq_m_s])
5289 ;;
5290 (define_insn "mve_vmovnbq_m_<supf><mode>"
5291   [
5292    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5293         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5294                        (match_operand:MVE_5 2 "s_register_operand" "w")
5295                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5296          VMOVNBQ_M))
5297   ]
5298   "TARGET_HAVE_MVE"
5299   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
5300   [(set_attr "type" "mve_move")
5301    (set_attr "length""8")])
5302
5303 ;;
5304 ;; [vmovntq_m_u, vmovntq_m_s])
5305 ;;
5306 (define_insn "mve_vmovntq_m_<supf><mode>"
5307   [
5308    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5309         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5310                        (match_operand:MVE_5 2 "s_register_operand" "w")
5311                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5312          VMOVNTQ_M))
5313   ]
5314   "TARGET_HAVE_MVE"
5315   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
5316   [(set_attr "type" "mve_move")
5317    (set_attr "length""8")])
5318
5319 ;;
5320 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
5321 ;;
5322 (define_insn "mve_vmvnq_m_n_<supf><mode>"
5323   [
5324    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5325         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5326                        (match_operand:SI 2 "immediate_operand" "i")
5327                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5328          VMVNQ_M_N))
5329   ]
5330   "TARGET_HAVE_MVE"
5331   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
5332   [(set_attr "type" "mve_move")
5333    (set_attr "length""8")])
5334 ;;
5335 ;; [vnegq_m_f])
5336 ;;
5337 (define_insn "mve_vnegq_m_f<mode>"
5338   [
5339    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5340         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5341                        (match_operand:MVE_0 2 "s_register_operand" "w")
5342                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5343          VNEGQ_M_F))
5344   ]
5345   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5346   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
5347   [(set_attr "type" "mve_move")
5348    (set_attr "length""8")])
5349
5350 ;;
5351 ;; [vorrq_m_n_s, vorrq_m_n_u])
5352 ;;
5353 (define_insn "mve_vorrq_m_n_<supf><mode>"
5354   [
5355    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5356         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5357                        (match_operand:SI 2 "immediate_operand" "i")
5358                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5359          VORRQ_M_N))
5360   ]
5361   "TARGET_HAVE_MVE"
5362   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
5363   [(set_attr "type" "mve_move")
5364    (set_attr "length""8")])
5365 ;;
5366 ;; [vpselq_f])
5367 ;;
5368 (define_insn "mve_vpselq_f<mode>"
5369   [
5370    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5371         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
5372                        (match_operand:MVE_0 2 "s_register_operand" "w")
5373                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5374          VPSELQ_F))
5375   ]
5376   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5377   "vpsel %q0, %q1, %q2"
5378   [(set_attr "type" "mve_move")
5379 ])
5380
5381 ;;
5382 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
5383 ;;
5384 (define_insn "mve_vqmovnbq_m_<supf><mode>"
5385   [
5386    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5387         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5388                        (match_operand:MVE_5 2 "s_register_operand" "w")
5389                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5390          VQMOVNBQ_M))
5391   ]
5392   "TARGET_HAVE_MVE"
5393   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
5394   [(set_attr "type" "mve_move")
5395    (set_attr "length""8")])
5396
5397 ;;
5398 ;; [vqmovntq_m_u, vqmovntq_m_s])
5399 ;;
5400 (define_insn "mve_vqmovntq_m_<supf><mode>"
5401   [
5402    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5403         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5404                        (match_operand:MVE_5 2 "s_register_operand" "w")
5405                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5406          VQMOVNTQ_M))
5407   ]
5408   "TARGET_HAVE_MVE"
5409   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
5410   [(set_attr "type" "mve_move")
5411    (set_attr "length""8")])
5412
5413 ;;
5414 ;; [vqmovunbq_m_s])
5415 ;;
5416 (define_insn "mve_vqmovunbq_m_s<mode>"
5417   [
5418    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5419         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5420                        (match_operand:MVE_5 2 "s_register_operand" "w")
5421                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5422          VQMOVUNBQ_M_S))
5423   ]
5424   "TARGET_HAVE_MVE"
5425   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
5426   [(set_attr "type" "mve_move")
5427    (set_attr "length""8")])
5428
5429 ;;
5430 ;; [vqmovuntq_m_s])
5431 ;;
5432 (define_insn "mve_vqmovuntq_m_s<mode>"
5433   [
5434    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5435         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5436                        (match_operand:MVE_5 2 "s_register_operand" "w")
5437                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5438          VQMOVUNTQ_M_S))
5439   ]
5440   "TARGET_HAVE_MVE"
5441   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
5442   [(set_attr "type" "mve_move")
5443    (set_attr "length""8")])
5444
5445 ;;
5446 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
5447 ;;
5448 (define_insn "mve_vqrshrntq_n_<supf><mode>"
5449   [
5450    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5451         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5452                        (match_operand:MVE_5 2 "s_register_operand" "w")
5453                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5454          VQRSHRNTQ_N))
5455   ]
5456   "TARGET_HAVE_MVE"
5457   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
5458   [(set_attr "type" "mve_move")
5459 ])
5460
5461 ;;
5462 ;; [vqrshruntq_n_s])
5463 ;;
5464 (define_insn "mve_vqrshruntq_n_s<mode>"
5465   [
5466    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5467         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5468                        (match_operand:MVE_5 2 "s_register_operand" "w")
5469                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5470          VQRSHRUNTQ_N_S))
5471   ]
5472   "TARGET_HAVE_MVE"
5473   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
5474   [(set_attr "type" "mve_move")
5475 ])
5476
5477 ;;
5478 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
5479 ;;
5480 (define_insn "mve_vqshrnbq_n_<supf><mode>"
5481   [
5482    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5483         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5484                        (match_operand:MVE_5 2 "s_register_operand" "w")
5485                        (match_operand:SI 3 "<MVE_pred1>" "<MVE_constraint1>")]
5486          VQSHRNBQ_N))
5487   ]
5488   "TARGET_HAVE_MVE"
5489   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5490   [(set_attr "type" "mve_move")
5491 ])
5492
5493 ;;
5494 ;; [vqshrntq_n_u, vqshrntq_n_s])
5495 ;;
5496 (define_insn "mve_vqshrntq_n_<supf><mode>"
5497   [
5498    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5499         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5500                        (match_operand:MVE_5 2 "s_register_operand" "w")
5501                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5502          VQSHRNTQ_N))
5503   ]
5504   "TARGET_HAVE_MVE"
5505   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
5506   [(set_attr "type" "mve_move")
5507 ])
5508
5509 ;;
5510 ;; [vqshrunbq_n_s])
5511 ;;
5512 (define_insn "mve_vqshrunbq_n_s<mode>"
5513   [
5514    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5515         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5516                        (match_operand:MVE_5 2 "s_register_operand" "w")
5517                        (match_operand:SI 3 "immediate_operand" "i")]
5518          VQSHRUNBQ_N_S))
5519   ]
5520   "TARGET_HAVE_MVE"
5521   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5522   [(set_attr "type" "mve_move")
5523 ])
5524
5525 ;;
5526 ;; [vqshruntq_n_s])
5527 ;;
5528 (define_insn "mve_vqshruntq_n_s<mode>"
5529   [
5530    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5531         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5532                        (match_operand:MVE_5 2 "s_register_operand" "w")
5533                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5534          VQSHRUNTQ_N_S))
5535   ]
5536   "TARGET_HAVE_MVE"
5537   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5538   [(set_attr "type" "mve_move")
5539 ])
5540
5541 ;;
5542 ;; [vrev32q_m_f])
5543 ;;
5544 (define_insn "mve_vrev32q_m_fv8hf"
5545   [
5546    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5547         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5548                        (match_operand:V8HF 2 "s_register_operand" "w")
5549                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5550          VREV32Q_M_F))
5551   ]
5552   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5553   "vpst\;vrev32t.16 %q0, %q2"
5554   [(set_attr "type" "mve_move")
5555    (set_attr "length""8")])
5556
5557 ;;
5558 ;; [vrev32q_m_s, vrev32q_m_u])
5559 ;;
5560 (define_insn "mve_vrev32q_m_<supf><mode>"
5561   [
5562    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5563         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5564                        (match_operand:MVE_3 2 "s_register_operand" "w")
5565                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5566          VREV32Q_M))
5567   ]
5568   "TARGET_HAVE_MVE"
5569   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5570   [(set_attr "type" "mve_move")
5571    (set_attr "length""8")])
5572
5573 ;;
5574 ;; [vrev64q_m_f])
5575 ;;
5576 (define_insn "mve_vrev64q_m_f<mode>"
5577   [
5578    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5579         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5580                        (match_operand:MVE_0 2 "s_register_operand" "w")
5581                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5582          VREV64Q_M_F))
5583   ]
5584   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5585   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5586   [(set_attr "type" "mve_move")
5587    (set_attr "length""8")])
5588
5589 ;;
5590 ;; [vrmlaldavhaxq_s])
5591 ;;
5592 (define_insn "mve_vrmlaldavhaxq_sv4si"
5593   [
5594    (set (match_operand:DI 0 "s_register_operand" "=r")
5595         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5596                        (match_operand:V4SI 2 "s_register_operand" "w")
5597                        (match_operand:V4SI 3 "s_register_operand" "w")]
5598          VRMLALDAVHAXQ_S))
5599   ]
5600   "TARGET_HAVE_MVE"
5601   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5602   [(set_attr "type" "mve_move")
5603 ])
5604
5605 ;;
5606 ;; [vrmlaldavhxq_p_s])
5607 ;;
5608 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5609   [
5610    (set (match_operand:DI 0 "s_register_operand" "=r")
5611         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5612                        (match_operand:V4SI 2 "s_register_operand" "w")
5613                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5614          VRMLALDAVHXQ_P_S))
5615   ]
5616   "TARGET_HAVE_MVE"
5617   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5618   [(set_attr "type" "mve_move")
5619    (set_attr "length""8")])
5620
5621 ;;
5622 ;; [vrmlsldavhaxq_s])
5623 ;;
5624 (define_insn "mve_vrmlsldavhaxq_sv4si"
5625   [
5626    (set (match_operand:DI 0 "s_register_operand" "=r")
5627         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5628                        (match_operand:V4SI 2 "s_register_operand" "w")
5629                        (match_operand:V4SI 3 "s_register_operand" "w")]
5630          VRMLSLDAVHAXQ_S))
5631   ]
5632   "TARGET_HAVE_MVE"
5633   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5634   [(set_attr "type" "mve_move")
5635 ])
5636
5637 ;;
5638 ;; [vrmlsldavhq_p_s])
5639 ;;
5640 (define_insn "mve_vrmlsldavhq_p_sv4si"
5641   [
5642    (set (match_operand:DI 0 "s_register_operand" "=r")
5643         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5644                        (match_operand:V4SI 2 "s_register_operand" "w")
5645                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5646          VRMLSLDAVHQ_P_S))
5647   ]
5648   "TARGET_HAVE_MVE"
5649   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5650   [(set_attr "type" "mve_move")
5651    (set_attr "length""8")])
5652
5653 ;;
5654 ;; [vrmlsldavhxq_p_s])
5655 ;;
5656 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5657   [
5658    (set (match_operand:DI 0 "s_register_operand" "=r")
5659         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5660                        (match_operand:V4SI 2 "s_register_operand" "w")
5661                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5662          VRMLSLDAVHXQ_P_S))
5663   ]
5664   "TARGET_HAVE_MVE"
5665   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5666   [(set_attr "type" "mve_move")
5667    (set_attr "length""8")])
5668
5669 ;;
5670 ;; [vrndaq_m_f])
5671 ;;
5672 (define_insn "mve_vrndaq_m_f<mode>"
5673   [
5674    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5675         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5676                        (match_operand:MVE_0 2 "s_register_operand" "w")
5677                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5678          VRNDAQ_M_F))
5679   ]
5680   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5681   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5682   [(set_attr "type" "mve_move")
5683    (set_attr "length""8")])
5684
5685 ;;
5686 ;; [vrndmq_m_f])
5687 ;;
5688 (define_insn "mve_vrndmq_m_f<mode>"
5689   [
5690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5691         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5692                        (match_operand:MVE_0 2 "s_register_operand" "w")
5693                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5694          VRNDMQ_M_F))
5695   ]
5696   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5697   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5698   [(set_attr "type" "mve_move")
5699    (set_attr "length""8")])
5700
5701 ;;
5702 ;; [vrndnq_m_f])
5703 ;;
5704 (define_insn "mve_vrndnq_m_f<mode>"
5705   [
5706    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5707         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5708                        (match_operand:MVE_0 2 "s_register_operand" "w")
5709                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5710          VRNDNQ_M_F))
5711   ]
5712   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5713   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5714   [(set_attr "type" "mve_move")
5715    (set_attr "length""8")])
5716
5717 ;;
5718 ;; [vrndpq_m_f])
5719 ;;
5720 (define_insn "mve_vrndpq_m_f<mode>"
5721   [
5722    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5723         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5724                        (match_operand:MVE_0 2 "s_register_operand" "w")
5725                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5726          VRNDPQ_M_F))
5727   ]
5728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5729   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5730   [(set_attr "type" "mve_move")
5731    (set_attr "length""8")])
5732
5733 ;;
5734 ;; [vrndxq_m_f])
5735 ;;
5736 (define_insn "mve_vrndxq_m_f<mode>"
5737   [
5738    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5739         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5740                        (match_operand:MVE_0 2 "s_register_operand" "w")
5741                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5742          VRNDXQ_M_F))
5743   ]
5744   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5745   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5746   [(set_attr "type" "mve_move")
5747    (set_attr "length""8")])
5748
5749 ;;
5750 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5751 ;;
5752 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5753   [
5754    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5755         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5756                        (match_operand:MVE_5 2 "s_register_operand" "w")
5757                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5758          VRSHRNBQ_N))
5759   ]
5760   "TARGET_HAVE_MVE"
5761   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5762   [(set_attr "type" "mve_move")
5763 ])
5764
5765 ;;
5766 ;; [vrshrntq_n_u, vrshrntq_n_s])
5767 ;;
5768 (define_insn "mve_vrshrntq_n_<supf><mode>"
5769   [
5770    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5771         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5772                        (match_operand:MVE_5 2 "s_register_operand" "w")
5773                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5774          VRSHRNTQ_N))
5775   ]
5776   "TARGET_HAVE_MVE"
5777   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5778   [(set_attr "type" "mve_move")
5779 ])
5780
5781 ;;
5782 ;; [vshrnbq_n_u, vshrnbq_n_s])
5783 ;;
5784 (define_insn "mve_vshrnbq_n_<supf><mode>"
5785   [
5786    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5787         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5788                        (match_operand:MVE_5 2 "s_register_operand" "w")
5789                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5790          VSHRNBQ_N))
5791   ]
5792   "TARGET_HAVE_MVE"
5793   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5794   [(set_attr "type" "mve_move")
5795 ])
5796
5797 ;;
5798 ;; [vshrntq_n_s, vshrntq_n_u])
5799 ;;
5800 (define_insn "mve_vshrntq_n_<supf><mode>"
5801   [
5802    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5803         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5804                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5805                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5806          VSHRNTQ_N))
5807   ]
5808   "TARGET_HAVE_MVE"
5809   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5810   [(set_attr "type" "mve_move")
5811 ])
5812
5813 ;;
5814 ;; [vcvtmq_m_s, vcvtmq_m_u])
5815 ;;
5816 (define_insn "mve_vcvtmq_m_<supf><mode>"
5817   [
5818    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5819         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5820                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5821                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5822          VCVTMQ_M))
5823   ]
5824   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5825   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5826   [(set_attr "type" "mve_move")
5827    (set_attr "length""8")])
5828
5829 ;;
5830 ;; [vcvtpq_m_u, vcvtpq_m_s])
5831 ;;
5832 (define_insn "mve_vcvtpq_m_<supf><mode>"
5833   [
5834    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5835         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5836                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5837                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5838          VCVTPQ_M))
5839   ]
5840   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5841   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5842   [(set_attr "type" "mve_move")
5843    (set_attr "length""8")])
5844
5845 ;;
5846 ;; [vcvtnq_m_s, vcvtnq_m_u])
5847 ;;
5848 (define_insn "mve_vcvtnq_m_<supf><mode>"
5849   [
5850    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5851         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5852                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5853                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5854          VCVTNQ_M))
5855   ]
5856   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5857   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5858   [(set_attr "type" "mve_move")
5859    (set_attr "length""8")])
5860
5861 ;;
5862 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5863 ;;
5864 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5865   [
5866    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5867         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5868                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5869                        (match_operand:SI 3 "mve_imm_16" "Rd")
5870                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5871          VCVTQ_M_N_FROM_F))
5872   ]
5873   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5874   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5875   [(set_attr "type" "mve_move")
5876    (set_attr "length""8")])
5877
5878 ;;
5879 ;; [vrev16q_m_u, vrev16q_m_s])
5880 ;;
5881 (define_insn "mve_vrev16q_m_<supf>v16qi"
5882   [
5883    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5884         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5885                        (match_operand:V16QI 2 "s_register_operand" "w")
5886                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5887          VREV16Q_M))
5888   ]
5889   "TARGET_HAVE_MVE"
5890   "vpst\;vrev16t.8 %q0, %q2"
5891   [(set_attr "type" "mve_move")
5892    (set_attr "length""8")])
5893
5894 ;;
5895 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5896 ;;
5897 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5898   [
5899    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5900         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5901                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5902                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5903          VCVTQ_M_FROM_F))
5904   ]
5905   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5906   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5907   [(set_attr "type" "mve_move")
5908    (set_attr "length""8")])
5909
5910 ;;
5911 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5912 ;;
5913 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5914   [
5915    (set (match_operand:DI 0 "s_register_operand" "=r")
5916         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5917                     (match_operand:V4SI 2 "s_register_operand" "w")
5918                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5919          VRMLALDAVHQ_P))
5920   ]
5921   "TARGET_HAVE_MVE"
5922   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5923   [(set_attr "type" "mve_move")
5924    (set_attr "length""8")])
5925
5926 ;;
5927 ;; [vrmlsldavhaq_s])
5928 ;;
5929 (define_insn "mve_vrmlsldavhaq_sv4si"
5930   [
5931    (set (match_operand:DI 0 "s_register_operand" "=r")
5932         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5933                     (match_operand:V4SI 2 "s_register_operand" "w")
5934                     (match_operand:V4SI 3 "s_register_operand" "w")]
5935          VRMLSLDAVHAQ_S))
5936   ]
5937   "TARGET_HAVE_MVE"
5938   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5939   [(set_attr "type" "mve_move")
5940 ])
5941
5942 ;;
5943 ;; [vabavq_p_s, vabavq_p_u])
5944 ;;
5945 (define_insn "mve_vabavq_p_<supf><mode>"
5946   [
5947    (set (match_operand:SI 0 "s_register_operand" "=r")
5948         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5949                     (match_operand:MVE_2 2 "s_register_operand" "w")
5950                     (match_operand:MVE_2 3 "s_register_operand" "w")
5951                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5952          VABAVQ_P))
5953   ]
5954   "TARGET_HAVE_MVE"
5955   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5956   [(set_attr "type" "mve_move")
5957 ])
5958
5959 ;;
5960 ;; [vqshluq_m_n_s])
5961 ;;
5962 (define_insn "mve_vqshluq_m_n_s<mode>"
5963   [
5964    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5965         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5966                        (match_operand:MVE_2 2 "s_register_operand" "w")
5967                        (match_operand:SI 3 "mve_imm_7" "Ra")
5968                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5969          VQSHLUQ_M_N_S))
5970   ]
5971   "TARGET_HAVE_MVE"
5972   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5973   [(set_attr "type" "mve_move")])
5974
5975 ;;
5976 ;; [vshlq_m_s, vshlq_m_u])
5977 ;;
5978 (define_insn "mve_vshlq_m_<supf><mode>"
5979   [
5980    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5981         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5982                        (match_operand:MVE_2 2 "s_register_operand" "w")
5983                        (match_operand:MVE_2 3 "s_register_operand" "w")
5984                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5985          VSHLQ_M))
5986   ]
5987   "TARGET_HAVE_MVE"
5988   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5989   [(set_attr "type" "mve_move")])
5990
5991 ;;
5992 ;; [vsriq_m_n_s, vsriq_m_n_u])
5993 ;;
5994 (define_insn "mve_vsriq_m_n_<supf><mode>"
5995   [
5996    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5997         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5998                        (match_operand:MVE_2 2 "s_register_operand" "w")
5999                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
6000                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6001          VSRIQ_M_N))
6002   ]
6003   "TARGET_HAVE_MVE"
6004   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
6005   [(set_attr "type" "mve_move")])
6006
6007 ;;
6008 ;; [vsubq_m_u, vsubq_m_s])
6009 ;;
6010 (define_insn "mve_vsubq_m_<supf><mode>"
6011   [
6012    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6013         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6014                        (match_operand:MVE_2 2 "s_register_operand" "w")
6015                        (match_operand:MVE_2 3 "s_register_operand" "w")
6016                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6017          VSUBQ_M))
6018   ]
6019   "TARGET_HAVE_MVE"
6020   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
6021   [(set_attr "type" "mve_move")])
6022
6023 ;;
6024 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
6025 ;;
6026 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
6027   [
6028    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6029         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6030                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
6031                        (match_operand:SI 3 "mve_imm_16" "Rd")
6032                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6033          VCVTQ_M_N_TO_F))
6034   ]
6035   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6036   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6037   [(set_attr "type" "mve_move")
6038    (set_attr "length""8")])
6039 ;;
6040 ;; [vabdq_m_s, vabdq_m_u])
6041 ;;
6042 (define_insn "mve_vabdq_m_<supf><mode>"
6043   [
6044    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6045         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6046                        (match_operand:MVE_2 2 "s_register_operand" "w")
6047                        (match_operand:MVE_2 3 "s_register_operand" "w")
6048                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6049          VABDQ_M))
6050   ]
6051   "TARGET_HAVE_MVE"
6052   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6053   [(set_attr "type" "mve_move")
6054    (set_attr "length""8")])
6055
6056 ;;
6057 ;; [vaddq_m_n_s, vaddq_m_n_u])
6058 ;;
6059 (define_insn "mve_vaddq_m_n_<supf><mode>"
6060   [
6061    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6062         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6063                        (match_operand:MVE_2 2 "s_register_operand" "w")
6064                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6065                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6066          VADDQ_M_N))
6067   ]
6068   "TARGET_HAVE_MVE"
6069   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
6070   [(set_attr "type" "mve_move")
6071    (set_attr "length""8")])
6072
6073 ;;
6074 ;; [vaddq_m_u, vaddq_m_s])
6075 ;;
6076 (define_insn "mve_vaddq_m_<supf><mode>"
6077   [
6078    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6079         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6080                        (match_operand:MVE_2 2 "s_register_operand" "w")
6081                        (match_operand:MVE_2 3 "s_register_operand" "w")
6082                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6083          VADDQ_M))
6084   ]
6085   "TARGET_HAVE_MVE"
6086   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
6087   [(set_attr "type" "mve_move")
6088    (set_attr "length""8")])
6089
6090 ;;
6091 ;; [vandq_m_u, vandq_m_s])
6092 ;;
6093 (define_insn "mve_vandq_m_<supf><mode>"
6094   [
6095    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6096         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6097                        (match_operand:MVE_2 2 "s_register_operand" "w")
6098                        (match_operand:MVE_2 3 "s_register_operand" "w")
6099                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6100          VANDQ_M))
6101   ]
6102   "TARGET_HAVE_MVE"
6103   "vpst\;vandt %q0, %q2, %q3"
6104   [(set_attr "type" "mve_move")
6105    (set_attr "length""8")])
6106
6107 ;;
6108 ;; [vbicq_m_u, vbicq_m_s])
6109 ;;
6110 (define_insn "mve_vbicq_m_<supf><mode>"
6111   [
6112    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6113         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6114                        (match_operand:MVE_2 2 "s_register_operand" "w")
6115                        (match_operand:MVE_2 3 "s_register_operand" "w")
6116                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6117          VBICQ_M))
6118   ]
6119   "TARGET_HAVE_MVE"
6120   "vpst\;vbict %q0, %q2, %q3"
6121   [(set_attr "type" "mve_move")
6122    (set_attr "length""8")])
6123
6124 ;;
6125 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
6126 ;;
6127 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
6128   [
6129    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6130         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6131                        (match_operand:MVE_2 2 "s_register_operand" "w")
6132                        (match_operand:SI 3 "s_register_operand" "r")
6133                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6134          VBRSRQ_M_N))
6135   ]
6136   "TARGET_HAVE_MVE"
6137   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6138   [(set_attr "type" "mve_move")
6139    (set_attr "length""8")])
6140
6141 ;;
6142 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
6143 ;;
6144 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
6145   [
6146    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6147         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6148                        (match_operand:MVE_2 2 "s_register_operand" "w")
6149                        (match_operand:MVE_2 3 "s_register_operand" "w")
6150                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6151          VCADDQ_ROT270_M))
6152   ]
6153   "TARGET_HAVE_MVE"
6154   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
6155   [(set_attr "type" "mve_move")
6156    (set_attr "length""8")])
6157
6158 ;;
6159 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
6160 ;;
6161 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
6162   [
6163    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6164         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6165                        (match_operand:MVE_2 2 "s_register_operand" "w")
6166                        (match_operand:MVE_2 3 "s_register_operand" "w")
6167                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6168          VCADDQ_ROT90_M))
6169   ]
6170   "TARGET_HAVE_MVE"
6171   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
6172   [(set_attr "type" "mve_move")
6173    (set_attr "length""8")])
6174
6175 ;;
6176 ;; [veorq_m_s, veorq_m_u])
6177 ;;
6178 (define_insn "mve_veorq_m_<supf><mode>"
6179   [
6180    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6181         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6182                        (match_operand:MVE_2 2 "s_register_operand" "w")
6183                        (match_operand:MVE_2 3 "s_register_operand" "w")
6184                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6185          VEORQ_M))
6186   ]
6187   "TARGET_HAVE_MVE"
6188   "vpst\;veort %q0, %q2, %q3"
6189   [(set_attr "type" "mve_move")
6190    (set_attr "length""8")])
6191
6192 ;;
6193 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
6194 ;;
6195 (define_insn "mve_vhaddq_m_n_<supf><mode>"
6196   [
6197    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6198         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6199                        (match_operand:MVE_2 2 "s_register_operand" "w")
6200                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6201                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6202          VHADDQ_M_N))
6203   ]
6204   "TARGET_HAVE_MVE"
6205   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6206   [(set_attr "type" "mve_move")
6207    (set_attr "length""8")])
6208
6209 ;;
6210 ;; [vhaddq_m_s, vhaddq_m_u])
6211 ;;
6212 (define_insn "mve_vhaddq_m_<supf><mode>"
6213   [
6214    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6215         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6216                        (match_operand:MVE_2 2 "s_register_operand" "w")
6217                        (match_operand:MVE_2 3 "s_register_operand" "w")
6218                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6219          VHADDQ_M))
6220   ]
6221   "TARGET_HAVE_MVE"
6222   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6223   [(set_attr "type" "mve_move")
6224    (set_attr "length""8")])
6225
6226 ;;
6227 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
6228 ;;
6229 (define_insn "mve_vhsubq_m_n_<supf><mode>"
6230   [
6231    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6232         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6233                        (match_operand:MVE_2 2 "s_register_operand" "w")
6234                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6235                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6236          VHSUBQ_M_N))
6237   ]
6238   "TARGET_HAVE_MVE"
6239   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6240   [(set_attr "type" "mve_move")
6241    (set_attr "length""8")])
6242
6243 ;;
6244 ;; [vhsubq_m_s, vhsubq_m_u])
6245 ;;
6246 (define_insn "mve_vhsubq_m_<supf><mode>"
6247   [
6248    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6249         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6250                        (match_operand:MVE_2 2 "s_register_operand" "w")
6251                        (match_operand:MVE_2 3 "s_register_operand" "w")
6252                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6253          VHSUBQ_M))
6254   ]
6255   "TARGET_HAVE_MVE"
6256   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6257   [(set_attr "type" "mve_move")
6258    (set_attr "length""8")])
6259
6260 ;;
6261 ;; [vmaxq_m_s, vmaxq_m_u])
6262 ;;
6263 (define_insn "mve_vmaxq_m_<supf><mode>"
6264   [
6265    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6266         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6267                        (match_operand:MVE_2 2 "s_register_operand" "w")
6268                        (match_operand:MVE_2 3 "s_register_operand" "w")
6269                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6270          VMAXQ_M))
6271   ]
6272   "TARGET_HAVE_MVE"
6273   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6274   [(set_attr "type" "mve_move")
6275    (set_attr "length""8")])
6276
6277 ;;
6278 ;; [vminq_m_s, vminq_m_u])
6279 ;;
6280 (define_insn "mve_vminq_m_<supf><mode>"
6281   [
6282    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6283         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6284                        (match_operand:MVE_2 2 "s_register_operand" "w")
6285                        (match_operand:MVE_2 3 "s_register_operand" "w")
6286                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6287          VMINQ_M))
6288   ]
6289   "TARGET_HAVE_MVE"
6290   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6291   [(set_attr "type" "mve_move")
6292    (set_attr "length""8")])
6293
6294 ;;
6295 ;; [vmladavaq_p_u, vmladavaq_p_s])
6296 ;;
6297 (define_insn "mve_vmladavaq_p_<supf><mode>"
6298   [
6299    (set (match_operand:SI 0 "s_register_operand" "=e")
6300         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6301                     (match_operand:MVE_2 2 "s_register_operand" "w")
6302                     (match_operand:MVE_2 3 "s_register_operand" "w")
6303                     (match_operand:HI 4 "vpr_register_operand" "Up")]
6304          VMLADAVAQ_P))
6305   ]
6306   "TARGET_HAVE_MVE"
6307   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
6308   [(set_attr "type" "mve_move")
6309    (set_attr "length""8")])
6310
6311 ;;
6312 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
6313 ;;
6314 (define_insn "mve_vmlaq_m_n_<supf><mode>"
6315   [
6316    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6317         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6318                        (match_operand:MVE_2 2 "s_register_operand" "w")
6319                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6320                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6321          VMLAQ_M_N))
6322   ]
6323   "TARGET_HAVE_MVE"
6324   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
6325   [(set_attr "type" "mve_move")
6326    (set_attr "length""8")])
6327
6328 ;;
6329 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
6330 ;;
6331 (define_insn "mve_vmlasq_m_n_<supf><mode>"
6332   [
6333    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6334         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6335                        (match_operand:MVE_2 2 "s_register_operand" "w")
6336                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6337                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6338          VMLASQ_M_N))
6339   ]
6340   "TARGET_HAVE_MVE"
6341   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6342   [(set_attr "type" "mve_move")
6343    (set_attr "length""8")])
6344
6345 ;;
6346 ;; [vmulhq_m_s, vmulhq_m_u])
6347 ;;
6348 (define_insn "mve_vmulhq_m_<supf><mode>"
6349   [
6350    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6351         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6352                        (match_operand:MVE_2 2 "s_register_operand" "w")
6353                        (match_operand:MVE_2 3 "s_register_operand" "w")
6354                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6355          VMULHQ_M))
6356   ]
6357   "TARGET_HAVE_MVE"
6358   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6359   [(set_attr "type" "mve_move")
6360    (set_attr "length""8")])
6361
6362 ;;
6363 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
6364 ;;
6365 (define_insn "mve_vmullbq_int_m_<supf><mode>"
6366   [
6367    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6368         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6369                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6370                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6371                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6372          VMULLBQ_INT_M))
6373   ]
6374   "TARGET_HAVE_MVE"
6375   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6376   [(set_attr "type" "mve_move")
6377    (set_attr "length""8")])
6378
6379 ;;
6380 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
6381 ;;
6382 (define_insn "mve_vmulltq_int_m_<supf><mode>"
6383   [
6384    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6385         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6386                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6387                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6388                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6389          VMULLTQ_INT_M))
6390   ]
6391   "TARGET_HAVE_MVE"
6392   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6393   [(set_attr "type" "mve_move")
6394    (set_attr "length""8")])
6395
6396 ;;
6397 ;; [vmulq_m_n_u, vmulq_m_n_s])
6398 ;;
6399 (define_insn "mve_vmulq_m_n_<supf><mode>"
6400   [
6401    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6402         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6403                        (match_operand:MVE_2 2 "s_register_operand" "w")
6404                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6405                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6406          VMULQ_M_N))
6407   ]
6408   "TARGET_HAVE_MVE"
6409   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
6410   [(set_attr "type" "mve_move")
6411    (set_attr "length""8")])
6412
6413 ;;
6414 ;; [vmulq_m_s, vmulq_m_u])
6415 ;;
6416 (define_insn "mve_vmulq_m_<supf><mode>"
6417   [
6418    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6419         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6420                        (match_operand:MVE_2 2 "s_register_operand" "w")
6421                        (match_operand:MVE_2 3 "s_register_operand" "w")
6422                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6423          VMULQ_M))
6424   ]
6425   "TARGET_HAVE_MVE"
6426   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
6427   [(set_attr "type" "mve_move")
6428    (set_attr "length""8")])
6429
6430 ;;
6431 ;; [vornq_m_u, vornq_m_s])
6432 ;;
6433 (define_insn "mve_vornq_m_<supf><mode>"
6434   [
6435    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6436         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6437                        (match_operand:MVE_2 2 "s_register_operand" "w")
6438                        (match_operand:MVE_2 3 "s_register_operand" "w")
6439                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6440          VORNQ_M))
6441   ]
6442   "TARGET_HAVE_MVE"
6443   "vpst\;vornt %q0, %q2, %q3"
6444   [(set_attr "type" "mve_move")
6445    (set_attr "length""8")])
6446
6447 ;;
6448 ;; [vorrq_m_s, vorrq_m_u])
6449 ;;
6450 (define_insn "mve_vorrq_m_<supf><mode>"
6451   [
6452    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6453         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6454                        (match_operand:MVE_2 2 "s_register_operand" "w")
6455                        (match_operand:MVE_2 3 "s_register_operand" "w")
6456                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6457          VORRQ_M))
6458   ]
6459   "TARGET_HAVE_MVE"
6460   "vpst\;vorrt %q0, %q2, %q3"
6461   [(set_attr "type" "mve_move")
6462    (set_attr "length""8")])
6463
6464 ;;
6465 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
6466 ;;
6467 (define_insn "mve_vqaddq_m_n_<supf><mode>"
6468   [
6469    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6470         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6471                        (match_operand:MVE_2 2 "s_register_operand" "w")
6472                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6473                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6474          VQADDQ_M_N))
6475   ]
6476   "TARGET_HAVE_MVE"
6477   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6478   [(set_attr "type" "mve_move")
6479    (set_attr "length""8")])
6480
6481 ;;
6482 ;; [vqaddq_m_u, vqaddq_m_s])
6483 ;;
6484 (define_insn "mve_vqaddq_m_<supf><mode>"
6485   [
6486    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6487         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6488                        (match_operand:MVE_2 2 "s_register_operand" "w")
6489                        (match_operand:MVE_2 3 "s_register_operand" "w")
6490                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6491          VQADDQ_M))
6492   ]
6493   "TARGET_HAVE_MVE"
6494   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6495   [(set_attr "type" "mve_move")
6496    (set_attr "length""8")])
6497
6498 ;;
6499 ;; [vqdmlahq_m_n_s])
6500 ;;
6501 (define_insn "mve_vqdmlahq_m_n_s<mode>"
6502   [
6503    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6504         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6505                        (match_operand:MVE_2 2 "s_register_operand" "w")
6506                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6507                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6508          VQDMLAHQ_M_N_S))
6509   ]
6510   "TARGET_HAVE_MVE"
6511   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6512   [(set_attr "type" "mve_move")
6513    (set_attr "length""8")])
6514
6515 ;;
6516 ;; [vqrdmlahq_m_n_s])
6517 ;;
6518 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6519   [
6520    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6521         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6522                        (match_operand:MVE_2 2 "s_register_operand" "w")
6523                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6524                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6525          VQRDMLAHQ_M_N_S))
6526   ]
6527   "TARGET_HAVE_MVE"
6528   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6529   [(set_attr "type" "mve_move")
6530    (set_attr "length""8")])
6531
6532 ;;
6533 ;; [vqrdmlashq_m_n_s])
6534 ;;
6535 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6536   [
6537    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6538         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6539                        (match_operand:MVE_2 2 "s_register_operand" "w")
6540                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6541                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6542          VQRDMLASHQ_M_N_S))
6543   ]
6544   "TARGET_HAVE_MVE"
6545   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6546   [(set_attr "type" "mve_move")
6547    (set_attr "length""8")])
6548
6549 ;;
6550 ;; [vqrshlq_m_u, vqrshlq_m_s])
6551 ;;
6552 (define_insn "mve_vqrshlq_m_<supf><mode>"
6553   [
6554    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6555         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6556                        (match_operand:MVE_2 2 "s_register_operand" "w")
6557                        (match_operand:MVE_2 3 "s_register_operand" "w")
6558                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6559          VQRSHLQ_M))
6560   ]
6561   "TARGET_HAVE_MVE"
6562   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6563   [(set_attr "type" "mve_move")
6564    (set_attr "length""8")])
6565
6566 ;;
6567 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6568 ;;
6569 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6570   [
6571    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6572         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6573                        (match_operand:MVE_2 2 "s_register_operand" "w")
6574                        (match_operand:SI 3 "immediate_operand" "i")
6575                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6576          VQSHLQ_M_N))
6577   ]
6578   "TARGET_HAVE_MVE"
6579   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6580   [(set_attr "type" "mve_move")
6581    (set_attr "length""8")])
6582
6583 ;;
6584 ;; [vqshlq_m_u, vqshlq_m_s])
6585 ;;
6586 (define_insn "mve_vqshlq_m_<supf><mode>"
6587   [
6588    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6589         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6590                        (match_operand:MVE_2 2 "s_register_operand" "w")
6591                        (match_operand:MVE_2 3 "s_register_operand" "w")
6592                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6593          VQSHLQ_M))
6594   ]
6595   "TARGET_HAVE_MVE"
6596   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6597   [(set_attr "type" "mve_move")
6598    (set_attr "length""8")])
6599
6600 ;;
6601 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6602 ;;
6603 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6604   [
6605    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6606         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6607                        (match_operand:MVE_2 2 "s_register_operand" "w")
6608                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6609                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6610          VQSUBQ_M_N))
6611   ]
6612   "TARGET_HAVE_MVE"
6613   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6614   [(set_attr "type" "mve_move")
6615    (set_attr "length""8")])
6616
6617 ;;
6618 ;; [vqsubq_m_u, vqsubq_m_s])
6619 ;;
6620 (define_insn "mve_vqsubq_m_<supf><mode>"
6621   [
6622    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6623         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6624                        (match_operand:MVE_2 2 "s_register_operand" "w")
6625                        (match_operand:MVE_2 3 "s_register_operand" "w")
6626                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6627          VQSUBQ_M))
6628   ]
6629   "TARGET_HAVE_MVE"
6630   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6631   [(set_attr "type" "mve_move")
6632    (set_attr "length""8")])
6633
6634 ;;
6635 ;; [vrhaddq_m_u, vrhaddq_m_s])
6636 ;;
6637 (define_insn "mve_vrhaddq_m_<supf><mode>"
6638   [
6639    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6640         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6641                        (match_operand:MVE_2 2 "s_register_operand" "w")
6642                        (match_operand:MVE_2 3 "s_register_operand" "w")
6643                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6644          VRHADDQ_M))
6645   ]
6646   "TARGET_HAVE_MVE"
6647   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6648   [(set_attr "type" "mve_move")
6649    (set_attr "length""8")])
6650
6651 ;;
6652 ;; [vrmulhq_m_u, vrmulhq_m_s])
6653 ;;
6654 (define_insn "mve_vrmulhq_m_<supf><mode>"
6655   [
6656    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6657         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6658                        (match_operand:MVE_2 2 "s_register_operand" "w")
6659                        (match_operand:MVE_2 3 "s_register_operand" "w")
6660                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6661          VRMULHQ_M))
6662   ]
6663   "TARGET_HAVE_MVE"
6664   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6665   [(set_attr "type" "mve_move")
6666    (set_attr "length""8")])
6667
6668 ;;
6669 ;; [vrshlq_m_s, vrshlq_m_u])
6670 ;;
6671 (define_insn "mve_vrshlq_m_<supf><mode>"
6672   [
6673    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6674         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6675                        (match_operand:MVE_2 2 "s_register_operand" "w")
6676                        (match_operand:MVE_2 3 "s_register_operand" "w")
6677                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6678          VRSHLQ_M))
6679   ]
6680   "TARGET_HAVE_MVE"
6681   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6682   [(set_attr "type" "mve_move")
6683    (set_attr "length""8")])
6684
6685 ;;
6686 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6687 ;;
6688 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6689   [
6690    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6691         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6692                        (match_operand:MVE_2 2 "s_register_operand" "w")
6693                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6694                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6695          VRSHRQ_M_N))
6696   ]
6697   "TARGET_HAVE_MVE"
6698   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6699   [(set_attr "type" "mve_move")
6700    (set_attr "length""8")])
6701
6702 ;;
6703 ;; [vshlq_m_n_s, vshlq_m_n_u])
6704 ;;
6705 (define_insn "mve_vshlq_m_n_<supf><mode>"
6706   [
6707    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6708         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6709                        (match_operand:MVE_2 2 "s_register_operand" "w")
6710                        (match_operand:SI 3 "immediate_operand" "i")
6711                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6712          VSHLQ_M_N))
6713   ]
6714   "TARGET_HAVE_MVE"
6715   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6716   [(set_attr "type" "mve_move")
6717    (set_attr "length""8")])
6718
6719 ;;
6720 ;; [vshrq_m_n_s, vshrq_m_n_u])
6721 ;;
6722 (define_insn "mve_vshrq_m_n_<supf><mode>"
6723   [
6724    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6725         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6726                        (match_operand:MVE_2 2 "s_register_operand" "w")
6727                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6728                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6729          VSHRQ_M_N))
6730   ]
6731   "TARGET_HAVE_MVE"
6732   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6733   [(set_attr "type" "mve_move")
6734    (set_attr "length""8")])
6735
6736 ;;
6737 ;; [vsliq_m_n_u, vsliq_m_n_s])
6738 ;;
6739 (define_insn "mve_vsliq_m_n_<supf><mode>"
6740    [
6741    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6742        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6743                        (match_operand:MVE_2 2 "s_register_operand" "w")
6744                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6745                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6746          VSLIQ_M_N))
6747   ]
6748   "TARGET_HAVE_MVE"
6749   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6750   [(set_attr "type" "mve_move")
6751    (set_attr "length""8")])
6752
6753 ;;
6754 ;; [vsubq_m_n_s, vsubq_m_n_u])
6755 ;;
6756 (define_insn "mve_vsubq_m_n_<supf><mode>"
6757   [
6758    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6759         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6760                        (match_operand:MVE_2 2 "s_register_operand" "w")
6761                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6762                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6763          VSUBQ_M_N))
6764   ]
6765   "TARGET_HAVE_MVE"
6766   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6767   [(set_attr "type" "mve_move")
6768    (set_attr "length""8")])
6769
6770 ;;
6771 ;; [vhcaddq_rot270_m_s])
6772 ;;
6773 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6774   [
6775    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6776         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6777                        (match_operand:MVE_2 2 "s_register_operand" "w")
6778                        (match_operand:MVE_2 3 "s_register_operand" "w")
6779                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6780          VHCADDQ_ROT270_M_S))
6781   ]
6782   "TARGET_HAVE_MVE"
6783   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6784   [(set_attr "type" "mve_move")
6785    (set_attr "length""8")])
6786
6787 ;;
6788 ;; [vhcaddq_rot90_m_s])
6789 ;;
6790 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6791   [
6792    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6793         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6794                        (match_operand:MVE_2 2 "s_register_operand" "w")
6795                        (match_operand:MVE_2 3 "s_register_operand" "w")
6796                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6797          VHCADDQ_ROT90_M_S))
6798   ]
6799   "TARGET_HAVE_MVE"
6800   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6801   [(set_attr "type" "mve_move")
6802    (set_attr "length""8")])
6803
6804 ;;
6805 ;; [vmladavaxq_p_s])
6806 ;;
6807 (define_insn "mve_vmladavaxq_p_s<mode>"
6808   [
6809    (set (match_operand:SI 0 "s_register_operand" "=e")
6810         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6811                        (match_operand:MVE_2 2 "s_register_operand" "w")
6812                        (match_operand:MVE_2 3 "s_register_operand" "w")
6813                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6814          VMLADAVAXQ_P_S))
6815   ]
6816   "TARGET_HAVE_MVE"
6817   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6818   [(set_attr "type" "mve_move")
6819    (set_attr "length""8")])
6820
6821 ;;
6822 ;; [vmlsdavaq_p_s])
6823 ;;
6824 (define_insn "mve_vmlsdavaq_p_s<mode>"
6825   [
6826    (set (match_operand:SI 0 "s_register_operand" "=e")
6827         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6828                        (match_operand:MVE_2 2 "s_register_operand" "w")
6829                        (match_operand:MVE_2 3 "s_register_operand" "w")
6830                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6831          VMLSDAVAQ_P_S))
6832   ]
6833   "TARGET_HAVE_MVE"
6834   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6835   [(set_attr "type" "mve_move")
6836    (set_attr "length""8")])
6837
6838 ;;
6839 ;; [vmlsdavaxq_p_s])
6840 ;;
6841 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6842   [
6843    (set (match_operand:SI 0 "s_register_operand" "=e")
6844         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6845                        (match_operand:MVE_2 2 "s_register_operand" "w")
6846                        (match_operand:MVE_2 3 "s_register_operand" "w")
6847                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6848          VMLSDAVAXQ_P_S))
6849   ]
6850   "TARGET_HAVE_MVE"
6851   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6852   [(set_attr "type" "mve_move")
6853    (set_attr "length""8")])
6854
6855 ;;
6856 ;; [vqdmladhq_m_s])
6857 ;;
6858 (define_insn "mve_vqdmladhq_m_s<mode>"
6859   [
6860    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6861         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6862                        (match_operand:MVE_2 2 "s_register_operand" "w")
6863                        (match_operand:MVE_2 3 "s_register_operand" "w")
6864                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6865          VQDMLADHQ_M_S))
6866   ]
6867   "TARGET_HAVE_MVE"
6868   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6869   [(set_attr "type" "mve_move")
6870    (set_attr "length""8")])
6871
6872 ;;
6873 ;; [vqdmladhxq_m_s])
6874 ;;
6875 (define_insn "mve_vqdmladhxq_m_s<mode>"
6876   [
6877    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6878         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6879                        (match_operand:MVE_2 2 "s_register_operand" "w")
6880                        (match_operand:MVE_2 3 "s_register_operand" "w")
6881                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6882          VQDMLADHXQ_M_S))
6883   ]
6884   "TARGET_HAVE_MVE"
6885   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6886   [(set_attr "type" "mve_move")
6887    (set_attr "length""8")])
6888
6889 ;;
6890 ;; [vqdmlsdhq_m_s])
6891 ;;
6892 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6893   [
6894    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6895         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6896                        (match_operand:MVE_2 2 "s_register_operand" "w")
6897                        (match_operand:MVE_2 3 "s_register_operand" "w")
6898                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6899          VQDMLSDHQ_M_S))
6900   ]
6901   "TARGET_HAVE_MVE"
6902   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6903   [(set_attr "type" "mve_move")
6904    (set_attr "length""8")])
6905
6906 ;;
6907 ;; [vqdmlsdhxq_m_s])
6908 ;;
6909 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6910   [
6911    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6912         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6913                        (match_operand:MVE_2 2 "s_register_operand" "w")
6914                        (match_operand:MVE_2 3 "s_register_operand" "w")
6915                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6916          VQDMLSDHXQ_M_S))
6917   ]
6918   "TARGET_HAVE_MVE"
6919   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6920   [(set_attr "type" "mve_move")
6921    (set_attr "length""8")])
6922
6923 ;;
6924 ;; [vqdmulhq_m_n_s])
6925 ;;
6926 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6927   [
6928    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6929         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6930                        (match_operand:MVE_2 2 "s_register_operand" "w")
6931                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6932                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6933          VQDMULHQ_M_N_S))
6934   ]
6935   "TARGET_HAVE_MVE"
6936   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6937   [(set_attr "type" "mve_move")
6938    (set_attr "length""8")])
6939
6940 ;;
6941 ;; [vqdmulhq_m_s])
6942 ;;
6943 (define_insn "mve_vqdmulhq_m_s<mode>"
6944   [
6945    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6946         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6947                        (match_operand:MVE_2 2 "s_register_operand" "w")
6948                        (match_operand:MVE_2 3 "s_register_operand" "w")
6949                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6950          VQDMULHQ_M_S))
6951   ]
6952   "TARGET_HAVE_MVE"
6953   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6954   [(set_attr "type" "mve_move")
6955    (set_attr "length""8")])
6956
6957 ;;
6958 ;; [vqrdmladhq_m_s])
6959 ;;
6960 (define_insn "mve_vqrdmladhq_m_s<mode>"
6961   [
6962    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6963         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6964                        (match_operand:MVE_2 2 "s_register_operand" "w")
6965                        (match_operand:MVE_2 3 "s_register_operand" "w")
6966                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6967          VQRDMLADHQ_M_S))
6968   ]
6969   "TARGET_HAVE_MVE"
6970   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6971   [(set_attr "type" "mve_move")
6972    (set_attr "length""8")])
6973
6974 ;;
6975 ;; [vqrdmladhxq_m_s])
6976 ;;
6977 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6978   [
6979    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6980         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6981                        (match_operand:MVE_2 2 "s_register_operand" "w")
6982                        (match_operand:MVE_2 3 "s_register_operand" "w")
6983                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6984          VQRDMLADHXQ_M_S))
6985   ]
6986   "TARGET_HAVE_MVE"
6987   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6988   [(set_attr "type" "mve_move")
6989    (set_attr "length""8")])
6990
6991 ;;
6992 ;; [vqrdmlsdhq_m_s])
6993 ;;
6994 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6995   [
6996    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6997         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6998                        (match_operand:MVE_2 2 "s_register_operand" "w")
6999                        (match_operand:MVE_2 3 "s_register_operand" "w")
7000                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7001          VQRDMLSDHQ_M_S))
7002   ]
7003   "TARGET_HAVE_MVE"
7004   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7005   [(set_attr "type" "mve_move")
7006    (set_attr "length""8")])
7007
7008 ;;
7009 ;; [vqrdmlsdhxq_m_s])
7010 ;;
7011 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
7012   [
7013    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7014         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7015                        (match_operand:MVE_2 2 "s_register_operand" "w")
7016                        (match_operand:MVE_2 3 "s_register_operand" "w")
7017                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7018          VQRDMLSDHXQ_M_S))
7019   ]
7020   "TARGET_HAVE_MVE"
7021   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7022   [(set_attr "type" "mve_move")
7023    (set_attr "length""8")])
7024
7025 ;;
7026 ;; [vqrdmulhq_m_n_s])
7027 ;;
7028 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
7029   [
7030    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7031         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7032                        (match_operand:MVE_2 2 "s_register_operand" "w")
7033                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7034                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7035          VQRDMULHQ_M_N_S))
7036   ]
7037   "TARGET_HAVE_MVE"
7038   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
7039   [(set_attr "type" "mve_move")
7040    (set_attr "length""8")])
7041
7042 ;;
7043 ;; [vqrdmulhq_m_s])
7044 ;;
7045 (define_insn "mve_vqrdmulhq_m_s<mode>"
7046   [
7047    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7048         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7049                        (match_operand:MVE_2 2 "s_register_operand" "w")
7050                        (match_operand:MVE_2 3 "s_register_operand" "w")
7051                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7052          VQRDMULHQ_M_S))
7053   ]
7054   "TARGET_HAVE_MVE"
7055   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7056   [(set_attr "type" "mve_move")
7057    (set_attr "length""8")])
7058
7059 ;;
7060 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
7061 ;;
7062 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
7063   [
7064    (set (match_operand:DI 0 "s_register_operand" "=r")
7065         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7066                        (match_operand:MVE_5 2 "s_register_operand" "w")
7067                        (match_operand:MVE_5 3 "s_register_operand" "w")
7068                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7069          VMLALDAVAQ_P))
7070   ]
7071   "TARGET_HAVE_MVE"
7072   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7073   [(set_attr "type" "mve_move")
7074    (set_attr "length""8")])
7075
7076 ;;
7077 ;; [vmlaldavaxq_p_u, vmlaldavaxq_p_s])
7078 ;;
7079 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
7080   [
7081    (set (match_operand:DI 0 "s_register_operand" "=r")
7082         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7083                        (match_operand:MVE_5 2 "s_register_operand" "w")
7084                        (match_operand:MVE_5 3 "s_register_operand" "w")
7085                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7086          VMLALDAVAXQ_P))
7087   ]
7088   "TARGET_HAVE_MVE"
7089   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7090   [(set_attr "type" "mve_move")
7091    (set_attr "length""8")])
7092
7093 ;;
7094 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
7095 ;;
7096 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
7097   [
7098    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7099         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7100                        (match_operand:MVE_5 2 "s_register_operand" "w")
7101                        (match_operand:SI 3 "mve_imm_8" "Rb")
7102                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7103          VQRSHRNBQ_M_N))
7104   ]
7105   "TARGET_HAVE_MVE"
7106   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7107   [(set_attr "type" "mve_move")
7108    (set_attr "length""8")])
7109
7110 ;;
7111 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
7112 ;;
7113 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
7114   [
7115    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7116         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7117                        (match_operand:MVE_5 2 "s_register_operand" "w")
7118                        (match_operand:SI 3 "mve_imm_8" "Rb")
7119                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7120          VQRSHRNTQ_M_N))
7121   ]
7122   "TARGET_HAVE_MVE"
7123   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7124   [(set_attr "type" "mve_move")
7125    (set_attr "length""8")])
7126
7127 ;;
7128 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
7129 ;;
7130 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
7131   [
7132    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7133         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7134                        (match_operand:MVE_5 2 "s_register_operand" "w")
7135                        (match_operand:SI 3 "<MVE_pred1>" "<MVE_constraint1>")
7136                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7137          VQSHRNBQ_M_N))
7138   ]
7139   "TARGET_HAVE_MVE && arm_mve_immediate_check (operands[3], <MODE>mode, 0)"
7140   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7141   [(set_attr "type" "mve_move")
7142    (set_attr "length""8")])
7143
7144 ;;
7145 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
7146 ;;
7147 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
7148   [
7149    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7150         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7151                        (match_operand:MVE_5 2 "s_register_operand" "w")
7152                        (match_operand:SI 3 "mve_imm_8" "Rb")
7153                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7154          VQSHRNTQ_M_N))
7155   ]
7156   "TARGET_HAVE_MVE"
7157   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7158   [(set_attr "type" "mve_move")
7159    (set_attr "length""8")])
7160
7161 ;;
7162 ;; [vrmlaldavhaq_p_s])
7163 ;;
7164 (define_insn "mve_vrmlaldavhaq_p_sv4si"
7165   [
7166    (set (match_operand:DI 0 "s_register_operand" "=r")
7167         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7168                        (match_operand:V4SI 2 "s_register_operand" "w")
7169                        (match_operand:V4SI 3 "s_register_operand" "w")
7170                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7171          VRMLALDAVHAQ_P_S))
7172   ]
7173   "TARGET_HAVE_MVE"
7174   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
7175   [(set_attr "type" "mve_move")
7176    (set_attr "length""8")])
7177
7178 ;;
7179 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
7180 ;;
7181 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
7182   [
7183    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7184         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7185                        (match_operand:MVE_5 2 "s_register_operand" "w")
7186                        (match_operand:SI 3 "mve_imm_8" "Rb")
7187                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7188          VRSHRNBQ_M_N))
7189   ]
7190   "TARGET_HAVE_MVE"
7191   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7192   [(set_attr "type" "mve_move")
7193    (set_attr "length""8")])
7194
7195 ;;
7196 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
7197 ;;
7198 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
7199   [
7200    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7201         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7202                        (match_operand:MVE_5 2 "s_register_operand" "w")
7203                        (match_operand:SI 3 "mve_imm_8" "Rb")
7204                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7205          VRSHRNTQ_M_N))
7206   ]
7207   "TARGET_HAVE_MVE"
7208   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7209   [(set_attr "type" "mve_move")
7210    (set_attr "length""8")])
7211
7212 ;;
7213 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
7214 ;;
7215 (define_insn "mve_vshllbq_m_n_<supf><mode>"
7216   [
7217    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7218         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7219                        (match_operand:MVE_3 2 "s_register_operand" "w")
7220                        (match_operand:SI 3 "immediate_operand" "i")
7221                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7222          VSHLLBQ_M_N))
7223   ]
7224   "TARGET_HAVE_MVE"
7225   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7226   [(set_attr "type" "mve_move")
7227    (set_attr "length""8")])
7228
7229 ;;
7230 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
7231 ;;
7232 (define_insn "mve_vshlltq_m_n_<supf><mode>"
7233   [
7234    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7235         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7236                        (match_operand:MVE_3 2 "s_register_operand" "w")
7237                        (match_operand:SI 3 "immediate_operand" "i")
7238                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7239          VSHLLTQ_M_N))
7240   ]
7241   "TARGET_HAVE_MVE"
7242   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7243   [(set_attr "type" "mve_move")
7244    (set_attr "length""8")])
7245
7246 ;;
7247 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
7248 ;;
7249 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
7250   [
7251    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7252         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7253                        (match_operand:MVE_5 2 "s_register_operand" "w")
7254                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7255                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7256          VSHRNBQ_M_N))
7257   ]
7258   "TARGET_HAVE_MVE"
7259   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7260   [(set_attr "type" "mve_move")
7261    (set_attr "length""8")])
7262
7263 ;;
7264 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
7265 ;;
7266 (define_insn "mve_vshrntq_m_n_<supf><mode>"
7267   [
7268    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7269         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7270                        (match_operand:MVE_5 2 "s_register_operand" "w")
7271                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7272                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7273          VSHRNTQ_M_N))
7274   ]
7275   "TARGET_HAVE_MVE"
7276   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7277   [(set_attr "type" "mve_move")
7278    (set_attr "length""8")])
7279
7280 ;;
7281 ;; [vmlsldavaq_p_s])
7282 ;;
7283 (define_insn "mve_vmlsldavaq_p_s<mode>"
7284   [
7285    (set (match_operand:DI 0 "s_register_operand" "=r")
7286         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7287                        (match_operand:MVE_5 2 "s_register_operand" "w")
7288                        (match_operand:MVE_5 3 "s_register_operand" "w")
7289                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7290          VMLSLDAVAQ_P_S))
7291   ]
7292   "TARGET_HAVE_MVE"
7293   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7294   [(set_attr "type" "mve_move")
7295    (set_attr "length""8")])
7296
7297 ;;
7298 ;; [vmlsldavaxq_p_s])
7299 ;;
7300 (define_insn "mve_vmlsldavaxq_p_s<mode>"
7301   [
7302    (set (match_operand:DI 0 "s_register_operand" "=r")
7303         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7304                        (match_operand:MVE_5 2 "s_register_operand" "w")
7305                        (match_operand:MVE_5 3 "s_register_operand" "w")
7306                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7307          VMLSLDAVAXQ_P_S))
7308   ]
7309   "TARGET_HAVE_MVE"
7310   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7311   [(set_attr "type" "mve_move")
7312    (set_attr "length""8")])
7313
7314 ;;
7315 ;; [vmullbq_poly_m_p])
7316 ;;
7317 (define_insn "mve_vmullbq_poly_m_p<mode>"
7318   [
7319    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7320         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7321                        (match_operand:MVE_3 2 "s_register_operand" "w")
7322                        (match_operand:MVE_3 3 "s_register_operand" "w")
7323                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7324          VMULLBQ_POLY_M_P))
7325   ]
7326   "TARGET_HAVE_MVE"
7327   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7328   [(set_attr "type" "mve_move")
7329    (set_attr "length""8")])
7330
7331 ;;
7332 ;; [vmulltq_poly_m_p])
7333 ;;
7334 (define_insn "mve_vmulltq_poly_m_p<mode>"
7335   [
7336    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7337         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7338                        (match_operand:MVE_3 2 "s_register_operand" "w")
7339                        (match_operand:MVE_3 3 "s_register_operand" "w")
7340                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7341          VMULLTQ_POLY_M_P))
7342   ]
7343   "TARGET_HAVE_MVE"
7344   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7345   [(set_attr "type" "mve_move")
7346    (set_attr "length""8")])
7347
7348 ;;
7349 ;; [vqdmullbq_m_n_s])
7350 ;;
7351 (define_insn "mve_vqdmullbq_m_n_s<mode>"
7352   [
7353    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7354         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7355                        (match_operand:MVE_5 2 "s_register_operand" "w")
7356                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7357                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7358          VQDMULLBQ_M_N_S))
7359   ]
7360   "TARGET_HAVE_MVE"
7361   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7362   [(set_attr "type" "mve_move")
7363    (set_attr "length""8")])
7364
7365 ;;
7366 ;; [vqdmullbq_m_s])
7367 ;;
7368 (define_insn "mve_vqdmullbq_m_s<mode>"
7369   [
7370    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7371         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7372                        (match_operand:MVE_5 2 "s_register_operand" "w")
7373                        (match_operand:MVE_5 3 "s_register_operand" "w")
7374                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7375          VQDMULLBQ_M_S))
7376   ]
7377   "TARGET_HAVE_MVE"
7378   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7379   [(set_attr "type" "mve_move")
7380    (set_attr "length""8")])
7381
7382 ;;
7383 ;; [vqdmulltq_m_n_s])
7384 ;;
7385 (define_insn "mve_vqdmulltq_m_n_s<mode>"
7386   [
7387    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7388         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7389                        (match_operand:MVE_5 2 "s_register_operand" "w")
7390                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7391                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7392          VQDMULLTQ_M_N_S))
7393   ]
7394   "TARGET_HAVE_MVE"
7395   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
7396   [(set_attr "type" "mve_move")
7397    (set_attr "length""8")])
7398
7399 ;;
7400 ;; [vqdmulltq_m_s])
7401 ;;
7402 (define_insn "mve_vqdmulltq_m_s<mode>"
7403   [
7404    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7405         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7406                        (match_operand:MVE_5 2 "s_register_operand" "w")
7407                        (match_operand:MVE_5 3 "s_register_operand" "w")
7408                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7409          VQDMULLTQ_M_S))
7410   ]
7411   "TARGET_HAVE_MVE"
7412   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7413   [(set_attr "type" "mve_move")
7414    (set_attr "length""8")])
7415
7416 ;;
7417 ;; [vqrshrunbq_m_n_s])
7418 ;;
7419 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
7420   [
7421    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7422         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7423                        (match_operand:MVE_5 2 "s_register_operand" "w")
7424                        (match_operand:SI 3 "mve_imm_8" "Rb")
7425                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7426          VQRSHRUNBQ_M_N_S))
7427   ]
7428   "TARGET_HAVE_MVE"
7429   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7430   [(set_attr "type" "mve_move")
7431    (set_attr "length""8")])
7432
7433 ;;
7434 ;; [vqrshruntq_m_n_s])
7435 ;;
7436 (define_insn "mve_vqrshruntq_m_n_s<mode>"
7437   [
7438    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7439         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7440                        (match_operand:MVE_5 2 "s_register_operand" "w")
7441                        (match_operand:SI 3 "mve_imm_8" "Rb")
7442                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7443          VQRSHRUNTQ_M_N_S))
7444   ]
7445   "TARGET_HAVE_MVE"
7446   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7447   [(set_attr "type" "mve_move")
7448    (set_attr "length""8")])
7449
7450 ;;
7451 ;; [vqshrunbq_m_n_s])
7452 ;;
7453 (define_insn "mve_vqshrunbq_m_n_s<mode>"
7454   [
7455    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7456         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7457                        (match_operand:MVE_5 2 "s_register_operand" "w")
7458                        (match_operand:SI 3 "mve_imm_8" "Rb")
7459                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7460          VQSHRUNBQ_M_N_S))
7461   ]
7462   "TARGET_HAVE_MVE"
7463   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7464   [(set_attr "type" "mve_move")
7465    (set_attr "length""8")])
7466
7467 ;;
7468 ;; [vqshruntq_m_n_s])
7469 ;;
7470 (define_insn "mve_vqshruntq_m_n_s<mode>"
7471   [
7472    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7473         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7474                        (match_operand:MVE_5 2 "s_register_operand" "w")
7475                        (match_operand:SI 3 "mve_imm_8" "Rb")
7476                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7477          VQSHRUNTQ_M_N_S))
7478   ]
7479   "TARGET_HAVE_MVE"
7480   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7481   [(set_attr "type" "mve_move")
7482    (set_attr "length""8")])
7483
7484 ;;
7485 ;; [vrmlaldavhaq_p_u])
7486 ;;
7487 (define_insn "mve_vrmlaldavhaq_p_uv4si"
7488   [
7489    (set (match_operand:DI 0 "s_register_operand" "=r")
7490         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7491                        (match_operand:V4SI 2 "s_register_operand" "w")
7492                        (match_operand:V4SI 3 "s_register_operand" "w")
7493                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7494          VRMLALDAVHAQ_P_U))
7495   ]
7496   "TARGET_HAVE_MVE"
7497   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7498   [(set_attr "type" "mve_move")
7499    (set_attr "length""8")])
7500
7501 ;;
7502 ;; [vrmlaldavhaxq_p_s])
7503 ;;
7504 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7505   [
7506    (set (match_operand:DI 0 "s_register_operand" "=r")
7507         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7508                        (match_operand:V4SI 2 "s_register_operand" "w")
7509                        (match_operand:V4SI 3 "s_register_operand" "w")
7510                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7511          VRMLALDAVHAXQ_P_S))
7512   ]
7513   "TARGET_HAVE_MVE"
7514   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7515   [(set_attr "type" "mve_move")
7516    (set_attr "length""8")])
7517
7518 ;;
7519 ;; [vrmlsldavhaq_p_s])
7520 ;;
7521 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7522   [
7523    (set (match_operand:DI 0 "s_register_operand" "=r")
7524         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7525                        (match_operand:V4SI 2 "s_register_operand" "w")
7526                        (match_operand:V4SI 3 "s_register_operand" "w")
7527                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7528          VRMLSLDAVHAQ_P_S))
7529   ]
7530   "TARGET_HAVE_MVE"
7531   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7532   [(set_attr "type" "mve_move")
7533    (set_attr "length""8")])
7534
7535 ;;
7536 ;; [vrmlsldavhaxq_p_s])
7537 ;;
7538 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7539   [
7540    (set (match_operand:DI 0 "s_register_operand" "=r")
7541         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7542                        (match_operand:V4SI 2 "s_register_operand" "w")
7543                        (match_operand:V4SI 3 "s_register_operand" "w")
7544                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7545          VRMLSLDAVHAXQ_P_S))
7546   ]
7547   "TARGET_HAVE_MVE"
7548   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7549   [(set_attr "type" "mve_move")
7550    (set_attr "length""8")])
7551 ;;
7552 ;; [vabdq_m_f])
7553 ;;
7554 (define_insn "mve_vabdq_m_f<mode>"
7555   [
7556    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7557         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7558                        (match_operand:MVE_0 2 "s_register_operand" "w")
7559                        (match_operand:MVE_0 3 "s_register_operand" "w")
7560                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7561          VABDQ_M_F))
7562   ]
7563   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7564   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7565   [(set_attr "type" "mve_move")
7566    (set_attr "length""8")])
7567
7568 ;;
7569 ;; [vaddq_m_f])
7570 ;;
7571 (define_insn "mve_vaddq_m_f<mode>"
7572   [
7573    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7574         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7575                        (match_operand:MVE_0 2 "s_register_operand" "w")
7576                        (match_operand:MVE_0 3 "s_register_operand" "w")
7577                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7578          VADDQ_M_F))
7579   ]
7580   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7581   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7582   [(set_attr "type" "mve_move")
7583    (set_attr "length""8")])
7584
7585 ;;
7586 ;; [vaddq_m_n_f])
7587 ;;
7588 (define_insn "mve_vaddq_m_n_f<mode>"
7589   [
7590    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7591         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7592                        (match_operand:MVE_0 2 "s_register_operand" "w")
7593                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7594                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7595          VADDQ_M_N_F))
7596   ]
7597   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7598   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7599   [(set_attr "type" "mve_move")
7600    (set_attr "length""8")])
7601
7602 ;;
7603 ;; [vandq_m_f])
7604 ;;
7605 (define_insn "mve_vandq_m_f<mode>"
7606   [
7607    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7608         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7609                        (match_operand:MVE_0 2 "s_register_operand" "w")
7610                        (match_operand:MVE_0 3 "s_register_operand" "w")
7611                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7612          VANDQ_M_F))
7613   ]
7614   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7615   "vpst\;vandt %q0, %q2, %q3"
7616   [(set_attr "type" "mve_move")
7617    (set_attr "length""8")])
7618
7619 ;;
7620 ;; [vbicq_m_f])
7621 ;;
7622 (define_insn "mve_vbicq_m_f<mode>"
7623   [
7624    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7625         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7626                        (match_operand:MVE_0 2 "s_register_operand" "w")
7627                        (match_operand:MVE_0 3 "s_register_operand" "w")
7628                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7629          VBICQ_M_F))
7630   ]
7631   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7632   "vpst\;vbict %q0, %q2, %q3"
7633   [(set_attr "type" "mve_move")
7634    (set_attr "length""8")])
7635
7636 ;;
7637 ;; [vbrsrq_m_n_f])
7638 ;;
7639 (define_insn "mve_vbrsrq_m_n_f<mode>"
7640   [
7641    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7642         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7643                        (match_operand:MVE_0 2 "s_register_operand" "w")
7644                        (match_operand:SI 3 "s_register_operand" "r")
7645                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7646          VBRSRQ_M_N_F))
7647   ]
7648   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7649   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7650   [(set_attr "type" "mve_move")
7651    (set_attr "length""8")])
7652
7653 ;;
7654 ;; [vcaddq_rot270_m_f])
7655 ;;
7656 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7657   [
7658    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7659         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7660                        (match_operand:MVE_0 2 "s_register_operand" "w")
7661                        (match_operand:MVE_0 3 "s_register_operand" "w")
7662                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7663          VCADDQ_ROT270_M_F))
7664   ]
7665   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7666   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7667   [(set_attr "type" "mve_move")
7668    (set_attr "length""8")])
7669
7670 ;;
7671 ;; [vcaddq_rot90_m_f])
7672 ;;
7673 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7674   [
7675    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7676         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7677                        (match_operand:MVE_0 2 "s_register_operand" "w")
7678                        (match_operand:MVE_0 3 "s_register_operand" "w")
7679                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7680          VCADDQ_ROT90_M_F))
7681   ]
7682   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7683   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7684   [(set_attr "type" "mve_move")
7685    (set_attr "length""8")])
7686
7687 ;;
7688 ;; [vcmlaq_m_f])
7689 ;;
7690 (define_insn "mve_vcmlaq_m_f<mode>"
7691   [
7692    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7693         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7694                        (match_operand:MVE_0 2 "s_register_operand" "w")
7695                        (match_operand:MVE_0 3 "s_register_operand" "w")
7696                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7697          VCMLAQ_M_F))
7698   ]
7699   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7700   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7701   [(set_attr "type" "mve_move")
7702    (set_attr "length""8")])
7703
7704 ;;
7705 ;; [vcmlaq_rot180_m_f])
7706 ;;
7707 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7708   [
7709    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7710         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7711                        (match_operand:MVE_0 2 "s_register_operand" "w")
7712                        (match_operand:MVE_0 3 "s_register_operand" "w")
7713                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7714          VCMLAQ_ROT180_M_F))
7715   ]
7716   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7717   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7718   [(set_attr "type" "mve_move")
7719    (set_attr "length""8")])
7720
7721 ;;
7722 ;; [vcmlaq_rot270_m_f])
7723 ;;
7724 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7725   [
7726    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7727         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7728                        (match_operand:MVE_0 2 "s_register_operand" "w")
7729                        (match_operand:MVE_0 3 "s_register_operand" "w")
7730                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7731          VCMLAQ_ROT270_M_F))
7732   ]
7733   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7734   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7735   [(set_attr "type" "mve_move")
7736    (set_attr "length""8")])
7737
7738 ;;
7739 ;; [vcmlaq_rot90_m_f])
7740 ;;
7741 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7742   [
7743    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7744         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7745                        (match_operand:MVE_0 2 "s_register_operand" "w")
7746                        (match_operand:MVE_0 3 "s_register_operand" "w")
7747                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7748          VCMLAQ_ROT90_M_F))
7749   ]
7750   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7751   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7752   [(set_attr "type" "mve_move")
7753    (set_attr "length""8")])
7754
7755 ;;
7756 ;; [vcmulq_m_f])
7757 ;;
7758 (define_insn "mve_vcmulq_m_f<mode>"
7759   [
7760    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7761         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7762                        (match_operand:MVE_0 2 "s_register_operand" "w")
7763                        (match_operand:MVE_0 3 "s_register_operand" "w")
7764                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7765          VCMULQ_M_F))
7766   ]
7767   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7768   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7769   [(set_attr "type" "mve_move")
7770    (set_attr "length""8")])
7771
7772 ;;
7773 ;; [vcmulq_rot180_m_f])
7774 ;;
7775 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7776   [
7777    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7778         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7779                        (match_operand:MVE_0 2 "s_register_operand" "w")
7780                        (match_operand:MVE_0 3 "s_register_operand" "w")
7781                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7782          VCMULQ_ROT180_M_F))
7783   ]
7784   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7785   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7786   [(set_attr "type" "mve_move")
7787    (set_attr "length""8")])
7788
7789 ;;
7790 ;; [vcmulq_rot270_m_f])
7791 ;;
7792 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7793   [
7794    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7795         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7796                        (match_operand:MVE_0 2 "s_register_operand" "w")
7797                        (match_operand:MVE_0 3 "s_register_operand" "w")
7798                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7799          VCMULQ_ROT270_M_F))
7800   ]
7801   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7802   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7803   [(set_attr "type" "mve_move")
7804    (set_attr "length""8")])
7805
7806 ;;
7807 ;; [vcmulq_rot90_m_f])
7808 ;;
7809 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7810   [
7811    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7812         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7813                        (match_operand:MVE_0 2 "s_register_operand" "w")
7814                        (match_operand:MVE_0 3 "s_register_operand" "w")
7815                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7816          VCMULQ_ROT90_M_F))
7817   ]
7818   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7819   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7820   [(set_attr "type" "mve_move")
7821    (set_attr "length""8")])
7822
7823 ;;
7824 ;; [veorq_m_f])
7825 ;;
7826 (define_insn "mve_veorq_m_f<mode>"
7827   [
7828    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7829         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7830                        (match_operand:MVE_0 2 "s_register_operand" "w")
7831                        (match_operand:MVE_0 3 "s_register_operand" "w")
7832                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7833          VEORQ_M_F))
7834   ]
7835   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7836   "vpst\;veort %q0, %q2, %q3"
7837   [(set_attr "type" "mve_move")
7838    (set_attr "length""8")])
7839
7840 ;;
7841 ;; [vfmaq_m_f])
7842 ;;
7843 (define_insn "mve_vfmaq_m_f<mode>"
7844   [
7845    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7846         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7847                        (match_operand:MVE_0 2 "s_register_operand" "w")
7848                        (match_operand:MVE_0 3 "s_register_operand" "w")
7849                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7850          VFMAQ_M_F))
7851   ]
7852   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7853   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7854   [(set_attr "type" "mve_move")
7855    (set_attr "length""8")])
7856
7857 ;;
7858 ;; [vfmaq_m_n_f])
7859 ;;
7860 (define_insn "mve_vfmaq_m_n_f<mode>"
7861   [
7862    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7863         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7864                        (match_operand:MVE_0 2 "s_register_operand" "w")
7865                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7866                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7867          VFMAQ_M_N_F))
7868   ]
7869   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7870   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7871   [(set_attr "type" "mve_move")
7872    (set_attr "length""8")])
7873
7874 ;;
7875 ;; [vfmasq_m_n_f])
7876 ;;
7877 (define_insn "mve_vfmasq_m_n_f<mode>"
7878   [
7879    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7880         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7881                        (match_operand:MVE_0 2 "s_register_operand" "w")
7882                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7883                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7884          VFMASQ_M_N_F))
7885   ]
7886   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7887   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7888   [(set_attr "type" "mve_move")
7889    (set_attr "length""8")])
7890
7891 ;;
7892 ;; [vfmsq_m_f])
7893 ;;
7894 (define_insn "mve_vfmsq_m_f<mode>"
7895   [
7896    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7897         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7898                        (match_operand:MVE_0 2 "s_register_operand" "w")
7899                        (match_operand:MVE_0 3 "s_register_operand" "w")
7900                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7901          VFMSQ_M_F))
7902   ]
7903   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7904   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7905   [(set_attr "type" "mve_move")
7906    (set_attr "length""8")])
7907
7908 ;;
7909 ;; [vmaxnmq_m_f])
7910 ;;
7911 (define_insn "mve_vmaxnmq_m_f<mode>"
7912   [
7913    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7914         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7915                        (match_operand:MVE_0 2 "s_register_operand" "w")
7916                        (match_operand:MVE_0 3 "s_register_operand" "w")
7917                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7918          VMAXNMQ_M_F))
7919   ]
7920   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7921   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7922   [(set_attr "type" "mve_move")
7923    (set_attr "length""8")])
7924
7925 ;;
7926 ;; [vminnmq_m_f])
7927 ;;
7928 (define_insn "mve_vminnmq_m_f<mode>"
7929   [
7930    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7931         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7932                        (match_operand:MVE_0 2 "s_register_operand" "w")
7933                        (match_operand:MVE_0 3 "s_register_operand" "w")
7934                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7935          VMINNMQ_M_F))
7936   ]
7937   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7938   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7939   [(set_attr "type" "mve_move")
7940    (set_attr "length""8")])
7941
7942 ;;
7943 ;; [vmulq_m_f])
7944 ;;
7945 (define_insn "mve_vmulq_m_f<mode>"
7946   [
7947    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7948         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7949                        (match_operand:MVE_0 2 "s_register_operand" "w")
7950                        (match_operand:MVE_0 3 "s_register_operand" "w")
7951                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7952          VMULQ_M_F))
7953   ]
7954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7955   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7956   [(set_attr "type" "mve_move")
7957    (set_attr "length""8")])
7958
7959 ;;
7960 ;; [vmulq_m_n_f])
7961 ;;
7962 (define_insn "mve_vmulq_m_n_f<mode>"
7963   [
7964    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7965         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7966                        (match_operand:MVE_0 2 "s_register_operand" "w")
7967                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7968                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7969          VMULQ_M_N_F))
7970   ]
7971   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7972   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7973   [(set_attr "type" "mve_move")
7974    (set_attr "length""8")])
7975
7976 ;;
7977 ;; [vornq_m_f])
7978 ;;
7979 (define_insn "mve_vornq_m_f<mode>"
7980   [
7981    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7982         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7983                        (match_operand:MVE_0 2 "s_register_operand" "w")
7984                        (match_operand:MVE_0 3 "s_register_operand" "w")
7985                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7986          VORNQ_M_F))
7987   ]
7988   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7989   "vpst\;vornt %q0, %q2, %q3"
7990   [(set_attr "type" "mve_move")
7991    (set_attr "length""8")])
7992
7993 ;;
7994 ;; [vorrq_m_f])
7995 ;;
7996 (define_insn "mve_vorrq_m_f<mode>"
7997   [
7998    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7999         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8000                        (match_operand:MVE_0 2 "s_register_operand" "w")
8001                        (match_operand:MVE_0 3 "s_register_operand" "w")
8002                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8003          VORRQ_M_F))
8004   ]
8005   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8006   "vpst\;vorrt %q0, %q2, %q3"
8007   [(set_attr "type" "mve_move")
8008    (set_attr "length""8")])
8009
8010 ;;
8011 ;; [vsubq_m_f])
8012 ;;
8013 (define_insn "mve_vsubq_m_f<mode>"
8014   [
8015    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8016         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8017                        (match_operand:MVE_0 2 "s_register_operand" "w")
8018                        (match_operand:MVE_0 3 "s_register_operand" "w")
8019                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8020          VSUBQ_M_F))
8021   ]
8022   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8023   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
8024   [(set_attr "type" "mve_move")
8025    (set_attr "length""8")])
8026
8027 ;;
8028 ;; [vsubq_m_n_f])
8029 ;;
8030 (define_insn "mve_vsubq_m_n_f<mode>"
8031   [
8032    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8033         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8034                        (match_operand:MVE_0 2 "s_register_operand" "w")
8035                        (match_operand:<V_elem> 3 "s_register_operand" "r")
8036                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8037          VSUBQ_M_N_F))
8038   ]
8039   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8040   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
8041   [(set_attr "type" "mve_move")
8042    (set_attr "length""8")])
8043
8044 ;;
8045 ;; [vstrbq_s vstrbq_u]
8046 ;;
8047 (define_insn "mve_vstrbq_<supf><mode>"
8048   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8049         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
8050          VSTRBQ))
8051   ]
8052   "TARGET_HAVE_MVE"
8053 {
8054    rtx ops[2];
8055    int regno = REGNO (operands[1]);
8056    ops[1] = gen_rtx_REG (TImode, regno);
8057    ops[0]  = operands[0];
8058    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
8059    return "";
8060 }
8061   [(set_attr "length" "4")])
8062
8063 ;;
8064 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
8065 ;;
8066 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>"
8067   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8068         (unspec:<MVE_B_ELEM>
8069                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8070                  (match_operand:MVE_2 2 "s_register_operand" "w")]
8071          VSTRBSOQ))
8072   ]
8073   "TARGET_HAVE_MVE"
8074 {
8075    rtx ops[3];
8076    ops[0] = operands[0];
8077    ops[1] = operands[1];
8078    ops[2] = operands[2];
8079    output_asm_insn("vstrb.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8080    return "";
8081 }
8082   [(set_attr "length" "4")])
8083
8084 ;;
8085 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
8086 ;;
8087 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
8088   [(set (mem:BLK (scratch))
8089         (unspec:BLK
8090                 [(match_operand:V4SI 0 "s_register_operand" "w")
8091                  (match_operand:SI 1 "immediate_operand" "i")
8092                  (match_operand:V4SI 2 "s_register_operand" "w")]
8093          VSTRWSBQ))
8094   ]
8095   "TARGET_HAVE_MVE"
8096 {
8097    rtx ops[3];
8098    ops[0] = operands[0];
8099    ops[1] = operands[1];
8100    ops[2] = operands[2];
8101    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
8102    return "";
8103 }
8104   [(set_attr "length" "4")])
8105
8106 ;;
8107 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
8108 ;;
8109 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
8110   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8111         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8112                        (match_operand:MVE_2 2 "s_register_operand" "w")]
8113          VLDRBGOQ))
8114   ]
8115   "TARGET_HAVE_MVE"
8116 {
8117    rtx ops[3];
8118    ops[0] = operands[0];
8119    ops[1] = operands[1];
8120    ops[2] = operands[2];
8121    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8122      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
8123    else
8124      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8125    return "";
8126 }
8127   [(set_attr "length" "4")])
8128
8129 ;;
8130 ;; [vldrbq_s vldrbq_u]
8131 ;;
8132 (define_insn "mve_vldrbq_<supf><mode>"
8133   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8134         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")]
8135          VLDRBQ))
8136   ]
8137   "TARGET_HAVE_MVE"
8138 {
8139    rtx ops[2];
8140    int regno = REGNO (operands[0]);
8141    ops[0] = gen_rtx_REG (TImode, regno);
8142    ops[1]  = operands[1];
8143    output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
8144    return "";
8145 }
8146   [(set_attr "length" "4")])
8147
8148 ;;
8149 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
8150 ;;
8151 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
8152   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8153         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8154                       (match_operand:SI 2 "immediate_operand" "i")]
8155          VLDRWGBQ))
8156   ]
8157   "TARGET_HAVE_MVE"
8158 {
8159    rtx ops[3];
8160    ops[0] = operands[0];
8161    ops[1] = operands[1];
8162    ops[2] = operands[2];
8163    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8164    return "";
8165 }
8166   [(set_attr "length" "4")])
8167
8168 ;;
8169 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
8170 ;;
8171 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>"
8172   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8173         (unspec:<MVE_B_ELEM>
8174                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8175                  (match_operand:MVE_2 2 "s_register_operand" "w")
8176                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8177          VSTRBSOQ))
8178   ]
8179   "TARGET_HAVE_MVE"
8180 {
8181    rtx ops[3];
8182    ops[0] = operands[0];
8183    ops[1] = operands[1];
8184    ops[2] = operands[2];
8185    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8186    return "";
8187 }
8188   [(set_attr "length" "8")])
8189
8190 ;;
8191 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
8192 ;;
8193 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
8194   [(set (mem:BLK (scratch))
8195         (unspec:BLK
8196                 [(match_operand:V4SI 0 "s_register_operand" "w")
8197                  (match_operand:SI 1 "immediate_operand" "i")
8198                  (match_operand:V4SI 2 "s_register_operand" "w")
8199                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8200          VSTRWSBQ))
8201   ]
8202   "TARGET_HAVE_MVE"
8203 {
8204    rtx ops[3];
8205    ops[0] = operands[0];
8206    ops[1] = operands[1];
8207    ops[2] = operands[2];
8208    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8209    return "";
8210 }
8211   [(set_attr "length" "8")])
8212
8213 ;;
8214 ;; [vstrbq_p_s vstrbq_p_u]
8215 ;;
8216 (define_insn "mve_vstrbq_p_<supf><mode>"
8217   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8218         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
8219                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8220          VSTRBQ))
8221   ]
8222   "TARGET_HAVE_MVE"
8223 {
8224    rtx ops[2];
8225    int regno = REGNO (operands[1]);
8226    ops[1] = gen_rtx_REG (TImode, regno);
8227    ops[0]  = operands[0];
8228    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q1, %E0",ops);
8229    return "";
8230 }
8231   [(set_attr "length" "8")])
8232
8233 ;;
8234 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
8235 ;;
8236 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
8237   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8238         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8239                        (match_operand:MVE_2 2 "s_register_operand" "w")
8240                        (match_operand:HI 3 "vpr_register_operand" "Up")]
8241          VLDRBGOQ))
8242   ]
8243   "TARGET_HAVE_MVE"
8244 {
8245    rtx ops[4];
8246    ops[0] = operands[0];
8247    ops[1] = operands[1];
8248    ops[2] = operands[2];
8249    ops[3] = operands[3];
8250    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8251      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
8252    else
8253      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8254    return "";
8255 }
8256   [(set_attr "length" "8")])
8257
8258 ;;
8259 ;; [vldrbq_z_s vldrbq_z_u]
8260 ;;
8261 (define_insn "mve_vldrbq_z_<supf><mode>"
8262   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8263         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8264                        (match_operand:HI 2 "vpr_register_operand" "Up")]
8265          VLDRBQ))
8266   ]
8267   "TARGET_HAVE_MVE"
8268 {
8269    rtx ops[2];
8270    int regno = REGNO (operands[0]);
8271    ops[0] = gen_rtx_REG (TImode, regno);
8272    ops[1]  = operands[1];
8273    output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
8274    return "";
8275 }
8276   [(set_attr "length" "8")])
8277
8278 ;;
8279 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
8280 ;;
8281 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
8282   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8283         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8284                       (match_operand:SI 2 "immediate_operand" "i")
8285                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8286          VLDRWGBQ))
8287   ]
8288   "TARGET_HAVE_MVE"
8289 {
8290    rtx ops[3];
8291    ops[0] = operands[0];
8292    ops[1] = operands[1];
8293    ops[2] = operands[2];
8294    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8295    return "";
8296 }
8297   [(set_attr "length" "8")])
8298
8299 ;;
8300 ;; [vldrhq_f]
8301 ;;
8302 (define_insn "mve_vldrhq_fv8hf"
8303   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8304         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")]
8305          VLDRHQ_F))
8306   ]
8307   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8308 {
8309    rtx ops[2];
8310    int regno = REGNO (operands[0]);
8311    ops[0] = gen_rtx_REG (TImode, regno);
8312    ops[1]  = operands[1];
8313    output_asm_insn ("vldrh.f16\t%q0, %E1",ops);
8314    return "";
8315 }
8316   [(set_attr "length" "4")])
8317
8318 ;;
8319 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
8320 ;;
8321 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
8322   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8323         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8324                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8325         VLDRHGOQ))
8326   ]
8327   "TARGET_HAVE_MVE"
8328 {
8329    rtx ops[3];
8330    ops[0] = operands[0];
8331    ops[1] = operands[1];
8332    ops[2] = operands[2];
8333    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8334      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
8335    else
8336      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8337    return "";
8338 }
8339   [(set_attr "length" "4")])
8340
8341 ;;
8342 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
8343 ;;
8344 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
8345   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8346         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8347                        (match_operand:MVE_6 2 "s_register_operand" "w")
8348                        (match_operand:HI 3 "vpr_register_operand" "Up")
8349         ]VLDRHGOQ))
8350   ]
8351   "TARGET_HAVE_MVE"
8352 {
8353    rtx ops[4];
8354    ops[0] = operands[0];
8355    ops[1] = operands[1];
8356    ops[2] = operands[2];
8357    ops[3] = operands[3];
8358    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8359      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
8360    else
8361      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8362    return "";
8363 }
8364  [(set_attr "length" "8")])
8365
8366 ;;
8367 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
8368 ;;
8369 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
8370   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8371         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8372                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8373         VLDRHGSOQ))
8374   ]
8375   "TARGET_HAVE_MVE"
8376 {
8377    rtx ops[3];
8378    ops[0] = operands[0];
8379    ops[1] = operands[1];
8380    ops[2] = operands[2];
8381       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8382      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8383    else
8384      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8385    return "";
8386 }
8387   [(set_attr "length" "4")])
8388
8389 ;;
8390 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
8391 ;;
8392 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
8393   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8394         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8395                        (match_operand:MVE_6 2 "s_register_operand" "w")
8396                        (match_operand:HI 3 "vpr_register_operand" "Up")
8397         ]VLDRHGSOQ))
8398   ]
8399   "TARGET_HAVE_MVE"
8400 {
8401    rtx ops[4];
8402    ops[0] = operands[0];
8403    ops[1] = operands[1];
8404    ops[2] = operands[2];
8405    ops[3] = operands[3];
8406    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8407      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8408    else
8409      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8410    return "";
8411 }
8412   [(set_attr "length" "8")])
8413
8414 ;;
8415 ;;
8416 ;; [vldrhq_s, vldrhq_u]
8417 ;;
8418 (define_insn "mve_vldrhq_<supf><mode>"
8419   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8420         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")]
8421          VLDRHQ))
8422   ]
8423   "TARGET_HAVE_MVE"
8424 {
8425    rtx ops[2];
8426    int regno = REGNO (operands[0]);
8427    ops[0] = gen_rtx_REG (TImode, regno);
8428    ops[1]  = operands[1];
8429    output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
8430    return "";
8431 }
8432   [(set_attr "length" "4")])
8433
8434 ;;
8435 ;; [vldrhq_z_f]
8436 ;;
8437 (define_insn "mve_vldrhq_z_fv8hf"
8438   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8439         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8440         (match_operand:HI 2 "vpr_register_operand" "Up")]
8441          VLDRHQ_F))
8442   ]
8443   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8444 {
8445    rtx ops[2];
8446    int regno = REGNO (operands[0]);
8447    ops[0] = gen_rtx_REG (TImode, regno);
8448    ops[1]  = operands[1];
8449    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, %E1",ops);
8450    return "";
8451 }
8452   [(set_attr "length" "8")])
8453
8454 ;;
8455 ;; [vldrhq_z_s vldrhq_z_u]
8456 ;;
8457 (define_insn "mve_vldrhq_z_<supf><mode>"
8458   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8459         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8460         (match_operand:HI 2 "vpr_register_operand" "Up")]
8461          VLDRHQ))
8462   ]
8463   "TARGET_HAVE_MVE"
8464 {
8465    rtx ops[2];
8466    int regno = REGNO (operands[0]);
8467    ops[0] = gen_rtx_REG (TImode, regno);
8468    ops[1]  = operands[1];
8469    output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8470    return "";
8471 }
8472   [(set_attr "length" "8")])
8473
8474 ;;
8475 ;; [vldrwq_f]
8476 ;;
8477 (define_insn "mve_vldrwq_fv4sf"
8478   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8479         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")]
8480          VLDRWQ_F))
8481   ]
8482   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8483 {
8484    rtx ops[2];
8485    int regno = REGNO (operands[0]);
8486    ops[0] = gen_rtx_REG (TImode, regno);
8487    ops[1]  = operands[1];
8488    output_asm_insn ("vldrw.f32\t%q0, %E1",ops);
8489    return "";
8490 }
8491   [(set_attr "length" "4")])
8492
8493 ;;
8494 ;; [vldrwq_s vldrwq_u]
8495 ;;
8496 (define_insn "mve_vldrwq_<supf>v4si"
8497   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8498         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")]
8499          VLDRWQ))
8500   ]
8501   "TARGET_HAVE_MVE"
8502 {
8503    rtx ops[2];
8504    int regno = REGNO (operands[0]);
8505    ops[0] = gen_rtx_REG (TImode, regno);
8506    ops[1]  = operands[1];
8507    output_asm_insn ("vldrw.<supf>32\t%q0, %E1",ops);
8508    return "";
8509 }
8510   [(set_attr "length" "4")])
8511
8512 ;;
8513 ;; [vldrwq_z_f]
8514 ;;
8515 (define_insn "mve_vldrwq_z_fv4sf"
8516   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8517         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8518         (match_operand:HI 2 "vpr_register_operand" "Up")]
8519          VLDRWQ_F))
8520   ]
8521   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8522 {
8523    rtx ops[2];
8524    int regno = REGNO (operands[0]);
8525    ops[0] = gen_rtx_REG (TImode, regno);
8526    ops[1]  = operands[1];
8527    output_asm_insn ("vpst\n\tvldrwt.f32\t%q0, %E1",ops);
8528    return "";
8529 }
8530   [(set_attr "length" "8")])
8531
8532 ;;
8533 ;; [vldrwq_z_s vldrwq_z_u]
8534 ;;
8535 (define_insn "mve_vldrwq_z_<supf>v4si"
8536   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8537         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8538         (match_operand:HI 2 "vpr_register_operand" "Up")]
8539          VLDRWQ))
8540   ]
8541   "TARGET_HAVE_MVE"
8542 {
8543    rtx ops[2];
8544    int regno = REGNO (operands[0]);
8545    ops[0] = gen_rtx_REG (TImode, regno);
8546    ops[1]  = operands[1];
8547    output_asm_insn ("vpst\n\tvldrwt.<supf>32\t%q0, %E1",ops);
8548    return "";
8549 }
8550   [(set_attr "length" "8")])
8551
8552 (define_expand "mve_vld1q_f<mode>"
8553   [(match_operand:MVE_0 0 "s_register_operand")
8554    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "memory_operand")] VLD1Q_F)
8555   ]
8556   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8557 {
8558   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8559   DONE;
8560 })
8561
8562 (define_expand "mve_vld1q_<supf><mode>"
8563   [(match_operand:MVE_2 0 "s_register_operand")
8564    (unspec:MVE_2 [(match_operand:MVE_2 1 "memory_operand")] VLD1Q)
8565   ]
8566   "TARGET_HAVE_MVE"
8567 {
8568   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8569   DONE;
8570 })
8571
8572 ;;
8573 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8574 ;;
8575 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8576   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8577         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8578                       (match_operand:SI 2 "immediate_operand" "i")]
8579          VLDRDGBQ))
8580   ]
8581   "TARGET_HAVE_MVE"
8582 {
8583    rtx ops[3];
8584    ops[0] = operands[0];
8585    ops[1] = operands[1];
8586    ops[2] = operands[2];
8587    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8588    return "";
8589 }
8590   [(set_attr "length" "4")])
8591
8592 ;;
8593 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8594 ;;
8595 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8596   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8597         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8598                       (match_operand:SI 2 "immediate_operand" "i")
8599                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8600          VLDRDGBQ))
8601   ]
8602   "TARGET_HAVE_MVE"
8603 {
8604    rtx ops[3];
8605    ops[0] = operands[0];
8606    ops[1] = operands[1];
8607    ops[2] = operands[2];
8608    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8609    return "";
8610 }
8611   [(set_attr "length" "8")])
8612
8613 ;;
8614 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8615 ;;
8616 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8617  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8618        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8619                      (match_operand:V2DI 2 "s_register_operand" "w")]
8620         VLDRDGOQ))
8621  ]
8622  "TARGET_HAVE_MVE"
8623 {
8624   rtx ops[3];
8625   ops[0] = operands[0];
8626   ops[1] = operands[1];
8627   ops[2] = operands[2];
8628   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8629   return "";
8630 }
8631  [(set_attr "length" "4")])
8632
8633 ;;
8634 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8635 ;;
8636 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8637  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8638        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8639                      (match_operand:V2DI 2 "s_register_operand" "w")
8640                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8641         VLDRDGOQ))
8642  ]
8643  "TARGET_HAVE_MVE"
8644 {
8645   rtx ops[3];
8646   ops[0] = operands[0];
8647   ops[1] = operands[1];
8648   ops[2] = operands[2];
8649   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8650   return "";
8651 }
8652  [(set_attr "length" "8")])
8653
8654 ;;
8655 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8656 ;;
8657 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8658   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8659         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8660                       (match_operand:V2DI 2 "s_register_operand" "w")]
8661          VLDRDGSOQ))
8662   ]
8663   "TARGET_HAVE_MVE"
8664 {
8665    rtx ops[3];
8666    ops[0] = operands[0];
8667    ops[1] = operands[1];
8668    ops[2] = operands[2];
8669    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8670    return "";
8671 }
8672   [(set_attr "length" "4")])
8673
8674 ;;
8675 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8676 ;;
8677 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8678   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8679         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8680                       (match_operand:V2DI 2 "s_register_operand" "w")
8681                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8682          VLDRDGSOQ))
8683   ]
8684   "TARGET_HAVE_MVE"
8685 {
8686    rtx ops[3];
8687    ops[0] = operands[0];
8688    ops[1] = operands[1];
8689    ops[2] = operands[2];
8690    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8691    return "";
8692 }
8693   [(set_attr "length" "8")])
8694
8695 ;;
8696 ;; [vldrhq_gather_offset_f]
8697 ;;
8698 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8699   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8700         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8701                       (match_operand:V8HI 2 "s_register_operand" "w")]
8702          VLDRHQGO_F))
8703   ]
8704   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8705 {
8706    rtx ops[3];
8707    ops[0] = operands[0];
8708    ops[1] = operands[1];
8709    ops[2] = operands[2];
8710    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8711    return "";
8712 }
8713   [(set_attr "length" "4")])
8714
8715 ;;
8716 ;; [vldrhq_gather_offset_z_f]
8717 ;;
8718 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8719   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8720         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8721                       (match_operand:V8HI 2 "s_register_operand" "w")
8722                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8723          VLDRHQGO_F))
8724   ]
8725   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8726 {
8727    rtx ops[4];
8728    ops[0] = operands[0];
8729    ops[1] = operands[1];
8730    ops[2] = operands[2];
8731    ops[3] = operands[3];
8732    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8733    return "";
8734 }
8735   [(set_attr "length" "8")])
8736
8737 ;;
8738 ;; [vldrhq_gather_shifted_offset_f]
8739 ;;
8740 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8741   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8742         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8743                       (match_operand:V8HI 2 "s_register_operand" "w")]
8744          VLDRHQGSO_F))
8745   ]
8746   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8747 {
8748    rtx ops[3];
8749    ops[0] = operands[0];
8750    ops[1] = operands[1];
8751    ops[2] = operands[2];
8752    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8753    return "";
8754 }
8755   [(set_attr "length" "4")])
8756
8757 ;;
8758 ;; [vldrhq_gather_shifted_offset_z_f]
8759 ;;
8760 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8761   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8762         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8763                       (match_operand:V8HI 2 "s_register_operand" "w")
8764                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8765          VLDRHQGSO_F))
8766   ]
8767   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8768 {
8769    rtx ops[4];
8770    ops[0] = operands[0];
8771    ops[1] = operands[1];
8772    ops[2] = operands[2];
8773    ops[3] = operands[3];
8774    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8775    return "";
8776 }
8777   [(set_attr "length" "8")])
8778
8779 ;;
8780 ;; [vldrwq_gather_base_f]
8781 ;;
8782 (define_insn "mve_vldrwq_gather_base_fv4sf"
8783   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8784         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8785                       (match_operand:SI 2 "immediate_operand" "i")]
8786          VLDRWQGB_F))
8787   ]
8788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8789 {
8790    rtx ops[3];
8791    ops[0] = operands[0];
8792    ops[1] = operands[1];
8793    ops[2] = operands[2];
8794    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8795    return "";
8796 }
8797   [(set_attr "length" "4")])
8798
8799 ;;
8800 ;; [vldrwq_gather_base_z_f]
8801 ;;
8802 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8803   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8804         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8805                       (match_operand:SI 2 "immediate_operand" "i")
8806                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8807          VLDRWQGB_F))
8808   ]
8809   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8810 {
8811    rtx ops[3];
8812    ops[0] = operands[0];
8813    ops[1] = operands[1];
8814    ops[2] = operands[2];
8815    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8816    return "";
8817 }
8818   [(set_attr "length" "8")])
8819
8820 ;;
8821 ;; [vldrwq_gather_offset_f]
8822 ;;
8823 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8824   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8825         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8826                        (match_operand:V4SI 2 "s_register_operand" "w")]
8827          VLDRWQGO_F))
8828   ]
8829   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8830 {
8831    rtx ops[3];
8832    ops[0] = operands[0];
8833    ops[1] = operands[1];
8834    ops[2] = operands[2];
8835    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8836    return "";
8837 }
8838   [(set_attr "length" "4")])
8839
8840 ;;
8841 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8842 ;;
8843 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8844   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8845         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8846                        (match_operand:V4SI 2 "s_register_operand" "w")]
8847          VLDRWGOQ))
8848   ]
8849   "TARGET_HAVE_MVE"
8850 {
8851    rtx ops[3];
8852    ops[0] = operands[0];
8853    ops[1] = operands[1];
8854    ops[2] = operands[2];
8855    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8856    return "";
8857 }
8858   [(set_attr "length" "4")])
8859
8860 ;;
8861 ;; [vldrwq_gather_offset_z_f]
8862 ;;
8863 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8864   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8865         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8866                       (match_operand:V4SI 2 "s_register_operand" "w")
8867                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8868          VLDRWQGO_F))
8869   ]
8870   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8871 {
8872    rtx ops[4];
8873    ops[0] = operands[0];
8874    ops[1] = operands[1];
8875    ops[2] = operands[2];
8876    ops[3] = operands[3];
8877    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8878    return "";
8879 }
8880   [(set_attr "length" "8")])
8881
8882 ;;
8883 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8884 ;;
8885 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8886   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8887         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8888                       (match_operand:V4SI 2 "s_register_operand" "w")
8889                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8890          VLDRWGOQ))
8891   ]
8892   "TARGET_HAVE_MVE"
8893 {
8894    rtx ops[4];
8895    ops[0] = operands[0];
8896    ops[1] = operands[1];
8897    ops[2] = operands[2];
8898    ops[3] = operands[3];
8899    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8900    return "";
8901 }
8902   [(set_attr "length" "8")])
8903
8904 ;;
8905 ;; [vldrwq_gather_shifted_offset_f]
8906 ;;
8907 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8908   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8909         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8910                       (match_operand:V4SI 2 "s_register_operand" "w")]
8911          VLDRWQGSO_F))
8912   ]
8913   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8914 {
8915    rtx ops[3];
8916    ops[0] = operands[0];
8917    ops[1] = operands[1];
8918    ops[2] = operands[2];
8919    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8920    return "";
8921 }
8922   [(set_attr "length" "4")])
8923
8924 ;;
8925 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8926 ;;
8927 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8928   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8929         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8930                       (match_operand:V4SI 2 "s_register_operand" "w")]
8931          VLDRWGSOQ))
8932   ]
8933   "TARGET_HAVE_MVE"
8934 {
8935    rtx ops[3];
8936    ops[0] = operands[0];
8937    ops[1] = operands[1];
8938    ops[2] = operands[2];
8939    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8940    return "";
8941 }
8942   [(set_attr "length" "4")])
8943
8944 ;;
8945 ;; [vldrwq_gather_shifted_offset_z_f]
8946 ;;
8947 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8948   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8949         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8950                       (match_operand:V4SI 2 "s_register_operand" "w")
8951                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8952          VLDRWQGSO_F))
8953   ]
8954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8955 {
8956    rtx ops[4];
8957    ops[0] = operands[0];
8958    ops[1] = operands[1];
8959    ops[2] = operands[2];
8960    ops[3] = operands[3];
8961    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8962    return "";
8963 }
8964   [(set_attr "length" "8")])
8965
8966 ;;
8967 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8968 ;;
8969 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8970   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8971         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8972                       (match_operand:V4SI 2 "s_register_operand" "w")
8973                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8974          VLDRWGSOQ))
8975   ]
8976   "TARGET_HAVE_MVE"
8977 {
8978    rtx ops[4];
8979    ops[0] = operands[0];
8980    ops[1] = operands[1];
8981    ops[2] = operands[2];
8982    ops[3] = operands[3];
8983    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8984    return "";
8985 }
8986   [(set_attr "length" "8")])
8987
8988 ;;
8989 ;; [vstrhq_f]
8990 ;;
8991 (define_insn "mve_vstrhq_fv8hf"
8992   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
8993         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8994          VSTRHQ_F))
8995   ]
8996   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8997 {
8998    rtx ops[2];
8999    int regno = REGNO (operands[1]);
9000    ops[1] = gen_rtx_REG (TImode, regno);
9001    ops[0]  = operands[0];
9002    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
9003    return "";
9004 }
9005   [(set_attr "length" "4")])
9006
9007 ;;
9008 ;; [vstrhq_p_f]
9009 ;;
9010 (define_insn "mve_vstrhq_p_fv8hf"
9011   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9012         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
9013                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9014          VSTRHQ_F))
9015   ]
9016   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9017 {
9018    rtx ops[2];
9019    int regno = REGNO (operands[1]);
9020    ops[1] = gen_rtx_REG (TImode, regno);
9021    ops[0]  = operands[0];
9022    output_asm_insn ("vpst\n\tvstrht.16\t%q1, %E0",ops);
9023    return "";
9024 }
9025   [(set_attr "length" "8")])
9026
9027 ;;
9028 ;; [vstrhq_p_s vstrhq_p_u]
9029 ;;
9030 (define_insn "mve_vstrhq_p_<supf><mode>"
9031   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9032         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
9033                               (match_operand:HI 2 "vpr_register_operand" "Up")]
9034          VSTRHQ))
9035   ]
9036   "TARGET_HAVE_MVE"
9037 {
9038    rtx ops[2];
9039    int regno = REGNO (operands[1]);
9040    ops[1] = gen_rtx_REG (TImode, regno);
9041    ops[0]  = operands[0];
9042    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q1, %E0",ops);
9043    return "";
9044 }
9045   [(set_attr "length" "8")])
9046
9047 ;;
9048 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
9049 ;;
9050 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>"
9051   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9052         (unspec:<MVE_H_ELEM>
9053                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9054                  (match_operand:MVE_6 2 "s_register_operand" "w")
9055                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9056          VSTRHSOQ))
9057   ]
9058   "TARGET_HAVE_MVE"
9059 {
9060    rtx ops[3];
9061    ops[0] = operands[0];
9062    ops[1] = operands[1];
9063    ops[2] = operands[2];
9064    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9065    return "";
9066 }
9067   [(set_attr "length" "8")])
9068
9069 ;;
9070 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
9071 ;;
9072 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>"
9073   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9074         (unspec:<MVE_H_ELEM>
9075                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9076                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9077          VSTRHSOQ))
9078   ]
9079   "TARGET_HAVE_MVE"
9080 {
9081    rtx ops[3];
9082    ops[0] = operands[0];
9083    ops[1] = operands[1];
9084    ops[2] = operands[2];
9085    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9086    return "";
9087 }
9088   [(set_attr "length" "4")])
9089
9090 ;;
9091 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
9092 ;;
9093 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
9094   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9095         (unspec:<MVE_H_ELEM>
9096                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9097                  (match_operand:MVE_6 2 "s_register_operand" "w")
9098                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9099          VSTRHSSOQ))
9100   ]
9101   "TARGET_HAVE_MVE"
9102 {
9103    rtx ops[3];
9104    ops[0] = operands[0];
9105    ops[1] = operands[1];
9106    ops[2] = operands[2];
9107    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9108    return "";
9109 }
9110   [(set_attr "length" "8")])
9111
9112 ;;
9113 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
9114 ;;
9115 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
9116   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9117         (unspec:<MVE_H_ELEM>
9118                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9119                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9120          VSTRHSSOQ))
9121   ]
9122   "TARGET_HAVE_MVE"
9123 {
9124    rtx ops[3];
9125    ops[0] = operands[0];
9126    ops[1] = operands[1];
9127    ops[2] = operands[2];
9128    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9129    return "";
9130 }
9131   [(set_attr "length" "4")])
9132
9133 ;;
9134 ;; [vstrhq_s, vstrhq_u]
9135 ;;
9136 (define_insn "mve_vstrhq_<supf><mode>"
9137   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9138         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
9139          VSTRHQ))
9140   ]
9141   "TARGET_HAVE_MVE"
9142 {
9143    rtx ops[2];
9144    int regno = REGNO (operands[1]);
9145    ops[1] = gen_rtx_REG (TImode, regno);
9146    ops[0]  = operands[0];
9147    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
9148    return "";
9149 }
9150   [(set_attr "length" "4")])
9151
9152 ;;
9153 ;; [vstrwq_f]
9154 ;;
9155 (define_insn "mve_vstrwq_fv4sf"
9156   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9157         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
9158          VSTRWQ_F))
9159   ]
9160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9161 {
9162    rtx ops[2];
9163    int regno = REGNO (operands[1]);
9164    ops[1] = gen_rtx_REG (TImode, regno);
9165    ops[0]  = operands[0];
9166    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9167    return "";
9168 }
9169   [(set_attr "length" "4")])
9170
9171 ;;
9172 ;; [vstrwq_p_f]
9173 ;;
9174 (define_insn "mve_vstrwq_p_fv4sf"
9175   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9176         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
9177                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9178          VSTRWQ_F))
9179   ]
9180   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9181 {
9182    rtx ops[2];
9183    int regno = REGNO (operands[1]);
9184    ops[1] = gen_rtx_REG (TImode, regno);
9185    ops[0]  = operands[0];
9186    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9187    return "";
9188 }
9189   [(set_attr "length" "8")])
9190
9191 ;;
9192 ;; [vstrwq_p_s vstrwq_p_u]
9193 ;;
9194 (define_insn "mve_vstrwq_p_<supf>v4si"
9195   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9196         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9197                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9198          VSTRWQ))
9199   ]
9200   "TARGET_HAVE_MVE"
9201 {
9202    rtx ops[2];
9203    int regno = REGNO (operands[1]);
9204    ops[1] = gen_rtx_REG (TImode, regno);
9205    ops[0]  = operands[0];
9206    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9207    return "";
9208 }
9209   [(set_attr "length" "8")])
9210
9211 ;;
9212 ;; [vstrwq_s vstrwq_u]
9213 ;;
9214 (define_insn "mve_vstrwq_<supf>v4si"
9215   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9216         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
9217          VSTRWQ))
9218   ]
9219   "TARGET_HAVE_MVE"
9220 {
9221    rtx ops[2];
9222    int regno = REGNO (operands[1]);
9223    ops[1] = gen_rtx_REG (TImode, regno);
9224    ops[0]  = operands[0];
9225    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9226    return "";
9227 }
9228   [(set_attr "length" "4")])
9229
9230 (define_expand "mve_vst1q_f<mode>"
9231   [(match_operand:<MVE_CNVT> 0 "memory_operand")
9232    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
9233   ]
9234   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
9235 {
9236   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
9237   DONE;
9238 })
9239
9240 (define_expand "mve_vst1q_<supf><mode>"
9241   [(match_operand:MVE_2 0 "memory_operand")
9242    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
9243   ]
9244   "TARGET_HAVE_MVE"
9245 {
9246   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
9247   DONE;
9248 })
9249
9250 ;;
9251 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
9252 ;;
9253 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
9254   [(set (mem:BLK (scratch))
9255         (unspec:BLK
9256                 [(match_operand:V2DI 0 "s_register_operand" "w")
9257                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9258                  (match_operand:V2DI 2 "s_register_operand" "w")
9259                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9260          VSTRDSBQ))
9261   ]
9262   "TARGET_HAVE_MVE"
9263 {
9264    rtx ops[3];
9265    ops[0] = operands[0];
9266    ops[1] = operands[1];
9267    ops[2] = operands[2];
9268    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
9269    return "";
9270 }
9271   [(set_attr "length" "8")])
9272
9273 ;;
9274 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
9275 ;;
9276 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
9277   [(set (mem:BLK (scratch))
9278         (unspec:BLK
9279                 [(match_operand:V2DI 0 "s_register_operand" "=w")
9280                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9281                  (match_operand:V2DI 2 "s_register_operand" "w")]
9282          VSTRDSBQ))
9283   ]
9284   "TARGET_HAVE_MVE"
9285 {
9286    rtx ops[3];
9287    ops[0] = operands[0];
9288    ops[1] = operands[1];
9289    ops[2] = operands[2];
9290    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
9291    return "";
9292 }
9293   [(set_attr "length" "4")])
9294
9295 ;;
9296 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
9297 ;;
9298 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di"
9299   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9300         (unspec:V2DI
9301                 [(match_operand:V2DI 1 "s_register_operand" "w")
9302                  (match_operand:V2DI 2 "s_register_operand" "w")
9303                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9304          VSTRDSOQ))
9305   ]
9306   "TARGET_HAVE_MVE"
9307 {
9308    rtx ops[3];
9309    ops[0] = operands[0];
9310    ops[1] = operands[1];
9311    ops[2] = operands[2];
9312    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1]",ops);
9313    return "";
9314 }
9315   [(set_attr "length" "8")])
9316
9317 ;;
9318 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
9319 ;;
9320 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di"
9321   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9322         (unspec:V2DI
9323                 [(match_operand:V2DI 1 "s_register_operand" "w")
9324                  (match_operand:V2DI 2 "s_register_operand" "w")]
9325          VSTRDSOQ))
9326   ]
9327   "TARGET_HAVE_MVE"
9328 {
9329    rtx ops[3];
9330    ops[0] = operands[0];
9331    ops[1] = operands[1];
9332    ops[2] = operands[2];
9333    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1]",ops);
9334    return "";
9335 }
9336   [(set_attr "length" "4")])
9337
9338 ;;
9339 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
9340 ;;
9341 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
9342   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9343         (unspec:V2DI
9344                 [(match_operand:V2DI 1 "s_register_operand" "w")
9345                  (match_operand:V2DI 2 "s_register_operand" "w")
9346                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9347          VSTRDSSOQ))
9348   ]
9349   "TARGET_HAVE_MVE"
9350 {
9351    rtx ops[3];
9352    ops[0] = operands[0];
9353    ops[1] = operands[1];
9354    ops[2] = operands[2];
9355    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9356    return "";
9357 }
9358   [(set_attr "length" "8")])
9359
9360 ;;
9361 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
9362 ;;
9363 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
9364   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9365         (unspec:V2DI
9366                 [(match_operand:V2DI 1 "s_register_operand" "w")
9367                  (match_operand:V2DI 2 "s_register_operand" "w")]
9368          VSTRDSSOQ))
9369   ]
9370   "TARGET_HAVE_MVE"
9371 {
9372    rtx ops[3];
9373    ops[0] = operands[0];
9374    ops[1] = operands[1];
9375    ops[2] = operands[2];
9376    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9377    return "";
9378 }
9379   [(set_attr "length" "4")])
9380
9381 ;;
9382 ;; [vstrhq_scatter_offset_f]
9383 ;;
9384 (define_insn "mve_vstrhq_scatter_offset_fv8hf"
9385   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9386         (unspec:V8HI
9387                 [(match_operand:V8HI 1 "s_register_operand" "w")
9388                  (match_operand:V8HF 2 "s_register_operand" "w")]
9389          VSTRHQSO_F))
9390   ]
9391   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9392 {
9393    rtx ops[3];
9394    ops[0] = operands[0];
9395    ops[1] = operands[1];
9396    ops[2] = operands[2];
9397    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1]",ops);
9398    return "";
9399 }
9400   [(set_attr "length" "4")])
9401
9402 ;;
9403 ;; [vstrhq_scatter_offset_p_f]
9404 ;;
9405 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf"
9406   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9407         (unspec:V8HI
9408                 [(match_operand:V8HI 1 "s_register_operand" "w")
9409                  (match_operand:V8HF 2 "s_register_operand" "w")
9410                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9411          VSTRHQSO_F))
9412   ]
9413   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9414 {
9415    rtx ops[3];
9416    ops[0] = operands[0];
9417    ops[1] = operands[1];
9418    ops[2] = operands[2];
9419    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1]",ops);
9420    return "";
9421 }
9422   [(set_attr "length" "8")])
9423
9424 ;;
9425 ;; [vstrhq_scatter_shifted_offset_f]
9426 ;;
9427 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf"
9428   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9429         (unspec:V8HI
9430                 [(match_operand:V8HI 1 "s_register_operand" "w")
9431                  (match_operand:V8HF 2 "s_register_operand" "w")]
9432          VSTRHQSSO_F))
9433   ]
9434   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9435 {
9436    rtx ops[3];
9437    ops[0] = operands[0];
9438    ops[1] = operands[1];
9439    ops[2] = operands[2];
9440    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9441    return "";
9442 }
9443   [(set_attr "length" "4")])
9444
9445 ;;
9446 ;; [vstrhq_scatter_shifted_offset_p_f]
9447 ;;
9448 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9449   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9450         (unspec:V8HI
9451                 [(match_operand:V8HI 1 "s_register_operand" "w")
9452                  (match_operand:V8HF 2 "s_register_operand" "w")
9453                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9454          VSTRHQSSO_F))
9455   ]
9456   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9457 {
9458    rtx ops[3];
9459    ops[0] = operands[0];
9460    ops[1] = operands[1];
9461    ops[2] = operands[2];
9462    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9463    return "";
9464 }
9465   [(set_attr "length" "8")])
9466
9467 ;;
9468 ;; [vstrwq_scatter_base_f]
9469 ;;
9470 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9471   [(set (mem:BLK (scratch))
9472         (unspec:BLK
9473                 [(match_operand:V4SI 0 "s_register_operand" "w")
9474                  (match_operand:SI 1 "immediate_operand" "i")
9475                  (match_operand:V4SF 2 "s_register_operand" "w")]
9476          VSTRWQSB_F))
9477   ]
9478   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9479 {
9480    rtx ops[3];
9481    ops[0] = operands[0];
9482    ops[1] = operands[1];
9483    ops[2] = operands[2];
9484    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9485    return "";
9486 }
9487   [(set_attr "length" "4")])
9488
9489 ;;
9490 ;; [vstrwq_scatter_base_p_f]
9491 ;;
9492 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9493   [(set (mem:BLK (scratch))
9494         (unspec:BLK
9495                 [(match_operand:V4SI 0 "s_register_operand" "w")
9496                  (match_operand:SI 1 "immediate_operand" "i")
9497                  (match_operand:V4SF 2 "s_register_operand" "w")
9498                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9499          VSTRWQSB_F))
9500   ]
9501   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9502 {
9503    rtx ops[3];
9504    ops[0] = operands[0];
9505    ops[1] = operands[1];
9506    ops[2] = operands[2];
9507    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9508    return "";
9509 }
9510   [(set_attr "length" "8")])
9511
9512 ;;
9513 ;; [vstrwq_scatter_offset_f]
9514 ;;
9515 (define_insn "mve_vstrwq_scatter_offset_fv4sf"
9516   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9517         (unspec:V4SI
9518                 [(match_operand:V4SI 1 "s_register_operand" "w")
9519                  (match_operand:V4SF 2 "s_register_operand" "w")]
9520          VSTRWQSO_F))
9521   ]
9522   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9523 {
9524    rtx ops[3];
9525    ops[0] = operands[0];
9526    ops[1] = operands[1];
9527    ops[2] = operands[2];
9528    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9529    return "";
9530 }
9531   [(set_attr "length" "4")])
9532
9533 ;;
9534 ;; [vstrwq_scatter_offset_p_f]
9535 ;;
9536 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf"
9537   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9538         (unspec:V4SI
9539                 [(match_operand:V4SI 1 "s_register_operand" "w")
9540                  (match_operand:V4SF 2 "s_register_operand" "w")
9541                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9542          VSTRWQSO_F))
9543   ]
9544   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9545 {
9546    rtx ops[3];
9547    ops[0] = operands[0];
9548    ops[1] = operands[1];
9549    ops[2] = operands[2];
9550    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9551    return "";
9552 }
9553   [(set_attr "length" "8")])
9554
9555 ;;
9556 ;; [vstrwq_scatter_offset_p_s vstrwq_scatter_offset_p_u]
9557 ;;
9558 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si"
9559   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9560         (unspec:V4SI
9561                 [(match_operand:V4SI 1 "s_register_operand" "w")
9562                  (match_operand:V4SI 2 "s_register_operand" "w")
9563                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9564          VSTRWSOQ))
9565   ]
9566   "TARGET_HAVE_MVE"
9567 {
9568    rtx ops[3];
9569    ops[0] = operands[0];
9570    ops[1] = operands[1];
9571    ops[2] = operands[2];
9572    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9573    return "";
9574 }
9575   [(set_attr "length" "8")])
9576
9577 ;;
9578 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9579 ;;
9580 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si"
9581   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9582         (unspec:V4SI
9583                 [(match_operand:V4SI 1 "s_register_operand" "w")
9584                  (match_operand:V4SI 2 "s_register_operand" "w")]
9585          VSTRWSOQ))
9586   ]
9587   "TARGET_HAVE_MVE"
9588 {
9589    rtx ops[3];
9590    ops[0] = operands[0];
9591    ops[1] = operands[1];
9592    ops[2] = operands[2];
9593    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9594    return "";
9595 }
9596   [(set_attr "length" "4")])
9597
9598 ;;
9599 ;; [vstrwq_scatter_shifted_offset_f]
9600 ;;
9601 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf"
9602   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9603         (unspec:V4SI
9604                 [(match_operand:V4SI 1 "s_register_operand" "w")
9605                  (match_operand:V4SF 2 "s_register_operand" "w")]
9606          VSTRWQSSO_F))
9607   ]
9608   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9609 {
9610    rtx ops[3];
9611    ops[0] = operands[0];
9612    ops[1] = operands[1];
9613    ops[2] = operands[2];
9614    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9615    return "";
9616 }
9617   [(set_attr "length" "4")])
9618
9619 ;;
9620 ;; [vstrwq_scatter_shifted_offset_p_f]
9621 ;;
9622 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9623   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9624         (unspec:V4SI
9625                 [(match_operand:V4SI 1 "s_register_operand" "w")
9626                  (match_operand:V4SF 2 "s_register_operand" "w")
9627                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9628          VSTRWQSSO_F))
9629   ]
9630   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9631 {
9632    rtx ops[3];
9633    ops[0] = operands[0];
9634    ops[1] = operands[1];
9635    ops[2] = operands[2];
9636    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9637    return "";
9638 }
9639   [(set_attr "length" "8")])
9640
9641 ;;
9642 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9643 ;;
9644 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9645   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9646         (unspec:V4SI
9647                 [(match_operand:V4SI 1 "s_register_operand" "w")
9648                  (match_operand:V4SI 2 "s_register_operand" "w")
9649                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9650          VSTRWSSOQ))
9651   ]
9652   "TARGET_HAVE_MVE"
9653 {
9654    rtx ops[3];
9655    ops[0] = operands[0];
9656    ops[1] = operands[1];
9657    ops[2] = operands[2];
9658    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9659    return "";
9660 }
9661   [(set_attr "length" "8")])
9662
9663 ;;
9664 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9665 ;;
9666 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9667   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9668         (unspec:V4SI
9669                 [(match_operand:V4SI 1 "s_register_operand" "w")
9670                  (match_operand:V4SI 2 "s_register_operand" "w")]
9671          VSTRWSSOQ))
9672   ]
9673   "TARGET_HAVE_MVE"
9674 {
9675    rtx ops[3];
9676    ops[0] = operands[0];
9677    ops[1] = operands[1];
9678    ops[2] = operands[2];
9679    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9680    return "";
9681 }
9682   [(set_attr "length" "4")])
9683
9684 ;;
9685 ;; [vaddq_s, vaddq_u])
9686 ;;
9687 (define_insn "mve_vaddq<mode>"
9688   [
9689    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9690         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9691                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9692   ]
9693   "TARGET_HAVE_MVE"
9694   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9695   [(set_attr "type" "mve_move")
9696 ])
9697
9698 ;;
9699 ;; [vaddq_f])
9700 ;;
9701 (define_insn "mve_vaddq_f<mode>"
9702   [
9703    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9704         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9705                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9706   ]
9707   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9708   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9709   [(set_attr "type" "mve_move")
9710 ])
9711
9712 ;;
9713 ;; [vidupq_n_u])
9714 ;;
9715 (define_expand "mve_vidupq_n_u<mode>"
9716  [(match_operand:MVE_2 0 "s_register_operand")
9717   (match_operand:SI 1 "s_register_operand")
9718   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9719  "TARGET_HAVE_MVE"
9720 {
9721   rtx temp = gen_reg_rtx (SImode);
9722   emit_move_insn (temp, operands[1]);
9723   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9724   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9725                                           operands[2], inc));
9726   DONE;
9727 })
9728
9729 ;;
9730 ;; [vidupq_u_insn])
9731 ;;
9732 (define_insn "mve_vidupq_u<mode>_insn"
9733  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9734        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9735                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9736          VIDUPQ))
9737   (set (match_operand:SI 1 "s_register_operand" "=e")
9738        (plus:SI (match_dup 2)
9739                 (match_operand:SI 4 "immediate_operand" "i")))]
9740  "TARGET_HAVE_MVE"
9741  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9742
9743 ;;
9744 ;; [vidupq_m_n_u])
9745 ;;
9746 (define_expand "mve_vidupq_m_n_u<mode>"
9747   [(match_operand:MVE_2 0 "s_register_operand")
9748    (match_operand:MVE_2 1 "s_register_operand")
9749    (match_operand:SI 2 "s_register_operand")
9750    (match_operand:SI 3 "mve_imm_selective_upto_8")
9751    (match_operand:HI 4 "vpr_register_operand")]
9752   "TARGET_HAVE_MVE"
9753 {
9754   rtx temp = gen_reg_rtx (SImode);
9755   emit_move_insn (temp, operands[2]);
9756   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9757   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9758                                              operands[2], operands[3],
9759                                              operands[4], inc));
9760   DONE;
9761 })
9762
9763 ;;
9764 ;; [vidupq_m_wb_u_insn])
9765 ;;
9766 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9767  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9768        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9769                       (match_operand:SI 3 "s_register_operand" "2")
9770                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9771                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9772         VIDUPQ_M))
9773   (set (match_operand:SI 2 "s_register_operand" "=e")
9774        (plus:SI (match_dup 3)
9775                 (match_operand:SI 6 "immediate_operand" "i")))]
9776  "TARGET_HAVE_MVE"
9777  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9778  [(set_attr "length""8")])
9779
9780 ;;
9781 ;; [vddupq_n_u])
9782 ;;
9783 (define_expand "mve_vddupq_n_u<mode>"
9784  [(match_operand:MVE_2 0 "s_register_operand")
9785   (match_operand:SI 1 "s_register_operand")
9786   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9787  "TARGET_HAVE_MVE"
9788 {
9789   rtx temp = gen_reg_rtx (SImode);
9790   emit_move_insn (temp, operands[1]);
9791   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9792   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9793                                           operands[2], inc));
9794   DONE;
9795 })
9796
9797 ;;
9798 ;; [vddupq_u_insn])
9799 ;;
9800 (define_insn "mve_vddupq_u<mode>_insn"
9801  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9802        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9803                       (match_operand:SI 3 "immediate_operand" "i")]
9804         VDDUPQ))
9805   (set (match_operand:SI 1 "s_register_operand" "=e")
9806        (minus:SI (match_dup 2)
9807                  (match_operand:SI 4 "immediate_operand" "i")))]
9808  "TARGET_HAVE_MVE"
9809  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9810
9811 ;;
9812 ;; [vddupq_m_n_u])
9813 ;;
9814 (define_expand "mve_vddupq_m_n_u<mode>"
9815   [(match_operand:MVE_2 0 "s_register_operand")
9816    (match_operand:MVE_2 1 "s_register_operand")
9817    (match_operand:SI 2 "s_register_operand")
9818    (match_operand:SI 3 "mve_imm_selective_upto_8")
9819    (match_operand:HI 4 "vpr_register_operand")]
9820   "TARGET_HAVE_MVE"
9821 {
9822   rtx temp = gen_reg_rtx (SImode);
9823   emit_move_insn (temp, operands[2]);
9824   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9825   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9826                                              operands[2], operands[3],
9827                                              operands[4], inc));
9828   DONE;
9829 })
9830
9831 ;;
9832 ;; [vddupq_m_wb_u_insn])
9833 ;;
9834 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9835  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9836        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9837                       (match_operand:SI 3 "s_register_operand" "2")
9838                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9839                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9840         VDDUPQ_M))
9841   (set (match_operand:SI 2 "s_register_operand" "=e")
9842        (minus:SI (match_dup 3)
9843                  (match_operand:SI 6 "immediate_operand" "i")))]
9844  "TARGET_HAVE_MVE"
9845  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9846  [(set_attr "length""8")])
9847
9848 ;;
9849 ;; [vdwdupq_n_u])
9850 ;;
9851 (define_expand "mve_vdwdupq_n_u<mode>"
9852  [(match_operand:MVE_2 0 "s_register_operand")
9853   (match_operand:SI 1 "s_register_operand")
9854   (match_operand:SI 2 "s_register_operand")
9855   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9856  "TARGET_HAVE_MVE"
9857 {
9858   rtx ignore_wb = gen_reg_rtx (SImode);
9859   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9860                                               operands[1], operands[2],
9861                                               operands[3]));
9862   DONE;
9863 })
9864
9865 ;;
9866 ;; [vdwdupq_wb_u])
9867 ;;
9868 (define_expand "mve_vdwdupq_wb_u<mode>"
9869  [(match_operand:SI 0 "s_register_operand")
9870   (match_operand:SI 1 "s_register_operand")
9871   (match_operand:SI 2 "s_register_operand")
9872   (match_operand:SI 3 "mve_imm_selective_upto_8")
9873   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9874  "TARGET_HAVE_MVE"
9875 {
9876   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9877   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9878                                               operands[1], operands[2],
9879                                               operands[3]));
9880   DONE;
9881 })
9882
9883 ;;
9884 ;; [vdwdupq_wb_u_insn])
9885 ;;
9886 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9887   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9888         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9889                        (match_operand:SI 3 "s_register_operand" "r")
9890                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9891          VDWDUPQ))
9892    (set (match_operand:SI 1 "s_register_operand" "=e")
9893         (unspec:SI [(match_dup 2)
9894                     (match_dup 3)
9895                     (match_dup 4)]
9896          VDWDUPQ))]
9897   "TARGET_HAVE_MVE"
9898   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %3, %4"
9899 )
9900
9901 ;;
9902 ;; [vdwdupq_m_n_u])
9903 ;;
9904 (define_expand "mve_vdwdupq_m_n_u<mode>"
9905  [(match_operand:MVE_2 0 "s_register_operand")
9906   (match_operand:MVE_2 1 "s_register_operand")
9907   (match_operand:SI 2 "s_register_operand")
9908   (match_operand:SI 3 "s_register_operand")
9909   (match_operand:SI 4 "mve_imm_selective_upto_8")
9910   (match_operand:HI 5 "vpr_register_operand")]
9911  "TARGET_HAVE_MVE"
9912 {
9913   rtx ignore_wb = gen_reg_rtx (SImode);
9914   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9915                                                 operands[1], operands[2],
9916                                                 operands[3], operands[4],
9917                                                 operands[5]));
9918   DONE;
9919 })
9920
9921 ;;
9922 ;; [vdwdupq_m_wb_u])
9923 ;;
9924 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9925  [(match_operand:SI 0 "s_register_operand")
9926   (match_operand:MVE_2 1 "s_register_operand")
9927   (match_operand:SI 2 "s_register_operand")
9928   (match_operand:SI 3 "s_register_operand")
9929   (match_operand:SI 4 "mve_imm_selective_upto_8")
9930   (match_operand:HI 5 "vpr_register_operand")]
9931  "TARGET_HAVE_MVE"
9932 {
9933   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9934   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9935                                                 operands[1], operands[2],
9936                                                 operands[3], operands[4],
9937                                                 operands[5]));
9938   DONE;
9939 })
9940
9941 ;;
9942 ;; [vdwdupq_m_wb_u_insn])
9943 ;;
9944 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9945   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9946         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "w")
9947                        (match_operand:SI 3 "s_register_operand" "1")
9948                        (match_operand:SI 4 "s_register_operand" "r")
9949                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9950                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9951          VDWDUPQ_M))
9952    (set (match_operand:SI 1 "s_register_operand" "=e")
9953         (unspec:SI [(match_dup 2)
9954                     (match_dup 3)
9955                     (match_dup 4)
9956                     (match_dup 5)
9957                     (match_dup 6)]
9958          VDWDUPQ_M))
9959   ]
9960   "TARGET_HAVE_MVE"
9961   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %4, %5"
9962   [(set_attr "type" "mve_move")
9963    (set_attr "length""8")])
9964
9965 ;;
9966 ;; [viwdupq_n_u])
9967 ;;
9968 (define_expand "mve_viwdupq_n_u<mode>"
9969  [(match_operand:MVE_2 0 "s_register_operand")
9970   (match_operand:SI 1 "s_register_operand")
9971   (match_operand:SI 2 "s_register_operand")
9972   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9973  "TARGET_HAVE_MVE"
9974 {
9975   rtx ignore_wb = gen_reg_rtx (SImode);
9976   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9977                                               operands[1], operands[2],
9978                                               operands[3]));
9979   DONE;
9980 })
9981
9982 ;;
9983 ;; [viwdupq_wb_u])
9984 ;;
9985 (define_expand "mve_viwdupq_wb_u<mode>"
9986  [(match_operand:SI 0 "s_register_operand")
9987   (match_operand:SI 1 "s_register_operand")
9988   (match_operand:SI 2 "s_register_operand")
9989   (match_operand:SI 3 "mve_imm_selective_upto_8")
9990   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9991  "TARGET_HAVE_MVE"
9992 {
9993   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9994   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9995                                               operands[1], operands[2],
9996                                               operands[3]));
9997   DONE;
9998 })
9999
10000 ;;
10001 ;; [viwdupq_wb_u_insn])
10002 ;;
10003 (define_insn "mve_viwdupq_wb_u<mode>_insn"
10004   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10005         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10006                        (match_operand:SI 3 "s_register_operand" "r")
10007                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
10008          VIWDUPQ))
10009    (set (match_operand:SI 1 "s_register_operand" "=e")
10010         (unspec:SI [(match_dup 2)
10011                     (match_dup 3)
10012                     (match_dup 4)]
10013          VIWDUPQ))]
10014   "TARGET_HAVE_MVE"
10015   "viwdup.u%#<V_sz_elem>\t%q0, %2, %3, %4"
10016 )
10017
10018 ;;
10019 ;; [viwdupq_m_n_u])
10020 ;;
10021 (define_expand "mve_viwdupq_m_n_u<mode>"
10022  [(match_operand:MVE_2 0 "s_register_operand")
10023   (match_operand:MVE_2 1 "s_register_operand")
10024   (match_operand:SI 2 "s_register_operand")
10025   (match_operand:SI 3 "s_register_operand")
10026   (match_operand:SI 4 "mve_imm_selective_upto_8")
10027   (match_operand:HI 5 "vpr_register_operand")]
10028  "TARGET_HAVE_MVE"
10029 {
10030   rtx ignore_wb = gen_reg_rtx (SImode);
10031   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
10032                                                 operands[1], operands[2],
10033                                                 operands[3], operands[4],
10034                                                 operands[5]));
10035   DONE;
10036 })
10037
10038 ;;
10039 ;; [viwdupq_m_wb_u])
10040 ;;
10041 (define_expand "mve_viwdupq_m_wb_u<mode>"
10042  [(match_operand:SI 0 "s_register_operand")
10043   (match_operand:MVE_2 1 "s_register_operand")
10044   (match_operand:SI 2 "s_register_operand")
10045   (match_operand:SI 3 "s_register_operand")
10046   (match_operand:SI 4 "mve_imm_selective_upto_8")
10047   (match_operand:HI 5 "vpr_register_operand")]
10048  "TARGET_HAVE_MVE"
10049 {
10050   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10051   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
10052                                                 operands[1], operands[2],
10053                                                 operands[3], operands[4],
10054                                                 operands[5]));
10055   DONE;
10056 })
10057
10058 ;;
10059 ;; [viwdupq_m_wb_u_insn])
10060 ;;
10061 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
10062   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10063         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "w")
10064                        (match_operand:SI 3 "s_register_operand" "1")
10065                        (match_operand:SI 4 "s_register_operand" "r")
10066                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
10067                        (match_operand:HI 6 "vpr_register_operand" "Up")]
10068          VIWDUPQ_M))
10069    (set (match_operand:SI 1 "s_register_operand" "=e")
10070         (unspec:SI [(match_dup 2)
10071                     (match_dup 3)
10072                     (match_dup 4)
10073                     (match_dup 5)
10074                     (match_dup 6)]
10075          VIWDUPQ_M))
10076   ]
10077   "TARGET_HAVE_MVE"
10078   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %4, %5"
10079   [(set_attr "type" "mve_move")
10080    (set_attr "length""8")])
10081 (define_expand "mve_vstrwq_scatter_base_wb_<supf>v4si"
10082   [(match_operand:V4SI 0 "s_register_operand" "=w")
10083    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10084    (match_operand:V4SI 2 "s_register_operand" "w")
10085    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10086   "TARGET_HAVE_MVE"
10087 {
10088   rtx ignore_wb = gen_reg_rtx (V4SImode);
10089   emit_insn (
10090   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (ignore_wb, operands[0],
10091                                                   operands[1], operands[2]));
10092   DONE;
10093 })
10094
10095 (define_expand "mve_vstrwq_scatter_base_wb_add_<supf>v4si"
10096   [(match_operand:V4SI 0 "s_register_operand" "=w")
10097    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10098    (match_operand:V4SI 2 "s_register_operand" "0")
10099    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10100   "TARGET_HAVE_MVE"
10101 {
10102   rtx ignore_vec = gen_reg_rtx (V4SImode);
10103   emit_insn (
10104   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (operands[0], operands[2],
10105                                                   operands[1], ignore_vec));
10106   DONE;
10107 })
10108
10109 ;;
10110 ;; [vstrwq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10111 ;;
10112 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si_insn"
10113   [(set (mem:BLK (scratch))
10114         (unspec:BLK
10115                 [(match_operand:V4SI 1 "s_register_operand" "0")
10116                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10117                  (match_operand:V4SI 3 "s_register_operand" "w")]
10118          VSTRWSBWBQ))
10119    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10120         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10121          VSTRWSBWBQ))
10122   ]
10123   "TARGET_HAVE_MVE"
10124 {
10125    rtx ops[3];
10126    ops[0] = operands[1];
10127    ops[1] = operands[2];
10128    ops[2] = operands[3];
10129    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10130    return "";
10131 }
10132   [(set_attr "length" "4")])
10133
10134 (define_expand "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
10135   [(match_operand:V4SI 0 "s_register_operand" "=w")
10136    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10137    (match_operand:V4SI 2 "s_register_operand" "w")
10138    (match_operand:HI 3 "vpr_register_operand")
10139    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10140   "TARGET_HAVE_MVE"
10141 {
10142   rtx ignore_wb = gen_reg_rtx (V4SImode);
10143   emit_insn (
10144   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (ignore_wb, operands[0],
10145                                                     operands[1], operands[2],
10146                                                     operands[3]));
10147   DONE;
10148 })
10149
10150 (define_expand "mve_vstrwq_scatter_base_wb_p_add_<supf>v4si"
10151   [(match_operand:V4SI 0 "s_register_operand" "=w")
10152    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10153    (match_operand:V4SI 2 "s_register_operand" "0")
10154    (match_operand:HI 3 "vpr_register_operand")
10155    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10156   "TARGET_HAVE_MVE"
10157 {
10158   rtx ignore_vec = gen_reg_rtx (V4SImode);
10159   emit_insn (
10160   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (operands[0], operands[2],
10161                                                     operands[1], ignore_vec,
10162                                                     operands[3]));
10163   DONE;
10164 })
10165
10166 ;;
10167 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
10168 ;;
10169 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn"
10170  [(set (mem:BLK (scratch))
10171        (unspec:BLK
10172                 [(match_operand:V4SI 1 "s_register_operand" "0")
10173                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10174                  (match_operand:V4SI 3 "s_register_operand" "w")
10175                  (match_operand:HI 4 "vpr_register_operand")]
10176         VSTRWSBWBQ))
10177    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10178         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10179          VSTRWSBWBQ))
10180   ]
10181   "TARGET_HAVE_MVE"
10182 {
10183    rtx ops[3];
10184    ops[0] = operands[1];
10185    ops[1] = operands[2];
10186    ops[2] = operands[3];
10187    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10188    return "";
10189 }
10190   [(set_attr "length" "8")])
10191
10192 (define_expand "mve_vstrwq_scatter_base_wb_fv4sf"
10193   [(match_operand:V4SI 0 "s_register_operand" "=w")
10194    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10195    (match_operand:V4SF 2 "s_register_operand" "w")
10196    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10197   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10198 {
10199   rtx ignore_wb = gen_reg_rtx (V4SImode);
10200   emit_insn (
10201   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (ignore_wb,operands[0],
10202                                              operands[1], operands[2]));
10203   DONE;
10204 })
10205
10206 (define_expand "mve_vstrwq_scatter_base_wb_add_fv4sf"
10207   [(match_operand:V4SI 0 "s_register_operand" "=w")
10208    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10209    (match_operand:V4SI 2 "s_register_operand" "0")
10210    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10211   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10212 {
10213   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10214   emit_insn (
10215   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (operands[0], operands[2],
10216                                              operands[1], ignore_vec));
10217   DONE;
10218 })
10219
10220 ;;
10221 ;; [vstrwq_scatter_base_wb_f]
10222 ;;
10223 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf_insn"
10224  [(set (mem:BLK (scratch))
10225        (unspec:BLK
10226                 [(match_operand:V4SI 1 "s_register_operand" "0")
10227                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10228                  (match_operand:V4SF 3 "s_register_operand" "w")]
10229          VSTRWQSBWB_F))
10230    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10231         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10232          VSTRWQSBWB_F))
10233   ]
10234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10235 {
10236    rtx ops[3];
10237    ops[0] = operands[1];
10238    ops[1] = operands[2];
10239    ops[2] = operands[3];
10240    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10241    return "";
10242 }
10243   [(set_attr "length" "4")])
10244
10245 (define_expand "mve_vstrwq_scatter_base_wb_p_fv4sf"
10246   [(match_operand:V4SI 0 "s_register_operand" "=w")
10247    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10248    (match_operand:V4SF 2 "s_register_operand" "w")
10249    (match_operand:HI 3 "vpr_register_operand")
10250    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10251   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10252 {
10253   rtx ignore_wb = gen_reg_rtx (V4SImode);
10254   emit_insn (
10255   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (ignore_wb, operands[0],
10256                                                operands[1], operands[2],
10257                                                operands[3]));
10258   DONE;
10259 })
10260
10261 (define_expand "mve_vstrwq_scatter_base_wb_p_add_fv4sf"
10262   [(match_operand:V4SI 0 "s_register_operand" "=w")
10263    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10264    (match_operand:V4SI 2 "s_register_operand" "0")
10265    (match_operand:HI 3 "vpr_register_operand")
10266    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10267   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10268 {
10269   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10270   emit_insn (
10271   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (operands[0], operands[2],
10272                                                operands[1], ignore_vec,
10273                                                operands[3]));
10274   DONE;
10275 })
10276
10277 ;;
10278 ;; [vstrwq_scatter_base_wb_p_f]
10279 ;;
10280 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf_insn"
10281  [(set (mem:BLK (scratch))
10282        (unspec:BLK
10283                 [(match_operand:V4SI 1 "s_register_operand" "0")
10284                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10285                  (match_operand:V4SF 3 "s_register_operand" "w")
10286                  (match_operand:HI 4 "vpr_register_operand")]
10287         VSTRWQSBWB_F))
10288    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10289         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10290          VSTRWQSBWB_F))
10291   ]
10292   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10293 {
10294    rtx ops[3];
10295    ops[0] = operands[1];
10296    ops[1] = operands[2];
10297    ops[2] = operands[3];
10298    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10299    return "";
10300 }
10301   [(set_attr "length" "8")])
10302
10303 (define_expand "mve_vstrdq_scatter_base_wb_<supf>v2di"
10304   [(match_operand:V2DI 0 "s_register_operand" "=w")
10305    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10306    (match_operand:V2DI 2 "s_register_operand" "w")
10307    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10308   "TARGET_HAVE_MVE"
10309 {
10310   rtx ignore_wb = gen_reg_rtx (V2DImode);
10311   emit_insn (
10312   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (ignore_wb, operands[0],
10313                                                   operands[1], operands[2]));
10314   DONE;
10315 })
10316
10317 (define_expand "mve_vstrdq_scatter_base_wb_add_<supf>v2di"
10318   [(match_operand:V2DI 0 "s_register_operand" "=w")
10319    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10320    (match_operand:V2DI 2 "s_register_operand" "0")
10321    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10322   "TARGET_HAVE_MVE"
10323 {
10324   rtx ignore_vec = gen_reg_rtx (V2DImode);
10325   emit_insn (
10326   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (operands[0], operands[2],
10327                                                   operands[1], ignore_vec));
10328   DONE;
10329 })
10330
10331 ;;
10332 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10333 ;;
10334 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di_insn"
10335   [(set (mem:BLK (scratch))
10336         (unspec:BLK
10337                 [(match_operand:V2DI 1 "s_register_operand" "0")
10338                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10339                  (match_operand:V2DI 3 "s_register_operand" "w")]
10340          VSTRDSBWBQ))
10341    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
10342         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10343          VSTRDSBWBQ))
10344   ]
10345   "TARGET_HAVE_MVE"
10346 {
10347    rtx ops[3];
10348    ops[0] = operands[1];
10349    ops[1] = operands[2];
10350    ops[2] = operands[3];
10351    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
10352    return "";
10353 }
10354   [(set_attr "length" "4")])
10355
10356 (define_expand "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
10357   [(match_operand:V2DI 0 "s_register_operand" "=w")
10358    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10359    (match_operand:V2DI 2 "s_register_operand" "w")
10360    (match_operand:HI 3 "vpr_register_operand")
10361    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10362   "TARGET_HAVE_MVE"
10363 {
10364   rtx ignore_wb = gen_reg_rtx (V2DImode);
10365   emit_insn (
10366   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (ignore_wb, operands[0],
10367                                                     operands[1], operands[2],
10368                                                     operands[3]));
10369   DONE;
10370 })
10371
10372 (define_expand "mve_vstrdq_scatter_base_wb_p_add_<supf>v2di"
10373   [(match_operand:V2DI 0 "s_register_operand" "=w")
10374    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10375    (match_operand:V2DI 2 "s_register_operand" "0")
10376    (match_operand:HI 3 "vpr_register_operand")
10377    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10378   "TARGET_HAVE_MVE"
10379 {
10380   rtx ignore_vec = gen_reg_rtx (V2DImode);
10381   emit_insn (
10382   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (operands[0], operands[2],
10383                                                     operands[1], ignore_vec,
10384                                                     operands[3]));
10385   DONE;
10386 })
10387
10388 ;;
10389 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
10390 ;;
10391 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn"
10392   [(set (mem:BLK (scratch))
10393         (unspec:BLK
10394                 [(match_operand:V2DI 1 "s_register_operand" "0")
10395                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10396                  (match_operand:V2DI 3 "s_register_operand" "w")
10397                  (match_operand:HI 4 "vpr_register_operand")]
10398          VSTRDSBWBQ))
10399    (set (match_operand:V2DI 0 "s_register_operand" "=w")
10400         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10401          VSTRDSBWBQ))
10402   ]
10403   "TARGET_HAVE_MVE"
10404 {
10405    rtx ops[3];
10406    ops[0] = operands[1];
10407    ops[1] = operands[2];
10408    ops[2] = operands[3];
10409    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]!",ops);
10410    return "";
10411 }
10412   [(set_attr "length" "8")])
10413
10414 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
10415   [(match_operand:V4SI 0 "s_register_operand")
10416    (match_operand:V4SI 1 "s_register_operand")
10417    (match_operand:SI 2 "mve_vldrd_immediate")
10418    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10419   "TARGET_HAVE_MVE"
10420 {
10421   rtx ignore_wb = gen_reg_rtx (V4SImode);
10422   emit_insn (
10423   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
10424                                                  operands[1], operands[2]));
10425   DONE;
10426 })
10427
10428 ;;
10429 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
10430 ;;
10431 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
10432   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10433         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10434                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10435                       (mem:BLK (scratch))]
10436          VLDRWGBWBQ))
10437    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10438         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10439          VLDRWGBWBQ))
10440   ]
10441   "TARGET_HAVE_MVE"
10442 {
10443    rtx ops[3];
10444    ops[0] = operands[0];
10445    ops[1] = operands[2];
10446    ops[2] = operands[3];
10447    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10448    return "";
10449 }
10450   [(set_attr "length" "4")])
10451
10452 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
10453   [(match_operand:V4SI 0 "s_register_operand")
10454    (match_operand:V4SI 1 "s_register_operand")
10455    (match_operand:SI 2 "mve_vldrd_immediate")
10456    (match_operand:HI 3 "vpr_register_operand")
10457    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10458   "TARGET_HAVE_MVE"
10459 {
10460   rtx ignore_wb = gen_reg_rtx (V4SImode);
10461   emit_insn (
10462   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10463                                                    operands[1], operands[2],
10464                                                    operands[3]));
10465   DONE;
10466 })
10467
10468 ;;
10469 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10470 ;;
10471 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10472   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10473         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10474                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10475                       (match_operand:HI 4 "vpr_register_operand" "Up")
10476                       (mem:BLK (scratch))]
10477          VLDRWGBWBQ))
10478    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10479         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10480          VLDRWGBWBQ))
10481   ]
10482   "TARGET_HAVE_MVE"
10483 {
10484    rtx ops[3];
10485    ops[0] = operands[0];
10486    ops[1] = operands[2];
10487    ops[2] = operands[3];
10488    output_asm_insn ("vpst\;\tvldrwt.u32\t%q0, [%q1, %2]!",ops);
10489    return "";
10490 }
10491   [(set_attr "length" "8")])
10492
10493 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10494   [(match_operand:V4SF 0 "s_register_operand")
10495    (match_operand:V4SI 1 "s_register_operand")
10496    (match_operand:SI 2 "mve_vldrd_immediate")
10497    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10498   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10499 {
10500   rtx ignore_wb = gen_reg_rtx (V4SImode);
10501   emit_insn (
10502   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10503                                             operands[1], operands[2]));
10504   DONE;
10505 })
10506
10507 ;;
10508 ;; [vldrwq_gather_base_wb_f]
10509 ;;
10510 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10511   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10512         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10513                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10514                       (mem:BLK (scratch))]
10515          VLDRWQGBWB_F))
10516    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10517         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10518          VLDRWQGBWB_F))
10519   ]
10520   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10521 {
10522    rtx ops[3];
10523    ops[0] = operands[0];
10524    ops[1] = operands[2];
10525    ops[2] = operands[3];
10526    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10527    return "";
10528 }
10529   [(set_attr "length" "4")])
10530
10531 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10532   [(match_operand:V4SF 0 "s_register_operand")
10533    (match_operand:V4SI 1 "s_register_operand")
10534    (match_operand:SI 2 "mve_vldrd_immediate")
10535    (match_operand:HI 3 "vpr_register_operand")
10536    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10537   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10538 {
10539   rtx ignore_wb = gen_reg_rtx (V4SImode);
10540   emit_insn (
10541   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10542                                               operands[1], operands[2],
10543                                               operands[3]));
10544   DONE;
10545 })
10546
10547 ;;
10548 ;; [vldrwq_gather_base_wb_z_f]
10549 ;;
10550 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10551   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10552         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10553                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10554                       (match_operand:HI 4 "vpr_register_operand" "Up")
10555                       (mem:BLK (scratch))]
10556          VLDRWQGBWB_F))
10557    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10558         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10559          VLDRWQGBWB_F))
10560   ]
10561   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10562 {
10563    rtx ops[3];
10564    ops[0] = operands[0];
10565    ops[1] = operands[2];
10566    ops[2] = operands[3];
10567    output_asm_insn ("vpst\;\tvldrwt.u32\t%q0, [%q1, %2]!",ops);
10568    return "";
10569 }
10570   [(set_attr "length" "8")])
10571
10572 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10573   [(match_operand:V2DI 0 "s_register_operand")
10574    (match_operand:V2DI 1 "s_register_operand")
10575    (match_operand:SI 2 "mve_vldrd_immediate")
10576    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10577   "TARGET_HAVE_MVE"
10578 {
10579   rtx ignore_wb = gen_reg_rtx (V2DImode);
10580   emit_insn (
10581   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10582                                                  operands[1], operands[2]));
10583   DONE;
10584 })
10585
10586 ;;
10587 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10588 ;;
10589 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10590   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10591         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10592                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10593                       (mem:BLK (scratch))]
10594          VLDRDGBWBQ))
10595    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10596         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10597          VLDRDGBWBQ))
10598   ]
10599   "TARGET_HAVE_MVE"
10600 {
10601    rtx ops[3];
10602    ops[0] = operands[0];
10603    ops[1] = operands[2];
10604    ops[2] = operands[3];
10605    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10606    return "";
10607 }
10608   [(set_attr "length" "4")])
10609
10610 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10611   [(match_operand:V2DI 0 "s_register_operand")
10612    (match_operand:V2DI 1 "s_register_operand")
10613    (match_operand:SI 2 "mve_vldrd_immediate")
10614    (match_operand:HI 3 "vpr_register_operand")
10615    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10616   "TARGET_HAVE_MVE"
10617 {
10618   rtx ignore_wb = gen_reg_rtx (V2DImode);
10619   emit_insn (
10620   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10621                                                    operands[1], operands[2],
10622                                                    operands[3]));
10623   DONE;
10624 })
10625
10626 (define_insn "get_fpscr_nzcvqc"
10627  [(set (match_operand:SI 0 "register_operand" "=r")
10628    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10629  "TARGET_HAVE_MVE"
10630  "vmrs\\t%0, FPSCR_nzcvqc"
10631   [(set_attr "type" "mve_move")])
10632
10633 (define_insn "set_fpscr_nzcvqc"
10634  [(set (reg:SI VFPCC_REGNUM)
10635    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10636     VUNSPEC_SET_FPSCR_NZCVQC))]
10637  "TARGET_HAVE_MVE"
10638  "vmsr\\tFPSCR_nzcvqc, %0"
10639   [(set_attr "type" "mve_move")])
10640
10641 ;;
10642 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10643 ;;
10644 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10645   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10646         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10647                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10648                       (match_operand:HI 4 "vpr_register_operand" "Up")
10649                       (mem:BLK (scratch))]
10650          VLDRDGBWBQ))
10651    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10652         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10653          VLDRDGBWBQ))
10654   ]
10655   "TARGET_HAVE_MVE"
10656 {
10657    rtx ops[3];
10658    ops[0] = operands[0];
10659    ops[1] = operands[2];
10660    ops[2] = operands[3];
10661    output_asm_insn ("vpst\;\tvldrdt.u64\t%q0, [%q1, %2]!",ops);
10662    return "";
10663 }
10664   [(set_attr "length" "8")])
10665 ;;
10666 ;; [vadciq_m_s, vadciq_m_u])
10667 ;;
10668 (define_insn "mve_vadciq_m_<supf>v4si"
10669   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10670         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10671                       (match_operand:V4SI 2 "s_register_operand" "w")
10672                       (match_operand:V4SI 3 "s_register_operand" "w")
10673                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10674          VADCIQ_M))
10675    (set (reg:SI VFPCC_REGNUM)
10676         (unspec:SI [(const_int 0)]
10677          VADCIQ_M))
10678   ]
10679   "TARGET_HAVE_MVE"
10680   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10681   [(set_attr "type" "mve_move")
10682    (set_attr "length" "8")])
10683
10684 ;;
10685 ;; [vadciq_u, vadciq_s])
10686 ;;
10687 (define_insn "mve_vadciq_<supf>v4si"
10688   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10689         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10690                       (match_operand:V4SI 2 "s_register_operand" "w")]
10691          VADCIQ))
10692    (set (reg:SI VFPCC_REGNUM)
10693         (unspec:SI [(const_int 0)]
10694          VADCIQ))
10695   ]
10696   "TARGET_HAVE_MVE"
10697   "vadci.i32\t%q0, %q1, %q2"
10698   [(set_attr "type" "mve_move")
10699    (set_attr "length" "4")])
10700
10701 ;;
10702 ;; [vadcq_m_s, vadcq_m_u])
10703 ;;
10704 (define_insn "mve_vadcq_m_<supf>v4si"
10705   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10706         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10707                       (match_operand:V4SI 2 "s_register_operand" "w")
10708                       (match_operand:V4SI 3 "s_register_operand" "w")
10709                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10710          VADCQ_M))
10711    (set (reg:SI VFPCC_REGNUM)
10712         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10713          VADCQ_M))
10714   ]
10715   "TARGET_HAVE_MVE"
10716   "vpst\;vadct.i32\t%q0, %q2, %q3"
10717   [(set_attr "type" "mve_move")
10718    (set_attr "length" "8")])
10719
10720 ;;
10721 ;; [vadcq_u, vadcq_s])
10722 ;;
10723 (define_insn "mve_vadcq_<supf>v4si"
10724   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10725         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10726                        (match_operand:V4SI 2 "s_register_operand" "w")]
10727          VADCQ))
10728    (set (reg:SI VFPCC_REGNUM)
10729         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10730          VADCQ))
10731   ]
10732   "TARGET_HAVE_MVE"
10733   "vadc.i32\t%q0, %q1, %q2"
10734   [(set_attr "type" "mve_move")
10735    (set_attr "length" "4")
10736    (set_attr "conds" "set")])
10737
10738 ;;
10739 ;; [vsbciq_m_u, vsbciq_m_s])
10740 ;;
10741 (define_insn "mve_vsbciq_m_<supf>v4si"
10742   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10743         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10744                       (match_operand:V4SI 2 "s_register_operand" "w")
10745                       (match_operand:V4SI 3 "s_register_operand" "w")
10746                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10747          VSBCIQ_M))
10748    (set (reg:SI VFPCC_REGNUM)
10749         (unspec:SI [(const_int 0)]
10750          VSBCIQ_M))
10751   ]
10752   "TARGET_HAVE_MVE"
10753   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10754   [(set_attr "type" "mve_move")
10755    (set_attr "length" "8")])
10756
10757 ;;
10758 ;; [vsbciq_s, vsbciq_u])
10759 ;;
10760 (define_insn "mve_vsbciq_<supf>v4si"
10761   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10762         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10763                       (match_operand:V4SI 2 "s_register_operand" "w")]
10764          VSBCIQ))
10765    (set (reg:SI VFPCC_REGNUM)
10766         (unspec:SI [(const_int 0)]
10767          VSBCIQ))
10768   ]
10769   "TARGET_HAVE_MVE"
10770   "vsbci.i32\t%q0, %q1, %q2"
10771   [(set_attr "type" "mve_move")
10772    (set_attr "length" "4")])
10773
10774 ;;
10775 ;; [vsbcq_m_u, vsbcq_m_s])
10776 ;;
10777 (define_insn "mve_vsbcq_m_<supf>v4si"
10778   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10779         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10780                       (match_operand:V4SI 2 "s_register_operand" "w")
10781                       (match_operand:V4SI 3 "s_register_operand" "w")
10782                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10783          VSBCQ_M))
10784    (set (reg:SI VFPCC_REGNUM)
10785         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10786          VSBCQ_M))
10787   ]
10788   "TARGET_HAVE_MVE"
10789   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10790   [(set_attr "type" "mve_move")
10791    (set_attr "length" "8")])
10792
10793 ;;
10794 ;; [vsbcq_s, vsbcq_u])
10795 ;;
10796 (define_insn "mve_vsbcq_<supf>v4si"
10797   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10798         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10799                       (match_operand:V4SI 2 "s_register_operand" "w")]
10800          VSBCQ))
10801    (set (reg:SI VFPCC_REGNUM)
10802         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10803          VSBCQ))
10804   ]
10805   "TARGET_HAVE_MVE"
10806   "vsbc.i32\t%q0, %q1, %q2"
10807   [(set_attr "type" "mve_move")
10808    (set_attr "length" "4")])
10809
10810 ;;
10811 ;; [vst2q])
10812 ;;
10813 (define_insn "mve_vst2q<mode>"
10814   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10815         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10816                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10817          VST2Q))
10818   ]
10819   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10820    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10821 {
10822    rtx ops[4];
10823    int regno = REGNO (operands[1]);
10824    ops[0] = gen_rtx_REG (TImode, regno);
10825    ops[1] = gen_rtx_REG (TImode, regno + 4);
10826    rtx reg  = operands[0];
10827    while (reg && !REG_P (reg))
10828     reg = XEXP (reg, 0);
10829    gcc_assert (REG_P (reg));
10830    ops[2] = reg;
10831    ops[3] = operands[0];
10832    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10833                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10834    return "";
10835 }
10836   [(set_attr "length" "8")])
10837
10838 ;;
10839 ;; [vld2q])
10840 ;;
10841 (define_insn "mve_vld2q<mode>"
10842   [(set (match_operand:OI 0 "s_register_operand" "=w")
10843         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10844                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10845          VLD2Q))
10846   ]
10847   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10848    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10849 {
10850    rtx ops[4];
10851    int regno = REGNO (operands[0]);
10852    ops[0] = gen_rtx_REG (TImode, regno);
10853    ops[1] = gen_rtx_REG (TImode, regno + 4);
10854    rtx reg  = operands[1];
10855    while (reg && !REG_P (reg))
10856     reg = XEXP (reg, 0);
10857    gcc_assert (REG_P (reg));
10858    ops[2] = reg;
10859    ops[3] = operands[1];
10860    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10861                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10862    return "";
10863 }
10864   [(set_attr "length" "8")])
10865
10866 ;;
10867 ;; [vld4q])
10868 ;;
10869 (define_insn "mve_vld4q<mode>"
10870   [(set (match_operand:XI 0 "s_register_operand" "=w")
10871         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10872                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10873          VLD4Q))
10874   ]
10875   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10876    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10877 {
10878    rtx ops[6];
10879    int regno = REGNO (operands[0]);
10880    ops[0] = gen_rtx_REG (TImode, regno);
10881    ops[1] = gen_rtx_REG (TImode, regno+4);
10882    ops[2] = gen_rtx_REG (TImode, regno+8);
10883    ops[3] = gen_rtx_REG (TImode, regno + 12);
10884    rtx reg  = operands[1];
10885    while (reg && !REG_P (reg))
10886     reg = XEXP (reg, 0);
10887    gcc_assert (REG_P (reg));
10888    ops[4] = reg;
10889    ops[5] = operands[1];
10890    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10891                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10892                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10893                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10894    return "";
10895 }
10896   [(set_attr "length" "16")])
10897 ;;
10898 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10899 ;;
10900 (define_insn "mve_vec_extract<mode><V_elem_l>"
10901  [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
10902    (vec_select:<V_elem>
10903     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10904     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10905   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10906    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10907 {
10908   if (BYTES_BIG_ENDIAN)
10909     {
10910       int elt = INTVAL (operands[2]);
10911       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10912       operands[2] = GEN_INT (elt);
10913     }
10914   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10915 }
10916  [(set_attr "type" "mve_move")])
10917
10918 (define_insn "mve_vec_extractv2didi"
10919  [(set (match_operand:DI 0 "s_register_operand" "=r")
10920    (vec_select:DI
10921     (match_operand:V2DI 1 "s_register_operand" "w")
10922     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10923   "TARGET_HAVE_MVE"
10924 {
10925   int elt = INTVAL (operands[2]);
10926   if (BYTES_BIG_ENDIAN)
10927     elt = 1 - elt;
10928
10929   if (elt == 0)
10930    return "vmov\t%Q0, %R0, %e1";
10931   else
10932    return "vmov\t%J0, %K0, %f1";
10933 }
10934  [(set_attr "type" "mve_move")])
10935
10936 (define_insn "*mve_vec_extract_sext_internal<mode>"
10937  [(set (match_operand:SI 0 "s_register_operand" "=r")
10938    (sign_extend:SI
10939     (vec_select:<V_elem>
10940      (match_operand:MVE_2 1 "s_register_operand" "w")
10941      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10942   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10943    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10944 {
10945   if (BYTES_BIG_ENDIAN)
10946     {
10947       int elt = INTVAL (operands[2]);
10948       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10949       operands[2] = GEN_INT (elt);
10950     }
10951   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10952 }
10953  [(set_attr "type" "mve_move")])
10954
10955 (define_insn "*mve_vec_extract_zext_internal<mode>"
10956  [(set (match_operand:SI 0 "s_register_operand" "=r")
10957    (zero_extend:SI
10958     (vec_select:<V_elem>
10959      (match_operand:MVE_2 1 "s_register_operand" "w")
10960      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10961   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10962    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10963 {
10964   if (BYTES_BIG_ENDIAN)
10965     {
10966       int elt = INTVAL (operands[2]);
10967       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10968       operands[2] = GEN_INT (elt);
10969     }
10970   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10971 }
10972  [(set_attr "type" "mve_move")])
10973
10974 ;;
10975 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10976 ;;
10977 (define_insn "mve_vec_set<mode>_internal"
10978  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10979        (vec_merge:VQ2
10980         (vec_duplicate:VQ2
10981           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10982         (match_operand:VQ2 3 "s_register_operand" "0")
10983         (match_operand:SI 2 "immediate_operand" "i")))]
10984   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10985    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10986 {
10987   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10988   if (BYTES_BIG_ENDIAN)
10989     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10990   operands[2] = GEN_INT (elt);
10991
10992   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10993 }
10994  [(set_attr "type" "mve_move")])
10995
10996 (define_insn "mve_vec_setv2di_internal"
10997  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10998        (vec_merge:V2DI
10999         (vec_duplicate:V2DI
11000           (match_operand:DI 1 "nonimmediate_operand" "r"))
11001         (match_operand:V2DI 3 "s_register_operand" "0")
11002         (match_operand:SI 2 "immediate_operand" "i")))]
11003  "TARGET_HAVE_MVE"
11004 {
11005   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11006   if (BYTES_BIG_ENDIAN)
11007     elt = 1 - elt;
11008
11009   if (elt == 0)
11010    return "vmov\t%e0, %Q1, %R1";
11011   else
11012    return "vmov\t%f0, %J1, %K1";
11013 }
11014  [(set_attr "type" "mve_move")])
11015
11016 ;;
11017 ;; [uqrshll_di]
11018 ;;
11019 (define_insn "mve_uqrshll_sat<supf>_di"
11020   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11021         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11022                     (match_operand:SI 2 "s_register_operand" "r")]
11023          UQRSHLLQ))]
11024   "TARGET_HAVE_MVE"
11025   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
11026   [(set_attr "predicable" "yes")])
11027
11028 ;;
11029 ;; [sqrshrl_di]
11030 ;;
11031 (define_insn "mve_sqrshrl_sat<supf>_di"
11032   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11033         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11034                     (match_operand:SI 2 "s_register_operand" "r")]
11035          SQRSHRLQ))]
11036   "TARGET_HAVE_MVE"
11037   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
11038   [(set_attr "predicable" "yes")])
11039
11040 ;;
11041 ;; [uqrshl_si]
11042 ;;
11043 (define_insn "mve_uqrshl_si"
11044   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11045         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11046                     (match_operand:SI 2 "s_register_operand" "r")]
11047          UQRSHL))]
11048   "TARGET_HAVE_MVE"
11049   "uqrshl%?\\t%1, %2"
11050   [(set_attr "predicable" "yes")])
11051
11052 ;;
11053 ;; [sqrshr_si]
11054 ;;
11055 (define_insn "mve_sqrshr_si"
11056   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11057         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11058                     (match_operand:SI 2 "s_register_operand" "r")]
11059          SQRSHR))]
11060   "TARGET_HAVE_MVE"
11061   "sqrshr%?\\t%1, %2"
11062   [(set_attr "predicable" "yes")])
11063
11064 ;;
11065 ;; [uqshll_di]
11066 ;;
11067 (define_insn "mve_uqshll_di"
11068   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11069         (us_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11070                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11071   "TARGET_HAVE_MVE"
11072   "uqshll%?\\t%Q1, %R1, %2"
11073   [(set_attr "predicable" "yes")])
11074
11075 ;;
11076 ;; [urshrl_di]
11077 ;;
11078 (define_insn "mve_urshrl_di"
11079   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11080         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11081                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11082          URSHRL))]
11083   "TARGET_HAVE_MVE"
11084   "urshrl%?\\t%Q1, %R1, %2"
11085   [(set_attr "predicable" "yes")])
11086
11087 ;;
11088 ;; [uqshl_si]
11089 ;;
11090 (define_insn "mve_uqshl_si"
11091   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11092         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "r")
11093                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11094   "TARGET_HAVE_MVE"
11095   "uqshl%?\\t%1, %2"
11096   [(set_attr "predicable" "yes")])
11097
11098 ;;
11099 ;; [urshr_si]
11100 ;;
11101 (define_insn "mve_urshr_si"
11102   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11103         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11104                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11105          URSHR))]
11106   "TARGET_HAVE_MVE"
11107   "urshr%?\\t%1, %2"
11108   [(set_attr "predicable" "yes")])
11109
11110 ;;
11111 ;; [sqshl_si]
11112 ;;
11113 (define_insn "mve_sqshl_si"
11114   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11115         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "r")
11116                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11117   "TARGET_HAVE_MVE"
11118   "sqshl%?\\t%1, %2"
11119   [(set_attr "predicable" "yes")])
11120
11121 ;;
11122 ;; [srshr_si]
11123 ;;
11124 (define_insn "mve_srshr_si"
11125   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11126         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "r")
11127                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11128          SRSHR))]
11129   "TARGET_HAVE_MVE"
11130   "srshr%?\\t%1, %2"
11131   [(set_attr "predicable" "yes")])
11132
11133 ;;
11134 ;; [srshrl_di]
11135 ;;
11136 (define_insn "mve_srshrl_di"
11137   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11138         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11139                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11140          SRSHRL))]
11141   "TARGET_HAVE_MVE"
11142   "srshrl%?\\t%Q1, %R1, %2"
11143   [(set_attr "predicable" "yes")])
11144
11145 ;;
11146 ;; [sqshll_di]
11147 ;;
11148 (define_insn "mve_sqshll_di"
11149   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11150         (ss_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11151                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11152   "TARGET_HAVE_MVE"
11153   "sqshll%?\\t%Q1, %R1, %2"
11154   [(set_attr "predicable" "yes")])