gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_mode_iterator MVE_types [V16QI V8HI V4SI V2DI TI V8HF V4SF V2DF])
  21 (define_mode_iterator MVE_VLD_ST [V16QI V8HI V4SI V8HF V4SF])
  22 (define_mode_iterator MVE_0 [V8HF V4SF])
  23 (define_mode_iterator MVE_1 [V16QI V8HI V4SI V2DI])
  24 (define_mode_iterator MVE_3 [V16QI V8HI])
  25 (define_mode_iterator MVE_2 [V16QI V8HI V4SI])
  26 (define_mode_iterator MVE_5 [V8HI V4SI])
  27 (define_mode_iterator MVE_6 [V8HI V4SI])
  28
  29 (define_c_enum "unspec" [VST4Q VRNDXQ_F VRNDQ_F VRNDPQ_F VRNDNQ_F VRNDMQ_F
  30                          VRNDAQ_F VREV64Q_F VNEGQ_F VDUPQ_N_F VABSQ_F VREV32Q_F
  31                          VCVTTQ_F32_F16 VCVTBQ_F32_F16 VCVTQ_TO_F_S VQNEGQ_S
  32                          VCVTQ_TO_F_U VREV16Q_S VREV16Q_U VADDLVQ_S VMVNQ_N_S
  33                          VMVNQ_N_U VCVTAQ_S VCVTAQ_U VREV64Q_S VREV64Q_U
  34                          VQABSQ_S VNEGQ_S VMVNQ_S VMVNQ_U VDUPQ_N_U VDUPQ_N_S
  35                          VCLZQ_U VCLZQ_S VCLSQ_S VADDVQ_S VADDVQ_U VABSQ_S
  36                          VREV32Q_U VREV32Q_S VMOVLTQ_U VMOVLTQ_S VMOVLBQ_S
  37                          VMOVLBQ_U VCVTQ_FROM_F_S VCVTQ_FROM_F_U VCVTPQ_S
  38                          VCVTPQ_U VCVTNQ_S VCVTNQ_U VCVTMQ_S VCVTMQ_U
  39                          VADDLVQ_U VCTP8Q VCTP16Q VCTP32Q VCTP64Q VPNOT
  40                          VCREATEQ_F VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U VBRSRQ_N_F
  41                          VSUBQ_N_F VCREATEQ_U VCREATEQ_S VSHRQ_N_S VSHRQ_N_U
  42                          VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U VADDLVQ_P_S
  43                          VADDLVQ_P_U VCMPNEQ_U VCMPNEQ_S VSHLQ_S VSHLQ_U VABDQ_S
  44                          VADDQ_N_S VADDVAQ_S VADDVQ_P_S VANDQ_S VBICQ_S
  45                          VBRSRQ_N_S VCADDQ_ROT270_S VCADDQ_ROT90_S VCMPEQQ_S
  46                          VCMPEQQ_N_S VCMPNEQ_N_S VEORQ_S VHADDQ_S VHADDQ_N_S
  47                          VHSUBQ_S VHSUBQ_N_S VMAXQ_S VMAXVQ_S VMINQ_S VMINVQ_S
  48                          VMLADAVQ_S VMULHQ_S VMULLBQ_INT_S VMULLTQ_INT_S VMULQ_S
  49                          VMULQ_N_S VORNQ_S VORRQ_S VQADDQ_S VQADDQ_N_S VQRSHLQ_S
  50                          VQRSHLQ_N_S VQSHLQ_S VQSHLQ_N_S VQSHLQ_R_S VQSUBQ_S
  51                          VQSUBQ_N_S VRHADDQ_S VRMULHQ_S VRSHLQ_S VRSHLQ_N_S
  52                          VRSHRQ_N_S VSHLQ_N_S VSHLQ_R_S VSUBQ_S VSUBQ_N_S
  53                          VABDQ_U VADDQ_N_U VADDVAQ_U VADDVQ_P_U VANDQ_U VBICQ_U
  54                          VBRSRQ_N_U VCADDQ_ROT270_U VCADDQ_ROT90_U VCMPEQQ_U
  55                          VCMPEQQ_N_U VCMPNEQ_N_U VEORQ_U VHADDQ_U VHADDQ_N_U
  56                          VHSUBQ_U VHSUBQ_N_U VMAXQ_U VMAXVQ_U VMINQ_U VMINVQ_U
  57                          VMLADAVQ_U VMULHQ_U VMULLBQ_INT_U VMULLTQ_INT_U VMULQ_U
  58                          VMULQ_N_U VORNQ_U VORRQ_U VQADDQ_U VQADDQ_N_U VQRSHLQ_U
  59                          VQRSHLQ_N_U VQSHLQ_U VQSHLQ_N_U VQSHLQ_R_U VQSUBQ_U
  60                          VQSUBQ_N_U VRHADDQ_U VRMULHQ_U VRSHLQ_U VRSHLQ_N_U
  61                          VRSHRQ_N_U VSHLQ_N_U VSHLQ_R_U VSUBQ_U VSUBQ_N_U
  62                          VCMPGEQ_N_S VCMPGEQ_S VCMPGTQ_N_S VCMPGTQ_S VCMPLEQ_N_S
  63                          VCMPLEQ_S VCMPLTQ_N_S VCMPLTQ_S VHCADDQ_ROT270_S
  64                          VHCADDQ_ROT90_S VMAXAQ_S VMAXAVQ_S VMINAQ_S VMINAVQ_S
  65                          VMLADAVXQ_S VMLSDAVQ_S VMLSDAVXQ_S VQDMULHQ_N_S
  66                          VQDMULHQ_S VQRDMULHQ_N_S VQRDMULHQ_S VQSHLUQ_N_S
  67                          VCMPCSQ_N_U VCMPCSQ_U VCMPHIQ_N_U VCMPHIQ_U VABDQ_M_S
  68                          VABDQ_M_U VABDQ_F VADDQ_N_F VANDQ_F VBICQ_F
  69                          VCADDQ_ROT270_F VCADDQ_ROT90_F VCMPEQQ_F VCMPEQQ_N_F
  70                          VCMPGEQ_F VCMPGEQ_N_F VCMPGTQ_F VCMPGTQ_N_F VCMPLEQ_F
  71                          VCMPLEQ_N_F VCMPLTQ_F VCMPLTQ_N_F VCMPNEQ_F VCMPNEQ_N_F
  72                          VCMULQ_F VCMULQ_ROT180_F VCMULQ_ROT270_F VCMULQ_ROT90_F
  73                          VEORQ_F VMAXNMAQ_F VMAXNMAVQ_F VMAXNMQ_F VMAXNMVQ_F
  74                          VMINNMAQ_F VMINNMAVQ_F VMINNMQ_F VMINNMVQ_F VMULQ_F
  75                          VMULQ_N_F VORNQ_F VORRQ_F VSUBQ_F VADDLVAQ_U
  76                          VADDLVAQ_S VBICQ_N_U VBICQ_N_S VCTP8Q_M VCTP16Q_M
  77                          VCTP32Q_M VCTP64Q_M VCVTBQ_F16_F32 VCVTTQ_F16_F32
  78                          VMLALDAVQ_U VMLALDAVXQ_U VMLALDAVXQ_S VMLALDAVQ_S
  79                          VMLSLDAVQ_S VMLSLDAVXQ_S VMOVNBQ_U VMOVNBQ_S
  80                          VMOVNTQ_U VMOVNTQ_S VORRQ_N_S VORRQ_N_U VQDMULLBQ_N_S
  81                          VQDMULLBQ_S VQDMULLTQ_N_S VQDMULLTQ_S VQMOVNBQ_U
  82                          VQMOVNBQ_S VQMOVUNBQ_S VQMOVUNTQ_S VRMLALDAVHXQ_S
  83                          VRMLSLDAVHQ_S VRMLSLDAVHXQ_S VSHLLBQ_S
  84                          VSHLLBQ_U VSHLLTQ_U VSHLLTQ_S VQMOVNTQ_U VQMOVNTQ_S
  85                          VSHLLBQ_N_S VSHLLBQ_N_U VSHLLTQ_N_U VSHLLTQ_N_S
  86                          VRMLALDAVHQ_U VRMLALDAVHQ_S VMULLTQ_POLY_P
  87                          VMULLBQ_POLY_P VBICQ_M_N_S VBICQ_M_N_U VCMPEQQ_M_F
  88                          VCVTAQ_M_S VCVTAQ_M_U VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U
  89                          VQRSHRNBQ_N_U VQRSHRNBQ_N_S VQRSHRUNBQ_N_S
  90                          VRMLALDAVHAQ_S VABAVQ_S VABAVQ_U VSHLCQ_S VSHLCQ_U
  91                          VRMLALDAVHAQ_U VABSQ_M_S VADDVAQ_P_S VADDVAQ_P_U
  92                          VCLSQ_M_S VCLZQ_M_S VCLZQ_M_U VCMPCSQ_M_N_U
  93                          VCMPCSQ_M_U VCMPEQQ_M_N_S VCMPEQQ_M_N_U VCMPEQQ_M_S
  94                          VCMPEQQ_M_U VCMPGEQ_M_N_S VCMPGEQ_M_S VCMPGTQ_M_N_S
  95                          VCMPGTQ_M_S VCMPHIQ_M_N_U VCMPHIQ_M_U VCMPLEQ_M_N_S
  96                          VCMPLEQ_M_S VCMPLTQ_M_N_S VCMPLTQ_M_S VCMPNEQ_M_N_S
  97                          VCMPNEQ_M_N_U VCMPNEQ_M_S VCMPNEQ_M_U VDUPQ_M_N_S
  98                          VDUPQ_M_N_U VDWDUPQ_N_U VDWDUPQ_WB_U VIWDUPQ_N_U
  99                          VIWDUPQ_WB_U VMAXAQ_M_S VMAXAVQ_P_S VMAXVQ_P_S
 100                          VMAXVQ_P_U VMINAQ_M_S VMINAVQ_P_S VMINVQ_P_S VMINVQ_P_U
 101                          VMLADAVAQ_S VMLADAVAQ_U VMLADAVQ_P_S VMLADAVQ_P_U
 102                          VMLADAVXQ_P_S VMLAQ_N_S VMLAQ_N_U VMLASQ_N_S VMLASQ_N_U
 103                          VMLSDAVQ_P_S VMLSDAVXQ_P_S VMVNQ_M_S VMVNQ_M_U
 104                          VNEGQ_M_S VPSELQ_S VPSELQ_U VQABSQ_M_S VQDMLAHQ_N_S
 105                          VQDMLAHQ_N_U VQNEGQ_M_S VQRDMLADHQ_S VQRDMLADHXQ_S
 106                          VQRDMLAHQ_N_S VQRDMLAHQ_N_U VQRDMLASHQ_N_S
 107                          VQRDMLASHQ_N_U VQRDMLSDHQ_S VQRDMLSDHXQ_S VQRSHLQ_M_N_S
 108                          VQRSHLQ_M_N_U VQSHLQ_M_R_S VQSHLQ_M_R_U VREV64Q_M_S
 109                          VREV64Q_M_U VRSHLQ_M_N_S VRSHLQ_M_N_U VSHLQ_M_R_S
 110                          VSHLQ_M_R_U VSLIQ_N_S VSLIQ_N_U VSRIQ_N_S VSRIQ_N_U
 111                          VQDMLSDHXQ_S VQDMLSDHQ_S VQDMLADHXQ_S VQDMLADHQ_S
 112                          VMLSDAVAXQ_S VMLSDAVAQ_S VMLADAVAXQ_S
 113                          VCMPGEQ_M_F VCMPGTQ_M_N_F VMLSLDAVQ_P_S VRMLALDAVHAXQ_S
 114                          VMLSLDAVXQ_P_S VFMAQ_F VMLSLDAVAQ_S VQSHRUNBQ_N_S
 115                          VQRSHRUNTQ_N_S VCMLAQ_F VMINNMAQ_M_F VFMASQ_N_F
 116                          VDUPQ_M_N_F VCMPGTQ_M_F VCMPLTQ_M_F VRMLSLDAVHQ_P_S
 117                          VQSHRUNTQ_N_S VABSQ_M_F VMAXNMAVQ_P_F VFMAQ_N_F
 118                          VRMLSLDAVHXQ_P_S VREV32Q_M_F VRMLSLDAVHAQ_S
 119                          VRMLSLDAVHAXQ_S VCMPLTQ_M_N_F VCMPNEQ_M_F VRNDAQ_M_F
 120                          VRNDPQ_M_F VADDLVAQ_P_S VQMOVUNBQ_M_S VCMPLEQ_M_F
 121                          VCMLAQ_ROT180_F VMLSLDAVAXQ_S VRNDXQ_M_F VFMSQ_F
 122                          VMINNMVQ_P_F VMAXNMVQ_P_F VPSELQ_F VCMLAQ_ROT90_F
 123                          VQMOVUNTQ_M_S VREV64Q_M_F VNEGQ_M_F VRNDMQ_M_F
 124                          VCMPLEQ_M_N_F VCMPGEQ_M_N_F VRNDNQ_M_F VMINNMAVQ_P_F
 125                          VCMPNEQ_M_N_F VRMLALDAVHQ_P_S VRMLALDAVHXQ_P_S
 126                          VCMPEQQ_M_N_F VCMLAQ_ROT270_F VMAXNMAQ_M_F VRNDQ_M_F
 127                          VMLALDAVQ_P_U VMLALDAVQ_P_S VQMOVNBQ_M_S VQMOVNBQ_M_U
 128                          VMOVLTQ_M_U VMOVLTQ_M_S VMOVNBQ_M_U VMOVNBQ_M_S
 129                          VRSHRNTQ_N_U VRSHRNTQ_N_S VORRQ_M_N_S VORRQ_M_N_U
 130                          VREV32Q_M_S VREV32Q_M_U VQRSHRNTQ_N_U VQRSHRNTQ_N_S
 131                          VMOVNTQ_M_U VMOVNTQ_M_S VMOVLBQ_M_U VMOVLBQ_M_S
 132                          VMLALDAVAQ_S VMLALDAVAQ_U VQSHRNBQ_N_U VQSHRNBQ_N_S
 133                          VSHRNBQ_N_U VSHRNBQ_N_S VRSHRNBQ_N_S VRSHRNBQ_N_U
 134                          VMLALDAVXQ_P_U VMLALDAVXQ_P_S VQMOVNTQ_M_U VQMOVNTQ_M_S
 135                          VMVNQ_M_N_U VMVNQ_M_N_S VQSHRNTQ_N_U VQSHRNTQ_N_S
 136                          VMLALDAVAXQ_S VMLALDAVAXQ_U VSHRNTQ_N_S VSHRNTQ_N_U
 137                          VCVTBQ_M_F16_F32 VCVTBQ_M_F32_F16 VCVTTQ_M_F16_F32
 138                          VCVTTQ_M_F32_F16 VCVTMQ_M_S VCVTMQ_M_U VCVTNQ_M_S
 139                          VCVTPQ_M_S VCVTPQ_M_U VCVTQ_M_N_FROM_F_S VCVTNQ_M_U
 140                          VREV16Q_M_S VREV16Q_M_U VREV32Q_M VCVTQ_M_FROM_F_U
 141                          VCVTQ_M_FROM_F_S VRMLALDAVHQ_P_U VADDLVAQ_P_U
 142                          VCVTQ_M_N_FROM_F_U VQSHLUQ_M_N_S VABAVQ_P_S
 143                          VABAVQ_P_U VSHLQ_M_S VSHLQ_M_U VSRIQ_M_N_S
 144                          VSRIQ_M_N_U VSUBQ_M_U VSUBQ_M_S VCVTQ_M_N_TO_F_U
 145                          VCVTQ_M_N_TO_F_S VQADDQ_M_U VQADDQ_M_S
 146                          VRSHRQ_M_N_S VSUBQ_M_N_S VSUBQ_M_N_U VBRSRQ_M_N_S
 147                          VSUBQ_M_N_F VBICQ_M_F VHADDQ_M_U VBICQ_M_U VBICQ_M_S
 148                          VMULQ_M_N_U VHADDQ_M_S VORNQ_M_F VMLAQ_M_N_S VQSUBQ_M_U
 149                          VQSUBQ_M_S VMLAQ_M_N_U VQSUBQ_M_N_U VQSUBQ_M_N_S
 150                          VMULLTQ_INT_M_S VMULLTQ_INT_M_U VMULQ_M_N_S VMULQ_M_N_F
 151                          VMLASQ_M_N_U VMLASQ_M_N_S VMAXQ_M_U VQRDMLAHQ_M_N_U
 152                          VCADDQ_ROT270_M_F VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S
 153                          VQRSHLQ_M_S VMULQ_M_F VRHADDQ_M_U VSHRQ_M_N_U
 154                          VRHADDQ_M_S VMULQ_M_S VMULQ_M_U VQRDMLASHQ_M_N_S
 155                          VRSHLQ_M_S VRSHLQ_M_U VRSHRQ_M_N_U VADDQ_M_N_F
 156                          VADDQ_M_N_S VADDQ_M_N_U VQRDMLASHQ_M_N_U VMAXQ_M_S
 157                          VQRDMLAHQ_M_N_S VORRQ_M_S VORRQ_M_U VORRQ_M_F
 158                          VQRSHLQ_M_U VRMULHQ_M_U VRMULHQ_M_S VMINQ_M_S VMINQ_M_U
 159                          VANDQ_M_F VANDQ_M_U VANDQ_M_S VHSUBQ_M_N_S VHSUBQ_M_N_U
 160                          VMULHQ_M_S VMULHQ_M_U VMULLBQ_INT_M_U
 161                          VMULLBQ_INT_M_S VCADDQ_ROT90_M_F
 162                          VSHRQ_M_N_S VADDQ_M_U VSLIQ_M_N_U
 163                          VQADDQ_M_N_S VBRSRQ_M_N_F VABDQ_M_F VBRSRQ_M_N_U
 164                          VEORQ_M_F VSHLQ_M_N_S VQDMLAHQ_M_N_U VQDMLAHQ_M_N_S
 165                          VSHLQ_M_N_U VMLADAVAQ_P_U VMLADAVAQ_P_S VSLIQ_M_N_S
 166                          VQSHLQ_M_U VQSHLQ_M_S VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S
 167                          VORNQ_M_U VORNQ_M_S VQSHLQ_M_N_S VQSHLQ_M_N_U VADDQ_M_S
 168                          VHADDQ_M_N_S VADDQ_M_F VQADDQ_M_N_U VEORQ_M_S VEORQ_M_U
 169                          VHSUBQ_M_S VHSUBQ_M_U VHADDQ_M_N_U VHCADDQ_ROT90_M_S
 170                          VQRDMLSDHQ_M_S VQRDMLSDHXQ_M_S VQRDMLADHXQ_M_S
 171                          VQDMULHQ_M_S VMLADAVAXQ_P_S VQDMLADHXQ_M_S
 172                          VQRDMULHQ_M_S VMLSDAVAXQ_P_S VQDMULHQ_M_N_S
 173                          VHCADDQ_ROT270_M_S VQDMLSDHQ_M_S VQDMLSDHXQ_M_S
 174                          VMLSDAVAQ_P_S VQRDMLADHQ_M_S VQDMLADHQ_M_S
 175                          VMLALDAVAQ_P_U VMLALDAVAQ_P_S VMLALDAVAXQ_P_U
 176                          VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S VQRSHRNTQ_M_N_S
 177                          VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S VQSHRNTQ_M_N_S
 178                          VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S VRSHRNTQ_M_N_U
 179                          VSHLLBQ_M_N_U VSHLLBQ_M_N_S VSHLLTQ_M_N_U VSHLLTQ_M_N_S
 180                          VSHRNBQ_M_N_S VSHRNBQ_M_N_U VSHRNTQ_M_N_S VSHRNTQ_M_N_U
 181                          VMLALDAVAXQ_P_S VQRSHRNTQ_M_N_U VQSHRNTQ_M_N_U
 182                          VRSHRNTQ_M_N_S VQRDMULHQ_M_N_S VRMLALDAVHAQ_P_S
 183                          VMLSLDAVAQ_P_S VMLSLDAVAXQ_P_S VMULLBQ_POLY_M_P
 184                          VMULLTQ_POLY_M_P VQDMULLBQ_M_N_S VQDMULLBQ_M_S
 185                          VQDMULLTQ_M_N_S VQDMULLTQ_M_S VQRSHRUNBQ_M_N_S
 186                          VQRSHRUNTQ_M_N_SVQSHRUNBQ_M_N_S VQSHRUNTQ_M_N_S
 187                          VRMLALDAVHAQ_P_U VRMLALDAVHAXQ_P_S VRMLSLDAVHAQ_P_S
 188                          VRMLSLDAVHAXQ_P_S VQRSHRUNTQ_M_N_S VQSHRUNBQ_M_N_S
 189                          VCMLAQ_M_F VCMLAQ_ROT180_M_F VCMLAQ_ROT270_M_F
 190                          VCMLAQ_ROT90_M_F VCMULQ_M_F VCMULQ_ROT180_M_F
 191                          VCMULQ_ROT270_M_F VCMULQ_ROT90_M_F VFMAQ_M_F
 192                          VFMAQ_M_N_F VFMASQ_M_N_F VFMSQ_M_F VMAXNMQ_M_F
 193                          VMINNMQ_M_F VSUBQ_M_F VSTRWQSB_S VSTRWQSB_U
 194                          VSTRBQSO_S VSTRBQSO_U VSTRBQ_S VSTRBQ_U VLDRBQGO_S
 195                          VLDRBQGO_U VLDRBQ_S VLDRBQ_U VLDRWQGB_S VLDRWQGB_U
 196                          VLD1Q_F VLD1Q_S VLD1Q_U VLDRHQ_F VLDRHQGO_S
 197                          VLDRHQGO_U VLDRHQGSO_S VLDRHQGSO_U VLDRHQ_S VLDRHQ_U
 198                          VLDRWQ_F VLDRWQ_S VLDRWQ_U VLDRDQGB_S VLDRDQGB_U
 199                          VLDRDQGO_S VLDRDQGO_U VLDRDQGSO_S VLDRDQGSO_U
 200                          VLDRHQGO_F VLDRHQGSO_F VLDRWQGB_F VLDRWQGO_F
 201                          VLDRWQGO_S VLDRWQGO_U VLDRWQGSO_F VLDRWQGSO_S
 202                          VLDRWQGSO_U VSTRHQ_F VST1Q_S VST1Q_U VSTRHQSO_S
 203                          VSTRHQSO_U VSTRHQSSO_S VSTRHQSSO_U VSTRHQ_S
 204                          VSTRHQ_U VSTRWQ_S VSTRWQ_U VSTRWQ_F VST1Q_F VSTRDQSB_S
 205                          VSTRDQSB_U VSTRDQSO_S VSTRDQSO_U VSTRDQSSO_S
 206                          VSTRDQSSO_U VSTRWQSO_S VSTRWQSO_U VSTRWQSSO_S
 207                          VSTRWQSSO_U VSTRHQSO_F VSTRHQSSO_F VSTRWQSB_F
 208                          VSTRWQSO_F VSTRWQSSO_F VDDUPQ VDDUPQ_M VDWDUPQ
 209                          VDWDUPQ_M VIDUPQ VIDUPQ_M VIWDUPQ VIWDUPQ_M
 210                          VSTRWQSBWB_S VSTRWQSBWB_U VLDRWQGBWB_S VLDRWQGBWB_U
 211                          VSTRWQSBWB_F VLDRWQGBWB_F VSTRDQSBWB_S VSTRDQSBWB_U
 212                          VLDRDQGBWB_S VLDRDQGBWB_U VADCQ_U VADCQ_M_U VADCQ_S
 213                          VADCQ_M_S VSBCIQ_U VSBCIQ_S VSBCIQ_M_U VSBCIQ_M_S
 214                          VSBCQ_U VSBCQ_S VSBCQ_M_U VSBCQ_M_S VADCIQ_U VADCIQ_M_U
 215                          VADCIQ_S VADCIQ_M_S VLD2Q VLD4Q VST2Q SRSHRL SRSHR
 216                          URSHR URSHRL SQRSHR UQRSHL UQRSHLL_64 VSHLCQ_M_U
 217                          UQRSHLL_48 SQRSHRL_64 SQRSHRL_48 VSHLCQ_M_S])
 218
 219 (define_mode_attr MVE_CNVT [(V8HI "V8HF") (V4SI "V4SF") (V8HF "V8HI")
 220                             (V4SF "V4SI")])
 221
 222 (define_int_attr supf [(VCVTQ_TO_F_S "s") (VCVTQ_TO_F_U "u") (VREV16Q_S "s")
 223                        (VREV16Q_U "u") (VMVNQ_N_S "s") (VMVNQ_N_U "u")
 224                        (VCVTAQ_U "u") (VCVTAQ_S "s") (VREV64Q_S "s")
 225                        (VREV64Q_U "u") (VMVNQ_S "s") (VMVNQ_U "u")
 226                        (VDUPQ_N_U "u") (VDUPQ_N_S"s") (VADDVQ_S "s")
 227                        (VADDVQ_U "u") (VADDVQ_S "s") (VADDVQ_U "u")
 228                        (VMOVLTQ_U "u") (VMOVLTQ_S "s") (VMOVLBQ_S "s")
 229                        (VMOVLBQ_U "u") (VCVTQ_FROM_F_S "s") (VCVTQ_FROM_F_U "u")
 230                        (VCVTPQ_S "s") (VCVTPQ_U "u") (VCVTNQ_S "s")
 231                        (VCVTNQ_U "u") (VCVTMQ_S "s") (VCVTMQ_U "u")
 232                        (VCLZQ_U "u") (VCLZQ_S "s") (VREV32Q_U "u")
 233                        (VREV32Q_S "s") (VADDLVQ_U "u") (VADDLVQ_S "s")
 234                        (VCVTQ_N_TO_F_S "s") (VCVTQ_N_TO_F_U "u")
 235                        (VCREATEQ_U "u") (VCREATEQ_S "s") (VSHRQ_N_S "s")
 236                        (VSHRQ_N_U "u") (VCVTQ_N_FROM_F_S "s") (VSHLQ_U "u")
 237                        (VCVTQ_N_FROM_F_U "u") (VADDLVQ_P_S "s") (VSHLQ_S "s")
 238                        (VADDLVQ_P_U "u") (VCMPNEQ_U "u") (VCMPNEQ_S "s")
 239                        (VABDQ_M_S "s") (VABDQ_M_U "u") (VABDQ_S "s")
 240                        (VABDQ_U "u") (VADDQ_N_S "s") (VADDQ_N_U "u")
 241                        (VADDVQ_P_S "s") (VADDVQ_P_U "u") (VANDQ_S "s")
 242                        (VANDQ_U "u") (VBICQ_S "s") (VBICQ_U "u")
 243                        (VBRSRQ_N_S "s") (VBRSRQ_N_U "u") (VCADDQ_ROT270_S "s")
 244                        (VCADDQ_ROT270_U "u") (VCADDQ_ROT90_S "s")
 245                        (VCMPEQQ_S "s") (VCMPEQQ_U "u") (VCADDQ_ROT90_U "u")
 246                        (VCMPEQQ_N_S "s") (VCMPEQQ_N_U "u") (VCMPNEQ_N_S "s")
 247                        (VCMPNEQ_N_U "u") (VEORQ_S "s") (VEORQ_U "u")
 248                        (VHADDQ_N_S "s") (VHADDQ_N_U "u") (VHADDQ_S "s")
 249                        (VHADDQ_U "u") (VHSUBQ_N_S "s")  (VHSUBQ_N_U "u")
 250                        (VHSUBQ_S "s") (VMAXQ_S "s") (VMAXQ_U "u") (VHSUBQ_U "u")
 251                        (VMAXVQ_S "s") (VMAXVQ_U "u") (VMINQ_S "s") (VMINQ_U "u")
 252                        (VMINVQ_S "s") (VMINVQ_U "u") (VMLADAVQ_S "s")
 253                        (VMLADAVQ_U "u") (VMULHQ_S "s") (VMULHQ_U "u")
 254                        (VMULLBQ_INT_S "s") (VMULLBQ_INT_U "u") (VQADDQ_S "s")
 255                        (VMULLTQ_INT_S "s") (VMULLTQ_INT_U "u") (VQADDQ_U "u")
 256                        (VMULQ_N_S "s") (VMULQ_N_U "u") (VMULQ_S "s")
 257                        (VMULQ_U "u") (VORNQ_S "s") (VORNQ_U "u") (VORRQ_S "s")
 258                        (VORRQ_U "u") (VQADDQ_N_S "s") (VQADDQ_N_U "u")
 259                        (VQRSHLQ_N_S "s") (VQRSHLQ_N_U "u") (VQRSHLQ_S "s")
 260                        (VQRSHLQ_U "u") (VQSHLQ_N_S "s") (VQSHLQ_N_U "u")
 261                        (VQSHLQ_R_S "s") (VQSHLQ_R_U "u") (VQSHLQ_S "s")
 262                        (VQSHLQ_U "u") (VQSUBQ_N_S "s") (VQSUBQ_N_U "u")
 263                        (VQSUBQ_S "s") (VQSUBQ_U "u") (VRHADDQ_S "s")
 264                        (VRHADDQ_U "u") (VRMULHQ_S "s") (VRMULHQ_U "u")
 265                        (VRSHLQ_N_S "s") (VRSHLQ_N_U "u") (VRSHLQ_S "s")
 266                        (VRSHLQ_U "u") (VRSHRQ_N_S "s") (VRSHRQ_N_U "u")
 267                        (VSHLQ_N_S "s") (VSHLQ_N_U "u") (VSHLQ_R_S "s")
 268                        (VSHLQ_R_U "u") (VSUBQ_N_S "s") (VSUBQ_N_U "u")
 269                        (VSUBQ_S "s") (VSUBQ_U "u") (VADDVAQ_S "s")
 270                        (VADDVAQ_U "u") (VADDLVAQ_S "s") (VADDLVAQ_U "u")
 271                        (VBICQ_N_S "s") (VBICQ_N_U "u") (VMLALDAVQ_U "u")
 272                        (VMLALDAVQ_S "s") (VMLALDAVXQ_U "u") (VMLALDAVXQ_S "s")
 273                        (VMOVNBQ_U "u") (VMOVNBQ_S "s") (VMOVNTQ_U "u")
 274                        (VMOVNTQ_S "s") (VORRQ_N_S "s") (VORRQ_N_U "u")
 275                        (VQMOVNBQ_U "u") (VQMOVNBQ_S "s") (VQMOVNTQ_S "s")
 276                        (VQMOVNTQ_U "u") (VSHLLBQ_N_U "u") (VSHLLBQ_N_S "s")
 277                        (VSHLLTQ_N_U "u") (VSHLLTQ_N_S "s") (VRMLALDAVHQ_U "u")
 278                        (VRMLALDAVHQ_S "s") (VBICQ_M_N_S "s") (VBICQ_M_N_U "u")
 279                        (VCVTAQ_M_S "s") (VCVTAQ_M_U "u") (VCVTQ_M_TO_F_S "s")
 280                        (VCVTQ_M_TO_F_U "u") (VQRSHRNBQ_N_S "s")
 281                        (VQRSHRNBQ_N_U "u") (VABAVQ_S "s") (VABAVQ_U "u")
 282                        (VRMLALDAVHAQ_U "u") (VRMLALDAVHAQ_S "s") (VSHLCQ_S "s")
 283                        (VSHLCQ_U "u") (VADDVAQ_P_S "s") (VADDVAQ_P_U "u")
 284                        (VCLZQ_M_S "s") (VCLZQ_M_U "u") (VCMPEQQ_M_N_S "s")
 285                        (VCMPEQQ_M_N_U "u") (VCMPEQQ_M_S "s") (VCMPEQQ_M_U "u")
 286                        (VCMPNEQ_M_N_S "s") (VCMPNEQ_M_N_U "u") (VCMPNEQ_M_S "s")
 287                        (VCMPNEQ_M_U "u") (VDUPQ_M_N_S "s") (VDUPQ_M_N_U "u")
 288                        (VMAXVQ_P_S "s") (VMAXVQ_P_U "u") (VMINVQ_P_S "s")
 289                        (VMINVQ_P_U "u") (VMLADAVAQ_S "s") (VMLADAVAQ_U "u")
 290                        (VMLADAVQ_P_S "s") (VMLADAVQ_P_U "u") (VMLAQ_N_S "s")
 291                        (VMLAQ_N_U "u") (VMLASQ_N_S "s") (VMLASQ_N_U "u")
 292                        (VMVNQ_M_S "s") (VMVNQ_M_U "u") (VPSELQ_S "s")
 293                        (VPSELQ_U "u") (VQDMLAHQ_N_S "s") (VQDMLAHQ_N_U "u")
 294                        (VQRDMLAHQ_N_S "s") (VQRDMLAHQ_N_U "u")
 295                        (VQRDMLASHQ_N_S "s") (VQRDMLASHQ_N_U "u")
 296                        (VQRSHLQ_M_N_S "s") (VQRSHLQ_M_N_U "u")
 297                        (VQSHLQ_M_R_S "s") (VQSHLQ_M_R_U "u") (VSRIQ_N_S "s")
 298                        (VREV64Q_M_S "s") (VREV64Q_M_U "u") (VSRIQ_N_U "u")
 299                        (VRSHLQ_M_N_S "s") (VRSHLQ_M_N_U "u") (VSHLQ_M_R_S "s")
 300                        (VSHLQ_M_R_U "u") (VSLIQ_N_S "s") (VSLIQ_N_U "u")
 301                        (VMLALDAVQ_P_S "s") (VQMOVNBQ_M_S "s") (VMOVLTQ_M_S "s")
 302                        (VMOVNBQ_M_S "s") (VRSHRNTQ_N_S "s") (VORRQ_M_N_S "s")
 303                        (VREV32Q_M_S "s") (VQRSHRNTQ_N_S "s") (VMOVNTQ_M_S "s")
 304                        (VMOVLBQ_M_S "s") (VMLALDAVAQ_S "s") (VQSHRNBQ_N_S "s")
 305                        (VSHRNBQ_N_S "s") (VRSHRNBQ_N_S "s") (VMLALDAVXQ_P_S "s")
 306                        (VQMOVNTQ_M_S "s") (VMVNQ_M_N_S "s") (VQSHRNTQ_N_S "s")
 307                        (VMLALDAVAXQ_S "s") (VSHRNTQ_N_S "s") (VMLALDAVQ_P_U "u")
 308                        (VQMOVNBQ_M_U "u") (VMOVLTQ_M_U "u") (VMOVNBQ_M_U "u")
 309                        (VRSHRNTQ_N_U "u") (VORRQ_M_N_U "u") (VREV32Q_M_U "u")
 310                        (VREV16Q_M_S "s") (VREV16Q_M_U "u")
 311                        (VQRSHRNTQ_N_U "u") (VMOVNTQ_M_U "u") (VMOVLBQ_M_U "u")
 312                        (VMLALDAVAQ_U "u") (VQSHRNBQ_N_U "u") (VSHRNBQ_N_U "u")
 313                        (VRSHRNBQ_N_U "u") (VMLALDAVXQ_P_U "u")
 314                        (VMVNQ_M_N_U "u") (VQSHRNTQ_N_U "u") (VMLALDAVAXQ_U "u")
 315                        (VQMOVNTQ_M_U "u") (VSHRNTQ_N_U "u") (VCVTMQ_M_S "s")
 316                        (VCVTMQ_M_U "u") (VCVTNQ_M_S "s") (VCVTNQ_M_U "u")
 317                        (VCVTPQ_M_S "s") (VCVTPQ_M_U "u") (VADDLVAQ_P_S "s")
 318                        (VCVTQ_M_N_FROM_F_U "u") (VCVTQ_M_FROM_F_S "s")
 319                        (VCVTQ_M_FROM_F_U "u") (VRMLALDAVHQ_P_U "u")
 320                        (VRMLALDAVHQ_P_S "s") (VADDLVAQ_P_U "u")
 321                        (VCVTQ_M_N_FROM_F_S "s") (VABAVQ_P_U "u")
 322                        (VABAVQ_P_S "s") (VSHLQ_M_S "s") (VSHLQ_M_U "u")
 323                        (VSRIQ_M_N_S "s") (VSRIQ_M_N_U "u") (VSUBQ_M_S "s")
 324                        (VSUBQ_M_U "u") (VCVTQ_M_N_TO_F_S "s")
 325                        (VCVTQ_M_N_TO_F_U "u") (VADDQ_M_N_U "u")
 326                        (VSHLQ_M_N_S "s") (VMAXQ_M_U "u") (VHSUBQ_M_N_U "u")
 327                        (VMULQ_M_N_S "s") (VQSHLQ_M_U "u") (VRHADDQ_M_S "s")
 328                        (VEORQ_M_U "u") (VSHRQ_M_N_U "u") (VCADDQ_ROT90_M_U "u")
 329                        (VMLADAVAQ_P_U "u") (VEORQ_M_S "s") (VBRSRQ_M_N_S "s")
 330                        (VMULQ_M_U "u") (VQRDMLAHQ_M_N_S "s") (VHSUBQ_M_N_S "s")
 331                        (VQRSHLQ_M_S "s") (VMULQ_M_N_U "u")
 332                        (VMULQ_M_S "s") (VQSHLQ_M_N_U "u") (VSLIQ_M_N_U "u")
 333                        (VMLADAVAQ_P_S "s") (VQRSHLQ_M_U "u")
 334                        (VMULLBQ_INT_M_U "u") (VSHLQ_M_N_U "u") (VQSUBQ_M_U "u")
 335                        (VQRDMLASHQ_M_N_U "u") (VRSHRQ_M_N_S "s")
 336                        (VORNQ_M_S "s") (VCADDQ_ROT270_M_S "s") (VRHADDQ_M_U "u")
 337                        (VRSHRQ_M_N_U "u") (VMLASQ_M_N_U "u") (VHSUBQ_M_U "u")
 338                        (VQSUBQ_M_N_S "s") (VMULLTQ_INT_M_S "s")
 339                        (VORRQ_M_S "s") (VQDMLAHQ_M_N_U "u") (VRSHLQ_M_S "s")
 340                        (VHADDQ_M_U "u") (VHADDQ_M_N_S "s") (VMULLTQ_INT_M_U "u")
 341                        (VORRQ_M_U "u") (VHADDQ_M_S "s") (VHADDQ_M_N_U "u")
 342                        (VQDMLAHQ_M_N_S "s") (VMAXQ_M_S "s") (VORNQ_M_U "u")
 343                        (VCADDQ_ROT270_M_U "u") (VQADDQ_M_U "u")
 344                        (VQRDMLASHQ_M_N_S "s") (VBICQ_M_U "u") (VMINQ_M_U "u")
 345                        (VSUBQ_M_N_S "s") (VMULLBQ_INT_M_S "s") (VQSUBQ_M_S "s")
 346                        (VCADDQ_ROT90_M_S "s") (VRMULHQ_M_S "s") (VANDQ_M_U "u")
 347                        (VMULHQ_M_S "s") (VADDQ_M_S "s") (VQRDMLAHQ_M_N_U "u")
 348                        (VMLASQ_M_N_S "s") (VHSUBQ_M_S "s") (VRMULHQ_M_U "u")
 349                        (VQADDQ_M_N_S "s") (VSHRQ_M_N_S "s") (VANDQ_M_S "s")
 350                        (VABDQ_M_U "u") (VQSHLQ_M_S "s") (VABDQ_M_S "s")
 351                        (VSUBQ_M_N_U "u") (VMLAQ_M_N_S "s") (VBRSRQ_M_N_U "u")
 352                        (VADDQ_M_U "u") (VRSHLQ_M_U "u") (VSLIQ_M_N_S "s")
 353                        (VQADDQ_M_N_U "u") (VADDQ_M_N_S "s") (VQSUBQ_M_N_U "u")
 354                        (VMLAQ_M_N_U "u") (VMINQ_M_S "s") (VMULHQ_M_U "u")
 355                        (VQADDQ_M_S "s") (VBICQ_M_S "s") (VQSHLQ_M_N_S "s")
 356                        (VQSHRNTQ_M_N_S "s") (VQSHRNTQ_M_N_U "u")
 357                        (VSHRNTQ_M_N_U "u") (VSHRNTQ_M_N_S "s")
 358                        (VSHRNBQ_M_N_S "s") (VSHRNBQ_M_N_U "u")
 359                        (VSHLLTQ_M_N_S "s") (VSHLLTQ_M_N_U "u")
 360                        (VSHLLBQ_M_N_S "s") (VSHLLBQ_M_N_U "u")
 361                        (VRSHRNTQ_M_N_S "s") (VRSHRNTQ_M_N_U "u")
 362                        (VRSHRNBQ_M_N_U "u") (VRSHRNBQ_M_N_S "s")
 363                        (VQSHRNTQ_M_N_U "u") (VQSHRNTQ_M_N_S "s")
 364                        (VQSHRNBQ_M_N_S "s") (VQSHRNBQ_M_N_U "u")
 365                        (VQRSHRNTQ_M_N_S "s") (VQRSHRNTQ_M_N_U "u")
 366                        (VQRSHRNBQ_M_N_S "s") (VQRSHRNBQ_M_N_U "u")
 367                        (VMLALDAVAXQ_P_S "s") (VMLALDAVAXQ_P_U "u")
 368                        (VMLALDAVAQ_P_S "s") (VMLALDAVAQ_P_U "u")
 369                        (VSTRWQSB_S "s") (VSTRWQSB_U "u") (VSTRBQSO_S "s")
 370                        (VSTRBQSO_U "u") (VSTRBQ_S "s") (VSTRBQ_U "u")
 371                        (VLDRBQGO_S "s") (VLDRBQGO_U "u") (VLDRBQ_S "s")
 372                        (VLDRBQ_U "u") (VLDRWQGB_S "s") (VLDRWQGB_U "u")
 373                        (VLD1Q_S "s") (VLD1Q_U "u") (VLDRHQGO_S "s")
 374                        (VLDRHQGO_U "u") (VLDRHQGSO_S "s") (VLDRHQGSO_U "u")
 375                        (VLDRHQ_S "s") (VLDRHQ_U "u") (VLDRWQ_S "s")
 376                        (VLDRWQ_U "u") (VLDRDQGB_S "s") (VLDRDQGB_U "u")
 377                        (VLDRDQGO_S "s") (VLDRDQGO_U "u") (VLDRDQGSO_S "s")
 378                        (VLDRDQGSO_U "u") (VLDRWQGO_S "s") (VLDRWQGO_U "u")
 379                        (VLDRWQGSO_S "s") (VLDRWQGSO_U "u") (VST1Q_S "s")
 380                        (VST1Q_U "u") (VSTRHQSO_S "s") (VSTRHQSO_U "u")
 381                        (VSTRHQSSO_S "s") (VSTRHQSSO_U "u") (VSTRHQ_S "s")
 382                        (VSTRHQ_U "u") (VSTRWQ_S "s") (VSTRWQ_U "u")
 383                        (VSTRDQSB_S "s") (VSTRDQSB_U "u") (VSTRDQSO_S "s")
 384                        (VSTRDQSO_U "u") (VSTRDQSSO_S "s") (VSTRDQSSO_U "u")
 385                        (VSTRWQSO_U "u") (VSTRWQSO_S "s") (VSTRWQSSO_U "u")
 386                        (VSTRWQSSO_S "s") (VSTRWQSBWB_S "s") (VSTRWQSBWB_U "u")
 387                        (VLDRWQGBWB_S "s") (VLDRWQGBWB_U "u") (VLDRDQGBWB_S "s")
 388                        (VLDRDQGBWB_U "u") (VSTRDQSBWB_S "s") (VADCQ_M_S "s")
 389                        (VSTRDQSBWB_U "u") (VSBCQ_U "u")  (VSBCQ_M_U "u")
 390                        (VSBCQ_S "s")  (VSBCQ_M_S "s") (VSBCIQ_U "u")
 391                        (VSBCIQ_M_U "u") (VSBCIQ_S "s") (VSBCIQ_M_S "s")
 392                        (VADCQ_U "u")  (VADCQ_M_U "u") (VADCQ_S "s")
 393                        (VADCIQ_U "u") (VADCIQ_M_U "u") (VADCIQ_S "s")
 394                        (VADCIQ_M_S "s") (SQRSHRL_64 "64") (SQRSHRL_48 "48")
 395                        (UQRSHLL_64 "64") (UQRSHLL_48 "48") (VSHLCQ_M_S "s")
 396                        (VSHLCQ_M_U "u")])
 397
 398 (define_int_attr mode1 [(VCTP8Q "8") (VCTP16Q "16") (VCTP32Q "32")
 399                         (VCTP64Q "64") (VCTP8Q_M "8") (VCTP16Q_M "16")
 400                         (VCTP32Q_M "32") (VCTP64Q_M "64")])
 401 (define_mode_attr MVE_pred2 [(V16QI "mve_imm_8") (V8HI "mve_imm_16")
 402                              (V4SI "mve_imm_32")
 403                              (V8HF "mve_imm_16") (V4SF "mve_imm_32")])
 404 (define_mode_attr MVE_constraint2 [(V16QI "Rb") (V8HI "Rd") (V4SI "Rf")
 405                                     (V8HF "Rd") (V4SF "Rf")])
 406 (define_mode_attr MVE_LANES [(V16QI "16") (V8HI "8") (V4SI "4")])
 407 (define_mode_attr MVE_constraint [ (V16QI "Ra") (V8HI "Rc") (V4SI "Re")])
 408 (define_mode_attr MVE_pred [ (V16QI "mve_imm_7") (V8HI "mve_imm_15")
 409                                    (V4SI "mve_imm_31")])
 410 (define_mode_attr MVE_constraint3 [ (V8HI "Rb") (V4SI "Rd")])
 411 (define_mode_attr MVE_pred3 [ (V8HI "mve_imm_8") (V4SI "mve_imm_16")])
 412 (define_mode_attr MVE_constraint1 [ (V8HI "Ra") (V4SI "Rc")])
 413 (define_mode_attr MVE_pred1 [ (V8HI "mve_imm_7") (V4SI "mve_imm_15")])
 414 (define_mode_attr MVE_B_ELEM [ (V16QI "V16QI") (V8HI "V8QI") (V4SI "V4QI")])
 415 (define_mode_attr MVE_H_ELEM [ (V8HI "V8HI") (V4SI "V4HI")])
 416 (define_mode_attr V_sz_elem1 [(V16QI "b") (V8HI  "h") (V4SI "w") (V8HF "h")
 417                               (V4SF "w")])
 418 (define_mode_attr V_extr_elem [(V16QI "u8") (V8HI "u16") (V4SI "32")
 419                               (V8HF "u16") (V4SF "32")])
 420
 421 (define_mode_attr earlyclobber_32 [(V16QI "=w") (V8HI "=w") (V4SI "=&w")
 422                                                 (V8HF "=w") (V4SF "=&w")])
 423
 424 (define_int_iterator VCVTQ_TO_F [VCVTQ_TO_F_S VCVTQ_TO_F_U])
 425 (define_int_iterator VMVNQ_N [VMVNQ_N_U VMVNQ_N_S])
 426 (define_int_iterator VREV64Q [VREV64Q_S VREV64Q_U])
 427 (define_int_iterator VCVTQ_FROM_F [VCVTQ_FROM_F_S VCVTQ_FROM_F_U])
 428 (define_int_iterator VREV16Q [VREV16Q_U VREV16Q_S])
 429 (define_int_iterator VCVTAQ [VCVTAQ_U VCVTAQ_S])
 430 (define_int_iterator VMVNQ [VMVNQ_U VMVNQ_S])
 431 (define_int_iterator VDUPQ_N [VDUPQ_N_U VDUPQ_N_S])
 432 (define_int_iterator VCLZQ [VCLZQ_U VCLZQ_S])
 433 (define_int_iterator VADDVQ [VADDVQ_U VADDVQ_S])
 434 (define_int_iterator VREV32Q [VREV32Q_U VREV32Q_S])
 435 (define_int_iterator VMOVLBQ [VMOVLBQ_S VMOVLBQ_U])
 436 (define_int_iterator VMOVLTQ [VMOVLTQ_U VMOVLTQ_S])
 437 (define_int_iterator VCVTPQ [VCVTPQ_S VCVTPQ_U])
 438 (define_int_iterator VCVTNQ [VCVTNQ_S VCVTNQ_U])
 439 (define_int_iterator VCVTMQ [VCVTMQ_S VCVTMQ_U])
 440 (define_int_iterator VADDLVQ [VADDLVQ_U VADDLVQ_S])
 441 (define_int_iterator VCTPQ [VCTP8Q VCTP16Q VCTP32Q VCTP64Q])
 442 (define_int_iterator VCTPQ_M [VCTP8Q_M VCTP16Q_M VCTP32Q_M VCTP64Q_M])
 443 (define_int_iterator VCVTQ_N_TO_F [VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U])
 444 (define_int_iterator VCREATEQ [VCREATEQ_U VCREATEQ_S])
 445 (define_int_iterator VSHRQ_N [VSHRQ_N_S VSHRQ_N_U])
 446 (define_int_iterator VCVTQ_N_FROM_F [VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U])
 447 (define_int_iterator VADDLVQ_P [VADDLVQ_P_S VADDLVQ_P_U])
 448 (define_int_iterator VCMPNEQ [VCMPNEQ_U VCMPNEQ_S])
 449 (define_int_iterator VSHLQ [VSHLQ_S VSHLQ_U])
 450 (define_int_iterator VABDQ [VABDQ_S VABDQ_U])
 451 (define_int_iterator VADDQ_N [VADDQ_N_S VADDQ_N_U])
 452 (define_int_iterator VADDVAQ [VADDVAQ_S VADDVAQ_U])
 453 (define_int_iterator VADDVQ_P [VADDVQ_P_U VADDVQ_P_S])
 454 (define_int_iterator VANDQ [VANDQ_U VANDQ_S])
 455 (define_int_iterator VBICQ [VBICQ_S VBICQ_U])
 456 (define_int_iterator VBRSRQ_N [VBRSRQ_N_U VBRSRQ_N_S])
 457 (define_int_iterator VCADDQ_ROT270 [VCADDQ_ROT270_S VCADDQ_ROT270_U])
 458 (define_int_iterator VCADDQ_ROT90 [VCADDQ_ROT90_U VCADDQ_ROT90_S])
 459 (define_int_iterator VCMPEQQ [VCMPEQQ_U VCMPEQQ_S])
 460 (define_int_iterator VCMPEQQ_N [VCMPEQQ_N_S VCMPEQQ_N_U])
 461 (define_int_iterator VCMPNEQ_N [VCMPNEQ_N_U VCMPNEQ_N_S])
 462 (define_int_iterator VEORQ [VEORQ_U VEORQ_S])
 463 (define_int_iterator VHADDQ [VHADDQ_S VHADDQ_U])
 464 (define_int_iterator VHADDQ_N [VHADDQ_N_U VHADDQ_N_S])
 465 (define_int_iterator VHSUBQ [VHSUBQ_S VHSUBQ_U])
 466 (define_int_iterator VHSUBQ_N [VHSUBQ_N_U VHSUBQ_N_S])
 467 (define_int_iterator VMAXQ [VMAXQ_U VMAXQ_S])
 468 (define_int_iterator VMAXVQ [VMAXVQ_U VMAXVQ_S])
 469 (define_int_iterator VMINQ [VMINQ_S VMINQ_U])
 470 (define_int_iterator VMINVQ [VMINVQ_U VMINVQ_S])
 471 (define_int_iterator VMLADAVQ [VMLADAVQ_U VMLADAVQ_S])
 472 (define_int_iterator VMULHQ [VMULHQ_S VMULHQ_U])
 473 (define_int_iterator VMULLBQ_INT [VMULLBQ_INT_U VMULLBQ_INT_S])
 474 (define_int_iterator VMULLTQ_INT [VMULLTQ_INT_U VMULLTQ_INT_S])
 475 (define_int_iterator VMULQ [VMULQ_U VMULQ_S])
 476 (define_int_iterator VMULQ_N [VMULQ_N_U VMULQ_N_S])
 477 (define_int_iterator VORNQ [VORNQ_U VORNQ_S])
 478 (define_int_iterator VORRQ [VORRQ_S VORRQ_U])
 479 (define_int_iterator VQADDQ [VQADDQ_U VQADDQ_S])
 480 (define_int_iterator VQADDQ_N [VQADDQ_N_S VQADDQ_N_U])
 481 (define_int_iterator VQRSHLQ [VQRSHLQ_S VQRSHLQ_U])
 482 (define_int_iterator VQRSHLQ_N [VQRSHLQ_N_S VQRSHLQ_N_U])
 483 (define_int_iterator VQSHLQ [VQSHLQ_S VQSHLQ_U])
 484 (define_int_iterator VQSHLQ_N [VQSHLQ_N_S VQSHLQ_N_U])
 485 (define_int_iterator VQSHLQ_R [VQSHLQ_R_U VQSHLQ_R_S])
 486 (define_int_iterator VQSUBQ [VQSUBQ_U VQSUBQ_S])
 487 (define_int_iterator VQSUBQ_N [VQSUBQ_N_S VQSUBQ_N_U])
 488 (define_int_iterator VRHADDQ [VRHADDQ_S VRHADDQ_U])
 489 (define_int_iterator VRMULHQ [VRMULHQ_S VRMULHQ_U])
 490 (define_int_iterator VRSHLQ [VRSHLQ_S VRSHLQ_U])
 491 (define_int_iterator VRSHLQ_N [VRSHLQ_N_U VRSHLQ_N_S])
 492 (define_int_iterator VRSHRQ_N [VRSHRQ_N_S VRSHRQ_N_U])
 493 (define_int_iterator VSHLQ_N [VSHLQ_N_U VSHLQ_N_S])
 494 (define_int_iterator VSHLQ_R [VSHLQ_R_S VSHLQ_R_U])
 495 (define_int_iterator VSUBQ [VSUBQ_S VSUBQ_U])
 496 (define_int_iterator VSUBQ_N [VSUBQ_N_S VSUBQ_N_U])
 497 (define_int_iterator VADDLVAQ [VADDLVAQ_S VADDLVAQ_U])
 498 (define_int_iterator VBICQ_N [VBICQ_N_S VBICQ_N_U])
 499 (define_int_iterator VMLALDAVQ [VMLALDAVQ_U VMLALDAVQ_S])
 500 (define_int_iterator VMLALDAVXQ [VMLALDAVXQ_U VMLALDAVXQ_S])
 501 (define_int_iterator VMOVNBQ [VMOVNBQ_U VMOVNBQ_S])
 502 (define_int_iterator VMOVNTQ [VMOVNTQ_S VMOVNTQ_U])
 503 (define_int_iterator VORRQ_N [VORRQ_N_U VORRQ_N_S])
 504 (define_int_iterator VQMOVNBQ [VQMOVNBQ_U VQMOVNBQ_S])
 505 (define_int_iterator VQMOVNTQ [VQMOVNTQ_U VQMOVNTQ_S])
 506 (define_int_iterator VSHLLBQ_N [VSHLLBQ_N_S VSHLLBQ_N_U])
 507 (define_int_iterator VSHLLTQ_N [VSHLLTQ_N_U VSHLLTQ_N_S])
 508 (define_int_iterator VRMLALDAVHQ [VRMLALDAVHQ_U VRMLALDAVHQ_S])
 509 (define_int_iterator VBICQ_M_N [VBICQ_M_N_S VBICQ_M_N_U])
 510 (define_int_iterator VCVTAQ_M [VCVTAQ_M_S VCVTAQ_M_U])
 511 (define_int_iterator VCVTQ_M_TO_F [VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U])
 512 (define_int_iterator VQRSHRNBQ_N [VQRSHRNBQ_N_U VQRSHRNBQ_N_S])
 513 (define_int_iterator VABAVQ [VABAVQ_S VABAVQ_U])
 514 (define_int_iterator VSHLCQ [VSHLCQ_S VSHLCQ_U])
 515 (define_int_iterator VRMLALDAVHAQ [VRMLALDAVHAQ_S VRMLALDAVHAQ_U])
 516 (define_int_iterator VADDVAQ_P [VADDVAQ_P_S VADDVAQ_P_U])
 517 (define_int_iterator VCLZQ_M [VCLZQ_M_S VCLZQ_M_U])
 518 (define_int_iterator VCMPEQQ_M_N [VCMPEQQ_M_N_S VCMPEQQ_M_N_U])
 519 (define_int_iterator VCMPEQQ_M [VCMPEQQ_M_S VCMPEQQ_M_U])
 520 (define_int_iterator VCMPNEQ_M_N [VCMPNEQ_M_N_S VCMPNEQ_M_N_U])
 521 (define_int_iterator VCMPNEQ_M [VCMPNEQ_M_S VCMPNEQ_M_U])
 522 (define_int_iterator VDUPQ_M_N [VDUPQ_M_N_S VDUPQ_M_N_U])
 523 (define_int_iterator VMAXVQ_P [VMAXVQ_P_S VMAXVQ_P_U])
 524 (define_int_iterator VMINVQ_P [VMINVQ_P_S VMINVQ_P_U])
 525 (define_int_iterator VMLADAVAQ [VMLADAVAQ_S VMLADAVAQ_U])
 526 (define_int_iterator VMLADAVQ_P [VMLADAVQ_P_S VMLADAVQ_P_U])
 527 (define_int_iterator VMLAQ_N [VMLAQ_N_S VMLAQ_N_U])
 528 (define_int_iterator VMLASQ_N [VMLASQ_N_S VMLASQ_N_U])
 529 (define_int_iterator VMVNQ_M [VMVNQ_M_S VMVNQ_M_U])
 530 (define_int_iterator VPSELQ [VPSELQ_S VPSELQ_U])
 531 (define_int_iterator VQDMLAHQ_N [VQDMLAHQ_N_S VQDMLAHQ_N_U])
 532 (define_int_iterator VQRDMLAHQ_N [VQRDMLAHQ_N_S VQRDMLAHQ_N_U])
 533 (define_int_iterator VQRDMLASHQ_N [VQRDMLASHQ_N_S VQRDMLASHQ_N_U])
 534 (define_int_iterator VQRSHLQ_M_N [VQRSHLQ_M_N_S VQRSHLQ_M_N_U])
 535 (define_int_iterator VQSHLQ_M_R [VQSHLQ_M_R_S VQSHLQ_M_R_U])
 536 (define_int_iterator VREV64Q_M [VREV64Q_M_S VREV64Q_M_U])
 537 (define_int_iterator VRSHLQ_M_N [VRSHLQ_M_N_S VRSHLQ_M_N_U])
 538 (define_int_iterator VSHLQ_M_R [VSHLQ_M_R_S VSHLQ_M_R_U])
 539 (define_int_iterator VSLIQ_N [VSLIQ_N_S VSLIQ_N_U])
 540 (define_int_iterator VSRIQ_N [VSRIQ_N_S VSRIQ_N_U])
 541 (define_int_iterator VMLALDAVQ_P [VMLALDAVQ_P_U VMLALDAVQ_P_S])
 542 (define_int_iterator VQMOVNBQ_M [VQMOVNBQ_M_S VQMOVNBQ_M_U])
 543 (define_int_iterator VMOVLTQ_M [VMOVLTQ_M_U VMOVLTQ_M_S])
 544 (define_int_iterator VMOVNBQ_M [VMOVNBQ_M_U VMOVNBQ_M_S])
 545 (define_int_iterator VRSHRNTQ_N [VRSHRNTQ_N_U VRSHRNTQ_N_S])
 546 (define_int_iterator VORRQ_M_N [VORRQ_M_N_S VORRQ_M_N_U])
 547 (define_int_iterator VREV32Q_M [VREV32Q_M_S VREV32Q_M_U])
 548 (define_int_iterator VREV16Q_M [VREV16Q_M_S VREV16Q_M_U])
 549 (define_int_iterator VQRSHRNTQ_N [VQRSHRNTQ_N_U VQRSHRNTQ_N_S])
 550 (define_int_iterator VMOVNTQ_M [VMOVNTQ_M_U VMOVNTQ_M_S])
 551 (define_int_iterator VMOVLBQ_M [VMOVLBQ_M_U VMOVLBQ_M_S])
 552 (define_int_iterator VMLALDAVAQ [VMLALDAVAQ_S VMLALDAVAQ_U])
 553 (define_int_iterator VQSHRNBQ_N [VQSHRNBQ_N_U VQSHRNBQ_N_S])
 554 (define_int_iterator VSHRNBQ_N [VSHRNBQ_N_U VSHRNBQ_N_S])
 555 (define_int_iterator VRSHRNBQ_N [VRSHRNBQ_N_S VRSHRNBQ_N_U])
 556 (define_int_iterator VMLALDAVXQ_P [VMLALDAVXQ_P_U VMLALDAVXQ_P_S])
 557 (define_int_iterator VQMOVNTQ_M [VQMOVNTQ_M_U VQMOVNTQ_M_S])
 558 (define_int_iterator VMVNQ_M_N [VMVNQ_M_N_U VMVNQ_M_N_S])
 559 (define_int_iterator VQSHRNTQ_N [VQSHRNTQ_N_U VQSHRNTQ_N_S])
 560 (define_int_iterator VMLALDAVAXQ [VMLALDAVAXQ_S VMLALDAVAXQ_U])
 561 (define_int_iterator VSHRNTQ_N [VSHRNTQ_N_S VSHRNTQ_N_U])
 562 (define_int_iterator VCVTMQ_M [VCVTMQ_M_S VCVTMQ_M_U])
 563 (define_int_iterator VCVTNQ_M [VCVTNQ_M_S VCVTNQ_M_U])
 564 (define_int_iterator VCVTPQ_M [VCVTPQ_M_S VCVTPQ_M_U])
 565 (define_int_iterator VCVTQ_M_N_FROM_F [VCVTQ_M_N_FROM_F_S VCVTQ_M_N_FROM_F_U])
 566 (define_int_iterator VCVTQ_M_FROM_F [VCVTQ_M_FROM_F_U VCVTQ_M_FROM_F_S])
 567 (define_int_iterator VRMLALDAVHQ_P [VRMLALDAVHQ_P_S VRMLALDAVHQ_P_U])
 568 (define_int_iterator VADDLVAQ_P [VADDLVAQ_P_U VADDLVAQ_P_S])
 569 (define_int_iterator VABAVQ_P [VABAVQ_P_S VABAVQ_P_U])
 570 (define_int_iterator VSHLQ_M [VSHLQ_M_S VSHLQ_M_U])
 571 (define_int_iterator VSRIQ_M_N [VSRIQ_M_N_S VSRIQ_M_N_U])
 572 (define_int_iterator VSUBQ_M [VSUBQ_M_U VSUBQ_M_S])
 573 (define_int_iterator VCVTQ_M_N_TO_F [VCVTQ_M_N_TO_F_U VCVTQ_M_N_TO_F_S])
 574 (define_int_iterator VHSUBQ_M [VHSUBQ_M_S VHSUBQ_M_U])
 575 (define_int_iterator VSLIQ_M_N [VSLIQ_M_N_U VSLIQ_M_N_S])
 576 (define_int_iterator VRSHLQ_M [VRSHLQ_M_S VRSHLQ_M_U])
 577 (define_int_iterator VMINQ_M [VMINQ_M_S VMINQ_M_U])
 578 (define_int_iterator VMULLBQ_INT_M [VMULLBQ_INT_M_U VMULLBQ_INT_M_S])
 579 (define_int_iterator VMULHQ_M [VMULHQ_M_S VMULHQ_M_U])
 580 (define_int_iterator VMULQ_M [VMULQ_M_S VMULQ_M_U])
 581 (define_int_iterator VHSUBQ_M_N [VHSUBQ_M_N_S VHSUBQ_M_N_U])
 582 (define_int_iterator VHADDQ_M_N [VHADDQ_M_N_S VHADDQ_M_N_U])
 583 (define_int_iterator VORRQ_M [VORRQ_M_S VORRQ_M_U])
 584 (define_int_iterator VRMULHQ_M [VRMULHQ_M_U VRMULHQ_M_S])
 585 (define_int_iterator VQADDQ_M [VQADDQ_M_U VQADDQ_M_S])
 586 (define_int_iterator VRSHRQ_M_N [VRSHRQ_M_N_S VRSHRQ_M_N_U])
 587 (define_int_iterator VQSUBQ_M_N [VQSUBQ_M_N_U VQSUBQ_M_N_S])
 588 (define_int_iterator VADDQ_M [VADDQ_M_U VADDQ_M_S])
 589 (define_int_iterator VORNQ_M [VORNQ_M_U VORNQ_M_S])
 590 (define_int_iterator VRHADDQ_M [VRHADDQ_M_U VRHADDQ_M_S])
 591 (define_int_iterator VQSHLQ_M [VQSHLQ_M_U VQSHLQ_M_S])
 592 (define_int_iterator VANDQ_M [VANDQ_M_U VANDQ_M_S])
 593 (define_int_iterator VBICQ_M [VBICQ_M_U VBICQ_M_S])
 594 (define_int_iterator VSHLQ_M_N [VSHLQ_M_N_S VSHLQ_M_N_U])
 595 (define_int_iterator VCADDQ_ROT270_M [VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S])
 596 (define_int_iterator VQRSHLQ_M [VQRSHLQ_M_U VQRSHLQ_M_S])
 597 (define_int_iterator VQADDQ_M_N [VQADDQ_M_N_U VQADDQ_M_N_S])
 598 (define_int_iterator VADDQ_M_N [VADDQ_M_N_S VADDQ_M_N_U])
 599 (define_int_iterator VMAXQ_M [VMAXQ_M_S VMAXQ_M_U])
 600 (define_int_iterator VQSUBQ_M [VQSUBQ_M_U VQSUBQ_M_S])
 601 (define_int_iterator VMLASQ_M_N [VMLASQ_M_N_U VMLASQ_M_N_S])
 602 (define_int_iterator VMLADAVAQ_P [VMLADAVAQ_P_U VMLADAVAQ_P_S])
 603 (define_int_iterator VBRSRQ_M_N [VBRSRQ_M_N_U VBRSRQ_M_N_S])
 604 (define_int_iterator VMULQ_M_N [VMULQ_M_N_U VMULQ_M_N_S])
 605 (define_int_iterator VCADDQ_ROT90_M [VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S])
 606 (define_int_iterator VMULLTQ_INT_M [VMULLTQ_INT_M_S VMULLTQ_INT_M_U])
 607 (define_int_iterator VEORQ_M [VEORQ_M_S VEORQ_M_U])
 608 (define_int_iterator VSHRQ_M_N [VSHRQ_M_N_S VSHRQ_M_N_U])
 609 (define_int_iterator VSUBQ_M_N [VSUBQ_M_N_S VSUBQ_M_N_U])
 610 (define_int_iterator VHADDQ_M [VHADDQ_M_S VHADDQ_M_U])
 611 (define_int_iterator VABDQ_M [VABDQ_M_S VABDQ_M_U])
 612 (define_int_iterator VMLAQ_M_N [VMLAQ_M_N_S VMLAQ_M_N_U])
 613 (define_int_iterator VQSHLQ_M_N [VQSHLQ_M_N_S VQSHLQ_M_N_U])
 614 (define_int_iterator VMLALDAVAQ_P [VMLALDAVAQ_P_U VMLALDAVAQ_P_S])
 615 (define_int_iterator VMLALDAVAXQ_P [VMLALDAVAXQ_P_U VMLALDAVAXQ_P_S])
 616 (define_int_iterator VQRSHRNBQ_M_N [VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S])
 617 (define_int_iterator VQRSHRNTQ_M_N [VQRSHRNTQ_M_N_S VQRSHRNTQ_M_N_U])
 618 (define_int_iterator VQSHRNBQ_M_N [VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S])
 619 (define_int_iterator VQSHRNTQ_M_N [VQSHRNTQ_M_N_S VQSHRNTQ_M_N_U])
 620 (define_int_iterator VRSHRNBQ_M_N [VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S])
 621 (define_int_iterator VRSHRNTQ_M_N [VRSHRNTQ_M_N_U VRSHRNTQ_M_N_S])
 622 (define_int_iterator VSHLLBQ_M_N [VSHLLBQ_M_N_U VSHLLBQ_M_N_S])
 623 (define_int_iterator VSHLLTQ_M_N [VSHLLTQ_M_N_U VSHLLTQ_M_N_S])
 624 (define_int_iterator VSHRNBQ_M_N [VSHRNBQ_M_N_S VSHRNBQ_M_N_U])
 625 (define_int_iterator VSHRNTQ_M_N [VSHRNTQ_M_N_S VSHRNTQ_M_N_U])
 626 (define_int_iterator VSTRWSBQ [VSTRWQSB_S VSTRWQSB_U])
 627 (define_int_iterator VSTRBSOQ [VSTRBQSO_S VSTRBQSO_U])
 628 (define_int_iterator VSTRBQ [VSTRBQ_S VSTRBQ_U])
 629 (define_int_iterator VLDRBGOQ [VLDRBQGO_S VLDRBQGO_U])
 630 (define_int_iterator VLDRBQ [VLDRBQ_S VLDRBQ_U])
 631 (define_int_iterator VLDRWGBQ [VLDRWQGB_S VLDRWQGB_U])
 632 (define_int_iterator VLD1Q [VLD1Q_S VLD1Q_U])
 633 (define_int_iterator VLDRHGOQ [VLDRHQGO_S VLDRHQGO_U])
 634 (define_int_iterator VLDRHGSOQ [VLDRHQGSO_S VLDRHQGSO_U])
 635 (define_int_iterator VLDRHQ [VLDRHQ_S VLDRHQ_U])
 636 (define_int_iterator VLDRWQ [VLDRWQ_S VLDRWQ_U])
 637 (define_int_iterator VLDRDGBQ [VLDRDQGB_S VLDRDQGB_U])
 638 (define_int_iterator VLDRDGOQ [VLDRDQGO_S VLDRDQGO_U])
 639 (define_int_iterator VLDRDGSOQ [VLDRDQGSO_S VLDRDQGSO_U])
 640 (define_int_iterator VLDRWGOQ [VLDRWQGO_S VLDRWQGO_U])
 641 (define_int_iterator VLDRWGSOQ [VLDRWQGSO_S VLDRWQGSO_U])
 642 (define_int_iterator VST1Q [VST1Q_S VST1Q_U])
 643 (define_int_iterator VSTRHSOQ [VSTRHQSO_S VSTRHQSO_U])
 644 (define_int_iterator VSTRHSSOQ [VSTRHQSSO_S VSTRHQSSO_U])
 645 (define_int_iterator VSTRHQ [VSTRHQ_S VSTRHQ_U])
 646 (define_int_iterator VSTRWQ [VSTRWQ_S VSTRWQ_U])
 647 (define_int_iterator VSTRDSBQ [VSTRDQSB_S VSTRDQSB_U])
 648 (define_int_iterator VSTRDSOQ [VSTRDQSO_S VSTRDQSO_U])
 649 (define_int_iterator VSTRDSSOQ [VSTRDQSSO_S VSTRDQSSO_U])
 650 (define_int_iterator VSTRWSOQ [VSTRWQSO_S VSTRWQSO_U])
 651 (define_int_iterator VSTRWSSOQ [VSTRWQSSO_S VSTRWQSSO_U])
 652 (define_int_iterator VSTRWSBWBQ [VSTRWQSBWB_S VSTRWQSBWB_U])
 653 (define_int_iterator VLDRWGBWBQ [VLDRWQGBWB_S VLDRWQGBWB_U])
 654 (define_int_iterator VSTRDSBWBQ [VSTRDQSBWB_S VSTRDQSBWB_U])
 655 (define_int_iterator VLDRDGBWBQ [VLDRDQGBWB_S VLDRDQGBWB_U])
 656 (define_int_iterator VADCIQ [VADCIQ_U VADCIQ_S])
 657 (define_int_iterator VADCIQ_M [VADCIQ_M_U VADCIQ_M_S])
 658 (define_int_iterator VSBCQ [VSBCQ_U VSBCQ_S])
 659 (define_int_iterator VSBCQ_M [VSBCQ_M_U VSBCQ_M_S])
 660 (define_int_iterator VSBCIQ [VSBCIQ_U VSBCIQ_S])
 661 (define_int_iterator VSBCIQ_M [VSBCIQ_M_U VSBCIQ_M_S])
 662 (define_int_iterator VADCQ [VADCQ_U VADCQ_S])
 663 (define_int_iterator VADCQ_M [VADCQ_M_U VADCQ_M_S])
 664 (define_int_iterator UQRSHLLQ [UQRSHLL_64 UQRSHLL_48])
 665 (define_int_iterator SQRSHRLQ [SQRSHRL_64 SQRSHRL_48])
 666 (define_int_iterator VSHLCQ_M [VSHLCQ_M_S VSHLCQ_M_U])
 667
 668 (define_insn "*mve_mov<mode>"
 669   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
 670         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Uxi,r,Dm,w,Ul"))]
 671   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 672 {
 673   if (which_alternative == 3 || which_alternative == 6)
 674     {
 675       int width, is_valid;
 676       static char templ[40];
 677
 678       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
 679         &operands[1], &width);
 680
 681       gcc_assert (is_valid != 0);
 682
 683       if (width == 0)
 684         return "vmov.f32\t%q0, %1  @ <mode>";
 685       else
 686         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
 687       return templ;
 688     }
 689
 690   if (which_alternative == 4 || which_alternative == 7)
 691     {
 692       rtx ops[2];
 693       int regno = (which_alternative == 7)
 694                   ? REGNO (operands[1]) : REGNO (operands[0]);
 695
 696       ops[0] = operands[0];
 697       ops[1] = operands[1];
 698       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode)
 699         {
 700           if (which_alternative == 7)
 701             {
 702               ops[1] = gen_rtx_REG (DImode, regno);
 703               output_asm_insn ("vstr.64\t%P1, %E0",ops);
 704             }
 705           else
 706             {
 707               ops[0] = gen_rtx_REG (DImode, regno);
 708               output_asm_insn ("vldr.64\t%P0, %E1",ops);
 709             }
 710         }
 711       else if (<MODE>mode == TImode)
 712         {
 713           if (which_alternative == 7)
 714             output_asm_insn ("vstr.64\t%q1, %E0",ops);
 715           else
 716             output_asm_insn ("vldr.64\t%q0, %E1",ops);
 717         }
 718       else
 719         {
 720           if (which_alternative == 7)
 721             {
 722               ops[1] = gen_rtx_REG (TImode, regno);
 723               output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0",ops);
 724             }
 725           else
 726             {
 727               ops[0] = gen_rtx_REG (TImode, regno);
 728               output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1",ops);
 729             }
 730         }
 731       return "";
 732     }
 733   switch (which_alternative)
 734     {
 735     case 0:
 736       return "vmov\t%q0, %q1";
 737     case 1:
 738       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 739     case 2:
 740       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 741     case 5:
 742       return output_move_quad (operands);
 743     case 8:
 744         return output_move_neon (operands);
 745     default:
 746       gcc_unreachable ();
 747       return "";
 748     }
 749 }
 750   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
 751    (set_attr "length" "4,8,8,4,8,8,4,4,4")
 752    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
 753    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
 754
 755 (define_insn "*mve_mov<mode>"
 756   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 757         (vec_duplicate:MVE_types
 758           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 759   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 760 {
 761   if (which_alternative == 0)
 762     return "vdup.<V_sz_elem>\t%q0, %1";
 763   return "vmov.<V_sz_elem>\t%q0, %1";
 764 }
 765   [(set_attr "length" "4,4")
 766    (set_attr "type" "mve_move,mve_move")])
 767
 768 ;;
 769 ;; [vst4q])
 770 ;;
 771 (define_insn "mve_vst4q<mode>"
 772   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 773         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 774                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 775          VST4Q))
 776   ]
 777   "TARGET_HAVE_MVE"
 778 {
 779    rtx ops[6];
 780    int regno = REGNO (operands[1]);
 781    ops[0] = gen_rtx_REG (TImode, regno);
 782    ops[1] = gen_rtx_REG (TImode, regno+4);
 783    ops[2] = gen_rtx_REG (TImode, regno+8);
 784    ops[3] = gen_rtx_REG (TImode, regno+12);
 785    rtx reg  = operands[0];
 786    while (reg && !REG_P (reg))
 787     reg = XEXP (reg, 0);
 788    gcc_assert (REG_P (reg));
 789    ops[4] = reg;
 790    ops[5] = operands[0];
 791    /* Here in first three instructions data is stored to ops[4]'s location but
 792       in the fourth instruction data is stored to operands[0], this is to
 793       support the writeback.  */
 794    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 795                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 796                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 797                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 798    return "";
 799 }
 800   [(set_attr "length" "16")])
 801
 802 ;;
 803 ;; [vrndq_m_f])
 804 ;;
 805 (define_insn "mve_vrndq_m_f<mode>"
 806   [
 807    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 808         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 809                        (match_operand:MVE_0 2 "s_register_operand" "w")
 810                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 811          VRNDQ_M_F))
 812   ]
 813   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 814   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 815   [(set_attr "type" "mve_move")
 816    (set_attr "length""8")])
 817
 818 ;;
 819 ;; [vrndxq_f])
 820 ;;
 821 (define_insn "mve_vrndxq_f<mode>"
 822   [
 823    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 824         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 825          VRNDXQ_F))
 826   ]
 827   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 828   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 829   [(set_attr "type" "mve_move")
 830 ])
 831
 832 ;;
 833 ;; [vrndq_f])
 834 ;;
 835 (define_insn "mve_vrndq_f<mode>"
 836   [
 837    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 838         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 839          VRNDQ_F))
 840   ]
 841   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 842   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 843   [(set_attr "type" "mve_move")
 844 ])
 845
 846 ;;
 847 ;; [vrndpq_f])
 848 ;;
 849 (define_insn "mve_vrndpq_f<mode>"
 850   [
 851    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 852         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 853          VRNDPQ_F))
 854   ]
 855   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 856   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 857   [(set_attr "type" "mve_move")
 858 ])
 859
 860 ;;
 861 ;; [vrndnq_f])
 862 ;;
 863 (define_insn "mve_vrndnq_f<mode>"
 864   [
 865    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 866         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 867          VRNDNQ_F))
 868   ]
 869   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 870   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 871   [(set_attr "type" "mve_move")
 872 ])
 873
 874 ;;
 875 ;; [vrndmq_f])
 876 ;;
 877 (define_insn "mve_vrndmq_f<mode>"
 878   [
 879    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 880         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 881          VRNDMQ_F))
 882   ]
 883   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 884   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 885   [(set_attr "type" "mve_move")
 886 ])
 887
 888 ;;
 889 ;; [vrndaq_f])
 890 ;;
 891 (define_insn "mve_vrndaq_f<mode>"
 892   [
 893    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 894         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 895          VRNDAQ_F))
 896   ]
 897   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 898   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 899   [(set_attr "type" "mve_move")
 900 ])
 901
 902 ;;
 903 ;; [vrev64q_f])
 904 ;;
 905 (define_insn "mve_vrev64q_f<mode>"
 906   [
 907    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 908         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 909          VREV64Q_F))
 910   ]
 911   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 912   "vrev64.%#<V_sz_elem> %q0, %q1"
 913   [(set_attr "type" "mve_move")
 914 ])
 915
 916 ;;
 917 ;; [vnegq_f])
 918 ;;
 919 (define_insn "mve_vnegq_f<mode>"
 920   [
 921    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 922         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 923          VNEGQ_F))
 924   ]
 925   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 926   "vneg.f%#<V_sz_elem>  %q0, %q1"
 927   [(set_attr "type" "mve_move")
 928 ])
 929
 930 ;;
 931 ;; [vdupq_n_f])
 932 ;;
 933 (define_insn "mve_vdupq_n_f<mode>"
 934   [
 935    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 936         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 937          VDUPQ_N_F))
 938   ]
 939   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 940   "vdup.%#<V_sz_elem>   %q0, %1"
 941   [(set_attr "type" "mve_move")
 942 ])
 943
 944 ;;
 945 ;; [vabsq_f])
 946 ;;
 947 (define_insn "mve_vabsq_f<mode>"
 948   [
 949    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 950         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 951          VABSQ_F))
 952   ]
 953   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 954   "vabs.f%#<V_sz_elem>  %q0, %q1"
 955   [(set_attr "type" "mve_move")
 956 ])
 957
 958 ;;
 959 ;; [vrev32q_f])
 960 ;;
 961 (define_insn "mve_vrev32q_fv8hf"
 962   [
 963    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 964         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 965          VREV32Q_F))
 966   ]
 967   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 968   "vrev32.16 %q0, %q1"
 969   [(set_attr "type" "mve_move")
 970 ])
 971 ;;
 972 ;; [vcvttq_f32_f16])
 973 ;;
 974 (define_insn "mve_vcvttq_f32_f16v4sf"
 975   [
 976    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 977         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 978          VCVTTQ_F32_F16))
 979   ]
 980   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 981   "vcvtt.f32.f16 %q0, %q1"
 982   [(set_attr "type" "mve_move")
 983 ])
 984
 985 ;;
 986 ;; [vcvtbq_f32_f16])
 987 ;;
 988 (define_insn "mve_vcvtbq_f32_f16v4sf"
 989   [
 990    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 991         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 992          VCVTBQ_F32_F16))
 993   ]
 994   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 995   "vcvtb.f32.f16 %q0, %q1"
 996   [(set_attr "type" "mve_move")
 997 ])
 998
 999 ;;
1000 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
1001 ;;
1002 (define_insn "mve_vcvtq_to_f_<supf><mode>"
1003   [
1004    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1005         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1006          VCVTQ_TO_F))
1007   ]
1008   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1009   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
1010   [(set_attr "type" "mve_move")
1011 ])
1012
1013 ;;
1014 ;; [vrev64q_u, vrev64q_s])
1015 ;;
1016 (define_insn "mve_vrev64q_<supf><mode>"
1017   [
1018    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
1019         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1020          VREV64Q))
1021   ]
1022   "TARGET_HAVE_MVE"
1023   "vrev64.%#<V_sz_elem> %q0, %q1"
1024   [(set_attr "type" "mve_move")
1025 ])
1026
1027 ;;
1028 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
1029 ;;
1030 (define_insn "mve_vcvtq_from_f_<supf><mode>"
1031   [
1032    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1033         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1034          VCVTQ_FROM_F))
1035   ]
1036   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1037   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
1038   [(set_attr "type" "mve_move")
1039 ])
1040 ;; [vqnegq_s])
1041 ;;
1042 (define_insn "mve_vqnegq_s<mode>"
1043   [
1044    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1045         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1046          VQNEGQ_S))
1047   ]
1048   "TARGET_HAVE_MVE"
1049   "vqneg.s%#<V_sz_elem> %q0, %q1"
1050   [(set_attr "type" "mve_move")
1051 ])
1052
1053 ;;
1054 ;; [vqabsq_s])
1055 ;;
1056 (define_insn "mve_vqabsq_s<mode>"
1057   [
1058    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1059         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1060          VQABSQ_S))
1061   ]
1062   "TARGET_HAVE_MVE"
1063   "vqabs.s%#<V_sz_elem> %q0, %q1"
1064   [(set_attr "type" "mve_move")
1065 ])
1066
1067 ;;
1068 ;; [vnegq_s])
1069 ;;
1070 (define_insn "mve_vnegq_s<mode>"
1071   [
1072    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1073         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1074          VNEGQ_S))
1075   ]
1076   "TARGET_HAVE_MVE"
1077   "vneg.s%#<V_sz_elem>  %q0, %q1"
1078   [(set_attr "type" "mve_move")
1079 ])
1080
1081 ;;
1082 ;; [vmvnq_u, vmvnq_s])
1083 ;;
1084 (define_insn "mve_vmvnq_<supf><mode>"
1085   [
1086    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1087         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1088          VMVNQ))
1089   ]
1090   "TARGET_HAVE_MVE"
1091   "vmvn %q0, %q1"
1092   [(set_attr "type" "mve_move")
1093 ])
1094
1095 ;;
1096 ;; [vdupq_n_u, vdupq_n_s])
1097 ;;
1098 (define_insn "mve_vdupq_n_<supf><mode>"
1099   [
1100    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1101         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
1102          VDUPQ_N))
1103   ]
1104   "TARGET_HAVE_MVE"
1105   "vdup.%#<V_sz_elem>   %q0, %1"
1106   [(set_attr "type" "mve_move")
1107 ])
1108
1109 ;;
1110 ;; [vclzq_u, vclzq_s])
1111 ;;
1112 (define_insn "mve_vclzq_<supf><mode>"
1113   [
1114    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1115         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1116          VCLZQ))
1117   ]
1118   "TARGET_HAVE_MVE"
1119   "vclz.i%#<V_sz_elem>  %q0, %q1"
1120   [(set_attr "type" "mve_move")
1121 ])
1122
1123 ;;
1124 ;; [vclsq_s])
1125 ;;
1126 (define_insn "mve_vclsq_s<mode>"
1127   [
1128    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1129         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1130          VCLSQ_S))
1131   ]
1132   "TARGET_HAVE_MVE"
1133   "vcls.s%#<V_sz_elem>  %q0, %q1"
1134   [(set_attr "type" "mve_move")
1135 ])
1136
1137 ;;
1138 ;; [vaddvq_s, vaddvq_u])
1139 ;;
1140 (define_insn "mve_vaddvq_<supf><mode>"
1141   [
1142    (set (match_operand:SI 0 "s_register_operand" "=Te")
1143         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
1144          VADDVQ))
1145   ]
1146   "TARGET_HAVE_MVE"
1147   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
1148   [(set_attr "type" "mve_move")
1149 ])
1150
1151 ;;
1152 ;; [vabsq_s])
1153 ;;
1154 (define_insn "mve_vabsq_s<mode>"
1155   [
1156    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1157         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1158          VABSQ_S))
1159   ]
1160   "TARGET_HAVE_MVE"
1161   "vabs.s%#<V_sz_elem>\t%q0, %q1"
1162   [(set_attr "type" "mve_move")
1163 ])
1164
1165 ;;
1166 ;; [vrev32q_u, vrev32q_s])
1167 ;;
1168 (define_insn "mve_vrev32q_<supf><mode>"
1169   [
1170    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
1171         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
1172          VREV32Q))
1173   ]
1174   "TARGET_HAVE_MVE"
1175   "vrev32.%#<V_sz_elem>\t%q0, %q1"
1176   [(set_attr "type" "mve_move")
1177 ])
1178
1179 ;;
1180 ;; [vmovltq_u, vmovltq_s])
1181 ;;
1182 (define_insn "mve_vmovltq_<supf><mode>"
1183   [
1184    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1185         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1186          VMOVLTQ))
1187   ]
1188   "TARGET_HAVE_MVE"
1189   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
1190   [(set_attr "type" "mve_move")
1191 ])
1192
1193 ;;
1194 ;; [vmovlbq_s, vmovlbq_u])
1195 ;;
1196 (define_insn "mve_vmovlbq_<supf><mode>"
1197   [
1198    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1199         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1200          VMOVLBQ))
1201   ]
1202   "TARGET_HAVE_MVE"
1203   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
1204   [(set_attr "type" "mve_move")
1205 ])
1206
1207 ;;
1208 ;; [vcvtpq_s, vcvtpq_u])
1209 ;;
1210 (define_insn "mve_vcvtpq_<supf><mode>"
1211   [
1212    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1213         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1214          VCVTPQ))
1215   ]
1216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1217   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1218   [(set_attr "type" "mve_move")
1219 ])
1220
1221 ;;
1222 ;; [vcvtnq_s, vcvtnq_u])
1223 ;;
1224 (define_insn "mve_vcvtnq_<supf><mode>"
1225   [
1226    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1227         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1228          VCVTNQ))
1229   ]
1230   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1231   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1232   [(set_attr "type" "mve_move")
1233 ])
1234
1235 ;;
1236 ;; [vcvtmq_s, vcvtmq_u])
1237 ;;
1238 (define_insn "mve_vcvtmq_<supf><mode>"
1239   [
1240    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1241         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1242          VCVTMQ))
1243   ]
1244   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1245   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1246   [(set_attr "type" "mve_move")
1247 ])
1248
1249 ;;
1250 ;; [vcvtaq_u, vcvtaq_s])
1251 ;;
1252 (define_insn "mve_vcvtaq_<supf><mode>"
1253   [
1254    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1255         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1256          VCVTAQ))
1257   ]
1258   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1259   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1260   [(set_attr "type" "mve_move")
1261 ])
1262
1263 ;;
1264 ;; [vmvnq_n_u, vmvnq_n_s])
1265 ;;
1266 (define_insn "mve_vmvnq_n_<supf><mode>"
1267   [
1268    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1269         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
1270          VMVNQ_N))
1271   ]
1272   "TARGET_HAVE_MVE"
1273   "vmvn.i%#<V_sz_elem>  %q0, %1"
1274   [(set_attr "type" "mve_move")
1275 ])
1276
1277 ;;
1278 ;; [vrev16q_u, vrev16q_s])
1279 ;;
1280 (define_insn "mve_vrev16q_<supf>v16qi"
1281   [
1282    (set (match_operand:V16QI 0 "s_register_operand" "=w")
1283         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
1284          VREV16Q))
1285   ]
1286   "TARGET_HAVE_MVE"
1287   "vrev16.8 %q0, %q1"
1288   [(set_attr "type" "mve_move")
1289 ])
1290
1291 ;;
1292 ;; [vaddlvq_s vaddlvq_u])
1293 ;;
1294 (define_insn "mve_vaddlvq_<supf>v4si"
1295   [
1296    (set (match_operand:DI 0 "s_register_operand" "=r")
1297         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
1298          VADDLVQ))
1299   ]
1300   "TARGET_HAVE_MVE"
1301   "vaddlv.<supf>32 %Q0, %R0, %q1"
1302   [(set_attr "type" "mve_move")
1303 ])
1304
1305 ;;
1306 ;; [vctp8q vctp16q vctp32q vctp64q])
1307 ;;
1308 (define_insn "mve_vctp<mode1>qhi"
1309   [
1310    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1311         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
1312         VCTPQ))
1313   ]
1314   "TARGET_HAVE_MVE"
1315   "vctp.<mode1> %1"
1316   [(set_attr "type" "mve_move")
1317 ])
1318
1319 ;;
1320 ;; [vpnot])
1321 ;;
1322 (define_insn "mve_vpnothi"
1323   [
1324    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1325         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
1326          VPNOT))
1327   ]
1328   "TARGET_HAVE_MVE"
1329   "vpnot"
1330   [(set_attr "type" "mve_move")
1331 ])
1332
1333 ;;
1334 ;; [vsubq_n_f])
1335 ;;
1336 (define_insn "mve_vsubq_n_f<mode>"
1337   [
1338    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1339         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1340                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1341          VSUBQ_N_F))
1342   ]
1343   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1344   "vsub.f<V_sz_elem>  %q0, %q1, %2"
1345   [(set_attr "type" "mve_move")
1346 ])
1347
1348 ;;
1349 ;; [vbrsrq_n_f])
1350 ;;
1351 (define_insn "mve_vbrsrq_n_f<mode>"
1352   [
1353    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1354         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1355                        (match_operand:SI 2 "s_register_operand" "r")]
1356          VBRSRQ_N_F))
1357   ]
1358   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1359   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
1360   [(set_attr "type" "mve_move")
1361 ])
1362
1363 ;;
1364 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
1365 ;;
1366 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
1367   [
1368    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1369         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1370                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1371          VCVTQ_N_TO_F))
1372   ]
1373   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1374   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
1375   [(set_attr "type" "mve_move")
1376 ])
1377
1378 ;; [vcreateq_f])
1379 ;;
1380 (define_insn "mve_vcreateq_f<mode>"
1381   [
1382    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1383         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
1384                        (match_operand:DI 2 "s_register_operand" "r")]
1385          VCREATEQ_F))
1386   ]
1387   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1388   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1389   [(set_attr "type" "mve_move")
1390    (set_attr "length""8")])
1391
1392 ;;
1393 ;; [vcreateq_u, vcreateq_s])
1394 ;;
1395 (define_insn "mve_vcreateq_<supf><mode>"
1396   [
1397    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
1398         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
1399                        (match_operand:DI 2 "s_register_operand" "r")]
1400          VCREATEQ))
1401   ]
1402   "TARGET_HAVE_MVE"
1403   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1404   [(set_attr "type" "mve_move")
1405    (set_attr "length""8")])
1406
1407 ;;
1408 ;; [vshrq_n_s, vshrq_n_u])
1409 ;;
1410 (define_insn "mve_vshrq_n_<supf><mode>"
1411   [
1412    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1413         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1414                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1415          VSHRQ_N))
1416   ]
1417   "TARGET_HAVE_MVE"
1418   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
1419   [(set_attr "type" "mve_move")
1420 ])
1421
1422 ;;
1423 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
1424 ;;
1425 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
1426   [
1427    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1428         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1429                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1430          VCVTQ_N_FROM_F))
1431   ]
1432   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1433   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
1434   [(set_attr "type" "mve_move")
1435 ])
1436
1437 ;;
1438 ;; [vaddlvq_p_s])
1439 ;;
1440 (define_insn "mve_vaddlvq_p_<supf>v4si"
1441   [
1442    (set (match_operand:DI 0 "s_register_operand" "=r")
1443         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
1444                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1445          VADDLVQ_P))
1446   ]
1447   "TARGET_HAVE_MVE"
1448   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
1449   [(set_attr "type" "mve_move")
1450    (set_attr "length""8")])
1451
1452 ;;
1453 ;; [vcmpneq_u, vcmpneq_s])
1454 ;;
1455 (define_insn "mve_vcmpneq_<supf><mode>"
1456   [
1457    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1458         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1459                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1460          VCMPNEQ))
1461   ]
1462   "TARGET_HAVE_MVE"
1463   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
1464   [(set_attr "type" "mve_move")
1465 ])
1466
1467 ;;
1468 ;; [vshlq_s, vshlq_u])
1469 ;;
1470 (define_insn "mve_vshlq_<supf><mode>"
1471   [
1472    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1473         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1474                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1475          VSHLQ))
1476   ]
1477   "TARGET_HAVE_MVE"
1478   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1479   [(set_attr "type" "mve_move")
1480 ])
1481
1482 ;;
1483 ;; [vabdq_s, vabdq_u])
1484 ;;
1485 (define_insn "mve_vabdq_<supf><mode>"
1486   [
1487    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1488         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1489                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1490          VABDQ))
1491   ]
1492   "TARGET_HAVE_MVE"
1493   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
1494   [(set_attr "type" "mve_move")
1495 ])
1496
1497 ;;
1498 ;; [vaddq_n_s, vaddq_n_u])
1499 ;;
1500 (define_insn "mve_vaddq_n_<supf><mode>"
1501   [
1502    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1503         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1504                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1505          VADDQ_N))
1506   ]
1507   "TARGET_HAVE_MVE"
1508   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
1509   [(set_attr "type" "mve_move")
1510 ])
1511
1512 ;;
1513 ;; [vaddvaq_s, vaddvaq_u])
1514 ;;
1515 (define_insn "mve_vaddvaq_<supf><mode>"
1516   [
1517    (set (match_operand:SI 0 "s_register_operand" "=Te")
1518         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1519                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1520          VADDVAQ))
1521   ]
1522   "TARGET_HAVE_MVE"
1523   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
1524   [(set_attr "type" "mve_move")
1525 ])
1526
1527 ;;
1528 ;; [vaddvq_p_u, vaddvq_p_s])
1529 ;;
1530 (define_insn "mve_vaddvq_p_<supf><mode>"
1531   [
1532    (set (match_operand:SI 0 "s_register_operand" "=Te")
1533         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1534                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1535          VADDVQ_P))
1536   ]
1537   "TARGET_HAVE_MVE"
1538   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
1539   [(set_attr "type" "mve_move")
1540    (set_attr "length""8")])
1541
1542 ;;
1543 ;; [vandq_u, vandq_s])
1544 ;;
1545 (define_insn "mve_vandq_<supf><mode>"
1546   [
1547    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1548         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1549                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1550          VANDQ))
1551   ]
1552   "TARGET_HAVE_MVE"
1553   "vand %q0, %q1, %q2"
1554   [(set_attr "type" "mve_move")
1555 ])
1556
1557 ;;
1558 ;; [vbicq_s, vbicq_u])
1559 ;;
1560 (define_insn "mve_vbicq_<supf><mode>"
1561   [
1562    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1563         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1564                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1565          VBICQ))
1566   ]
1567   "TARGET_HAVE_MVE"
1568   "vbic %q0, %q1, %q2"
1569   [(set_attr "type" "mve_move")
1570 ])
1571
1572 ;;
1573 ;; [vbrsrq_n_u, vbrsrq_n_s])
1574 ;;
1575 (define_insn "mve_vbrsrq_n_<supf><mode>"
1576   [
1577    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1578         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1579                        (match_operand:SI 2 "s_register_operand" "r")]
1580          VBRSRQ_N))
1581   ]
1582   "TARGET_HAVE_MVE"
1583   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
1584   [(set_attr "type" "mve_move")
1585 ])
1586
1587 ;;
1588 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
1589 ;;
1590 (define_insn "mve_vcaddq_rot270_<supf><mode>"
1591   [
1592    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1593         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1594                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1595          VCADDQ_ROT270))
1596   ]
1597   "TARGET_HAVE_MVE"
1598   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
1599   [(set_attr "type" "mve_move")
1600 ])
1601
1602 ;;
1603 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
1604 ;;
1605 (define_insn "mve_vcaddq_rot90_<supf><mode>"
1606   [
1607    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1608         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1609                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1610          VCADDQ_ROT90))
1611   ]
1612   "TARGET_HAVE_MVE"
1613   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
1614   [(set_attr "type" "mve_move")
1615 ])
1616
1617 ;;
1618 ;; [vcmpcsq_n_u])
1619 ;;
1620 (define_insn "mve_vcmpcsq_n_u<mode>"
1621   [
1622    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1623         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1624                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1625          VCMPCSQ_N_U))
1626   ]
1627   "TARGET_HAVE_MVE"
1628   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1629   [(set_attr "type" "mve_move")
1630 ])
1631
1632 ;;
1633 ;; [vcmpcsq_u])
1634 ;;
1635 (define_insn "mve_vcmpcsq_u<mode>"
1636   [
1637    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1638         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1639                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1640          VCMPCSQ_U))
1641   ]
1642   "TARGET_HAVE_MVE"
1643   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1644   [(set_attr "type" "mve_move")
1645 ])
1646
1647 ;;
1648 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1649 ;;
1650 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1651   [
1652    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1653         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1654                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1655          VCMPEQQ_N))
1656   ]
1657   "TARGET_HAVE_MVE"
1658   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1659   [(set_attr "type" "mve_move")
1660 ])
1661
1662 ;;
1663 ;; [vcmpeqq_u, vcmpeqq_s])
1664 ;;
1665 (define_insn "mve_vcmpeqq_<supf><mode>"
1666   [
1667    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1668         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1669                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1670          VCMPEQQ))
1671   ]
1672   "TARGET_HAVE_MVE"
1673   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1674   [(set_attr "type" "mve_move")
1675 ])
1676
1677 ;;
1678 ;; [vcmpgeq_n_s])
1679 ;;
1680 (define_insn "mve_vcmpgeq_n_s<mode>"
1681   [
1682    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1683         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1684                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1685          VCMPGEQ_N_S))
1686   ]
1687   "TARGET_HAVE_MVE"
1688   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1689   [(set_attr "type" "mve_move")
1690 ])
1691
1692 ;;
1693 ;; [vcmpgeq_s])
1694 ;;
1695 (define_insn "mve_vcmpgeq_s<mode>"
1696   [
1697    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1698         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1699                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1700          VCMPGEQ_S))
1701   ]
1702   "TARGET_HAVE_MVE"
1703   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1704   [(set_attr "type" "mve_move")
1705 ])
1706
1707 ;;
1708 ;; [vcmpgtq_n_s])
1709 ;;
1710 (define_insn "mve_vcmpgtq_n_s<mode>"
1711   [
1712    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1713         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1714                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1715          VCMPGTQ_N_S))
1716   ]
1717   "TARGET_HAVE_MVE"
1718   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1719   [(set_attr "type" "mve_move")
1720 ])
1721
1722 ;;
1723 ;; [vcmpgtq_s])
1724 ;;
1725 (define_insn "mve_vcmpgtq_s<mode>"
1726   [
1727    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1728         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1729                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1730          VCMPGTQ_S))
1731   ]
1732   "TARGET_HAVE_MVE"
1733   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1734   [(set_attr "type" "mve_move")
1735 ])
1736
1737 ;;
1738 ;; [vcmphiq_n_u])
1739 ;;
1740 (define_insn "mve_vcmphiq_n_u<mode>"
1741   [
1742    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1743         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1744                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1745          VCMPHIQ_N_U))
1746   ]
1747   "TARGET_HAVE_MVE"
1748   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1749   [(set_attr "type" "mve_move")
1750 ])
1751
1752 ;;
1753 ;; [vcmphiq_u])
1754 ;;
1755 (define_insn "mve_vcmphiq_u<mode>"
1756   [
1757    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1758         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1759                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1760          VCMPHIQ_U))
1761   ]
1762   "TARGET_HAVE_MVE"
1763   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1764   [(set_attr "type" "mve_move")
1765 ])
1766
1767 ;;
1768 ;; [vcmpleq_n_s])
1769 ;;
1770 (define_insn "mve_vcmpleq_n_s<mode>"
1771   [
1772    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1773         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1774                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1775          VCMPLEQ_N_S))
1776   ]
1777   "TARGET_HAVE_MVE"
1778   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1779   [(set_attr "type" "mve_move")
1780 ])
1781
1782 ;;
1783 ;; [vcmpleq_s])
1784 ;;
1785 (define_insn "mve_vcmpleq_s<mode>"
1786   [
1787    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1788         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1789                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1790          VCMPLEQ_S))
1791   ]
1792   "TARGET_HAVE_MVE"
1793   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1794   [(set_attr "type" "mve_move")
1795 ])
1796
1797 ;;
1798 ;; [vcmpltq_n_s])
1799 ;;
1800 (define_insn "mve_vcmpltq_n_s<mode>"
1801   [
1802    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1803         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1804                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1805          VCMPLTQ_N_S))
1806   ]
1807   "TARGET_HAVE_MVE"
1808   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1809   [(set_attr "type" "mve_move")
1810 ])
1811
1812 ;;
1813 ;; [vcmpltq_s])
1814 ;;
1815 (define_insn "mve_vcmpltq_s<mode>"
1816   [
1817    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1818         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1819                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1820          VCMPLTQ_S))
1821   ]
1822   "TARGET_HAVE_MVE"
1823   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1824   [(set_attr "type" "mve_move")
1825 ])
1826
1827 ;;
1828 ;; [vcmpneq_n_u, vcmpneq_n_s])
1829 ;;
1830 (define_insn "mve_vcmpneq_n_<supf><mode>"
1831   [
1832    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1833         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1834                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1835          VCMPNEQ_N))
1836   ]
1837   "TARGET_HAVE_MVE"
1838   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1839   [(set_attr "type" "mve_move")
1840 ])
1841
1842 ;;
1843 ;; [veorq_u, veorq_s])
1844 ;;
1845 (define_insn "mve_veorq_<supf><mode>"
1846   [
1847    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1848         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1849                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1850          VEORQ))
1851   ]
1852   "TARGET_HAVE_MVE"
1853   "veor %q0, %q1, %q2"
1854   [(set_attr "type" "mve_move")
1855 ])
1856
1857 ;;
1858 ;; [vhaddq_n_u, vhaddq_n_s])
1859 ;;
1860 (define_insn "mve_vhaddq_n_<supf><mode>"
1861   [
1862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1864                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1865          VHADDQ_N))
1866   ]
1867   "TARGET_HAVE_MVE"
1868   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1869   [(set_attr "type" "mve_move")
1870 ])
1871
1872 ;;
1873 ;; [vhaddq_s, vhaddq_u])
1874 ;;
1875 (define_insn "mve_vhaddq_<supf><mode>"
1876   [
1877    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1878         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1879                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1880          VHADDQ))
1881   ]
1882   "TARGET_HAVE_MVE"
1883   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1884   [(set_attr "type" "mve_move")
1885 ])
1886
1887 ;;
1888 ;; [vhcaddq_rot270_s])
1889 ;;
1890 (define_insn "mve_vhcaddq_rot270_s<mode>"
1891   [
1892    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1893         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1894                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1895          VHCADDQ_ROT270_S))
1896   ]
1897   "TARGET_HAVE_MVE"
1898   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1899   [(set_attr "type" "mve_move")
1900 ])
1901
1902 ;;
1903 ;; [vhcaddq_rot90_s])
1904 ;;
1905 (define_insn "mve_vhcaddq_rot90_s<mode>"
1906   [
1907    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1908         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1909                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1910          VHCADDQ_ROT90_S))
1911   ]
1912   "TARGET_HAVE_MVE"
1913   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1914   [(set_attr "type" "mve_move")
1915 ])
1916
1917 ;;
1918 ;; [vhsubq_n_u, vhsubq_n_s])
1919 ;;
1920 (define_insn "mve_vhsubq_n_<supf><mode>"
1921   [
1922    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1923         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1924                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1925          VHSUBQ_N))
1926   ]
1927   "TARGET_HAVE_MVE"
1928   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1929   [(set_attr "type" "mve_move")
1930 ])
1931
1932 ;;
1933 ;; [vhsubq_s, vhsubq_u])
1934 ;;
1935 (define_insn "mve_vhsubq_<supf><mode>"
1936   [
1937    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1938         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1939                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1940          VHSUBQ))
1941   ]
1942   "TARGET_HAVE_MVE"
1943   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1944   [(set_attr "type" "mve_move")
1945 ])
1946
1947 ;;
1948 ;; [vmaxaq_s])
1949 ;;
1950 (define_insn "mve_vmaxaq_s<mode>"
1951   [
1952    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1953         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1954                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1955          VMAXAQ_S))
1956   ]
1957   "TARGET_HAVE_MVE"
1958   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1959   [(set_attr "type" "mve_move")
1960 ])
1961
1962 ;;
1963 ;; [vmaxavq_s])
1964 ;;
1965 (define_insn "mve_vmaxavq_s<mode>"
1966   [
1967    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1968         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1969                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1970          VMAXAVQ_S))
1971   ]
1972   "TARGET_HAVE_MVE"
1973   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1974   [(set_attr "type" "mve_move")
1975 ])
1976
1977 ;;
1978 ;; [vmaxq_u, vmaxq_s])
1979 ;;
1980 (define_insn "mve_vmaxq_<supf><mode>"
1981   [
1982    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1983         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1984                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1985          VMAXQ))
1986   ]
1987   "TARGET_HAVE_MVE"
1988   "vmax.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1989   [(set_attr "type" "mve_move")
1990 ])
1991
1992 ;;
1993 ;; [vmaxvq_u, vmaxvq_s])
1994 ;;
1995 (define_insn "mve_vmaxvq_<supf><mode>"
1996   [
1997    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1998         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1999                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2000          VMAXVQ))
2001   ]
2002   "TARGET_HAVE_MVE"
2003   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
2004   [(set_attr "type" "mve_move")
2005 ])
2006
2007 ;;
2008 ;; [vminaq_s])
2009 ;;
2010 (define_insn "mve_vminaq_s<mode>"
2011   [
2012    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2013         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2014                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2015          VMINAQ_S))
2016   ]
2017   "TARGET_HAVE_MVE"
2018   "vmina.s%#<V_sz_elem>\t%q0, %q2"
2019   [(set_attr "type" "mve_move")
2020 ])
2021
2022 ;;
2023 ;; [vminavq_s])
2024 ;;
2025 (define_insn "mve_vminavq_s<mode>"
2026   [
2027    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2028         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2029                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2030          VMINAVQ_S))
2031   ]
2032   "TARGET_HAVE_MVE"
2033   "vminav.s%#<V_sz_elem>\t%0, %q2"
2034   [(set_attr "type" "mve_move")
2035 ])
2036
2037 ;;
2038 ;; [vminq_s, vminq_u])
2039 ;;
2040 (define_insn "mve_vminq_<supf><mode>"
2041   [
2042    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2043         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2044                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2045          VMINQ))
2046   ]
2047   "TARGET_HAVE_MVE"
2048   "vmin.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2049   [(set_attr "type" "mve_move")
2050 ])
2051
2052 ;;
2053 ;; [vminvq_u, vminvq_s])
2054 ;;
2055 (define_insn "mve_vminvq_<supf><mode>"
2056   [
2057    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2058         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2059                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2060          VMINVQ))
2061   ]
2062   "TARGET_HAVE_MVE"
2063   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
2064   [(set_attr "type" "mve_move")
2065 ])
2066
2067 ;;
2068 ;; [vmladavq_u, vmladavq_s])
2069 ;;
2070 (define_insn "mve_vmladavq_<supf><mode>"
2071   [
2072    (set (match_operand:SI 0 "s_register_operand" "=Te")
2073         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2074                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2075          VMLADAVQ))
2076   ]
2077   "TARGET_HAVE_MVE"
2078   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
2079   [(set_attr "type" "mve_move")
2080 ])
2081
2082 ;;
2083 ;; [vmladavxq_s])
2084 ;;
2085 (define_insn "mve_vmladavxq_s<mode>"
2086   [
2087    (set (match_operand:SI 0 "s_register_operand" "=Te")
2088         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2089                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2090          VMLADAVXQ_S))
2091   ]
2092   "TARGET_HAVE_MVE"
2093   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2094   [(set_attr "type" "mve_move")
2095 ])
2096
2097 ;;
2098 ;; [vmlsdavq_s])
2099 ;;
2100 (define_insn "mve_vmlsdavq_s<mode>"
2101   [
2102    (set (match_operand:SI 0 "s_register_operand" "=Te")
2103         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2104                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2105          VMLSDAVQ_S))
2106   ]
2107   "TARGET_HAVE_MVE"
2108   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
2109   [(set_attr "type" "mve_move")
2110 ])
2111
2112 ;;
2113 ;; [vmlsdavxq_s])
2114 ;;
2115 (define_insn "mve_vmlsdavxq_s<mode>"
2116   [
2117    (set (match_operand:SI 0 "s_register_operand" "=Te")
2118         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2119                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2120          VMLSDAVXQ_S))
2121   ]
2122   "TARGET_HAVE_MVE"
2123   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2124   [(set_attr "type" "mve_move")
2125 ])
2126
2127 ;;
2128 ;; [vmulhq_s, vmulhq_u])
2129 ;;
2130 (define_insn "mve_vmulhq_<supf><mode>"
2131   [
2132    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2133         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2134                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2135          VMULHQ))
2136   ]
2137   "TARGET_HAVE_MVE"
2138   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2139   [(set_attr "type" "mve_move")
2140 ])
2141
2142 ;;
2143 ;; [vmullbq_int_u, vmullbq_int_s])
2144 ;;
2145 (define_insn "mve_vmullbq_int_<supf><mode>"
2146   [
2147    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2148         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2149                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2150          VMULLBQ_INT))
2151   ]
2152   "TARGET_HAVE_MVE"
2153   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2154   [(set_attr "type" "mve_move")
2155 ])
2156
2157 ;;
2158 ;; [vmulltq_int_u, vmulltq_int_s])
2159 ;;
2160 (define_insn "mve_vmulltq_int_<supf><mode>"
2161   [
2162    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2163         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2164                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2165          VMULLTQ_INT))
2166   ]
2167   "TARGET_HAVE_MVE"
2168   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2169   [(set_attr "type" "mve_move")
2170 ])
2171
2172 ;;
2173 ;; [vmulq_n_u, vmulq_n_s])
2174 ;;
2175 (define_insn "mve_vmulq_n_<supf><mode>"
2176   [
2177    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2178         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2179                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2180          VMULQ_N))
2181   ]
2182   "TARGET_HAVE_MVE"
2183   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
2184   [(set_attr "type" "mve_move")
2185 ])
2186
2187 ;;
2188 ;; [vmulq_u, vmulq_s])
2189 ;;
2190 (define_insn "mve_vmulq_<supf><mode>"
2191   [
2192    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2193         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2194                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2195          VMULQ))
2196   ]
2197   "TARGET_HAVE_MVE"
2198   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
2199   [(set_attr "type" "mve_move")
2200 ])
2201
2202 ;;
2203 ;; [vornq_u, vornq_s])
2204 ;;
2205 (define_insn "mve_vornq_<supf><mode>"
2206   [
2207    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2208         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2209                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2210          VORNQ))
2211   ]
2212   "TARGET_HAVE_MVE"
2213   "vorn %q0, %q1, %q2"
2214   [(set_attr "type" "mve_move")
2215 ])
2216
2217 ;;
2218 ;; [vorrq_s, vorrq_u])
2219 ;;
2220 (define_insn "mve_vorrq_<supf><mode>"
2221   [
2222    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2223         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2224                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2225          VORRQ))
2226   ]
2227   "TARGET_HAVE_MVE"
2228   "vorr %q0, %q1, %q2"
2229   [(set_attr "type" "mve_move")
2230 ])
2231
2232 ;;
2233 ;; [vqaddq_n_s, vqaddq_n_u])
2234 ;;
2235 (define_insn "mve_vqaddq_n_<supf><mode>"
2236   [
2237    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2238         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2239                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2240          VQADDQ_N))
2241   ]
2242   "TARGET_HAVE_MVE"
2243   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2244   [(set_attr "type" "mve_move")
2245 ])
2246
2247 ;;
2248 ;; [vqaddq_u, vqaddq_s])
2249 ;;
2250 (define_insn "mve_vqaddq_<supf><mode>"
2251   [
2252    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2253         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2254                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2255          VQADDQ))
2256   ]
2257   "TARGET_HAVE_MVE"
2258   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2259   [(set_attr "type" "mve_move")
2260 ])
2261
2262 ;;
2263 ;; [vqdmulhq_n_s])
2264 ;;
2265 (define_insn "mve_vqdmulhq_n_s<mode>"
2266   [
2267    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2268         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2269                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2270          VQDMULHQ_N_S))
2271   ]
2272   "TARGET_HAVE_MVE"
2273   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2274   [(set_attr "type" "mve_move")
2275 ])
2276
2277 ;;
2278 ;; [vqdmulhq_s])
2279 ;;
2280 (define_insn "mve_vqdmulhq_s<mode>"
2281   [
2282    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2283         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2284                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2285          VQDMULHQ_S))
2286   ]
2287   "TARGET_HAVE_MVE"
2288   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2289   [(set_attr "type" "mve_move")
2290 ])
2291
2292 ;;
2293 ;; [vqrdmulhq_n_s])
2294 ;;
2295 (define_insn "mve_vqrdmulhq_n_s<mode>"
2296   [
2297    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2298         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2299                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2300          VQRDMULHQ_N_S))
2301   ]
2302   "TARGET_HAVE_MVE"
2303   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2304   [(set_attr "type" "mve_move")
2305 ])
2306
2307 ;;
2308 ;; [vqrdmulhq_s])
2309 ;;
2310 (define_insn "mve_vqrdmulhq_s<mode>"
2311   [
2312    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2313         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2314                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2315          VQRDMULHQ_S))
2316   ]
2317   "TARGET_HAVE_MVE"
2318   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2319   [(set_attr "type" "mve_move")
2320 ])
2321
2322 ;;
2323 ;; [vqrshlq_n_s, vqrshlq_n_u])
2324 ;;
2325 (define_insn "mve_vqrshlq_n_<supf><mode>"
2326   [
2327    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2328         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2329                        (match_operand:SI 2 "s_register_operand" "r")]
2330          VQRSHLQ_N))
2331   ]
2332   "TARGET_HAVE_MVE"
2333   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2334   [(set_attr "type" "mve_move")
2335 ])
2336
2337 ;;
2338 ;; [vqrshlq_s, vqrshlq_u])
2339 ;;
2340 (define_insn "mve_vqrshlq_<supf><mode>"
2341   [
2342    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2343         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2344                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2345          VQRSHLQ))
2346   ]
2347   "TARGET_HAVE_MVE"
2348   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2349   [(set_attr "type" "mve_move")
2350 ])
2351
2352 ;;
2353 ;; [vqshlq_n_s, vqshlq_n_u])
2354 ;;
2355 (define_insn "mve_vqshlq_n_<supf><mode>"
2356   [
2357    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2358         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2359                        (match_operand:SI 2 "immediate_operand" "i")]
2360          VQSHLQ_N))
2361   ]
2362   "TARGET_HAVE_MVE"
2363   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2364   [(set_attr "type" "mve_move")
2365 ])
2366
2367 ;;
2368 ;; [vqshlq_r_u, vqshlq_r_s])
2369 ;;
2370 (define_insn "mve_vqshlq_r_<supf><mode>"
2371   [
2372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2374                        (match_operand:SI 2 "s_register_operand" "r")]
2375          VQSHLQ_R))
2376   ]
2377   "TARGET_HAVE_MVE"
2378   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
2379   [(set_attr "type" "mve_move")
2380 ])
2381
2382 ;;
2383 ;; [vqshlq_s, vqshlq_u])
2384 ;;
2385 (define_insn "mve_vqshlq_<supf><mode>"
2386   [
2387    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2388         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2389                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2390          VQSHLQ))
2391   ]
2392   "TARGET_HAVE_MVE"
2393   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2394   [(set_attr "type" "mve_move")
2395 ])
2396
2397 ;;
2398 ;; [vqshluq_n_s])
2399 ;;
2400 (define_insn "mve_vqshluq_n_s<mode>"
2401   [
2402    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2403         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2404                        (match_operand:SI 2 "mve_imm_7" "Ra")]
2405          VQSHLUQ_N_S))
2406   ]
2407   "TARGET_HAVE_MVE"
2408   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
2409   [(set_attr "type" "mve_move")
2410 ])
2411
2412 ;;
2413 ;; [vqsubq_n_s, vqsubq_n_u])
2414 ;;
2415 (define_insn "mve_vqsubq_n_<supf><mode>"
2416   [
2417    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2418         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2419                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2420          VQSUBQ_N))
2421   ]
2422   "TARGET_HAVE_MVE"
2423   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2424   [(set_attr "type" "mve_move")
2425 ])
2426
2427 ;;
2428 ;; [vqsubq_u, vqsubq_s])
2429 ;;
2430 (define_insn "mve_vqsubq_<supf><mode>"
2431   [
2432    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2433         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2434                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2435          VQSUBQ))
2436   ]
2437   "TARGET_HAVE_MVE"
2438   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2439   [(set_attr "type" "mve_move")
2440 ])
2441
2442 ;;
2443 ;; [vrhaddq_s, vrhaddq_u])
2444 ;;
2445 (define_insn "mve_vrhaddq_<supf><mode>"
2446   [
2447    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2448         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2449                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2450          VRHADDQ))
2451   ]
2452   "TARGET_HAVE_MVE"
2453   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2454   [(set_attr "type" "mve_move")
2455 ])
2456
2457 ;;
2458 ;; [vrmulhq_s, vrmulhq_u])
2459 ;;
2460 (define_insn "mve_vrmulhq_<supf><mode>"
2461   [
2462    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2463         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2464                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2465          VRMULHQ))
2466   ]
2467   "TARGET_HAVE_MVE"
2468   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2469   [(set_attr "type" "mve_move")
2470 ])
2471
2472 ;;
2473 ;; [vrshlq_n_u, vrshlq_n_s])
2474 ;;
2475 (define_insn "mve_vrshlq_n_<supf><mode>"
2476   [
2477    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2478         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2479                        (match_operand:SI 2 "s_register_operand" "r")]
2480          VRSHLQ_N))
2481   ]
2482   "TARGET_HAVE_MVE"
2483   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2484   [(set_attr "type" "mve_move")
2485 ])
2486
2487 ;;
2488 ;; [vrshlq_s, vrshlq_u])
2489 ;;
2490 (define_insn "mve_vrshlq_<supf><mode>"
2491   [
2492    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2493         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2494                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2495          VRSHLQ))
2496   ]
2497   "TARGET_HAVE_MVE"
2498   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2499   [(set_attr "type" "mve_move")
2500 ])
2501
2502 ;;
2503 ;; [vrshrq_n_s, vrshrq_n_u])
2504 ;;
2505 (define_insn "mve_vrshrq_n_<supf><mode>"
2506   [
2507    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2508         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2509                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
2510          VRSHRQ_N))
2511   ]
2512   "TARGET_HAVE_MVE"
2513   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2514   [(set_attr "type" "mve_move")
2515 ])
2516
2517 ;;
2518 ;; [vshlq_n_u, vshlq_n_s])
2519 ;;
2520 (define_insn "mve_vshlq_n_<supf><mode>"
2521   [
2522    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2523         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2524                        (match_operand:SI 2 "immediate_operand" "i")]
2525          VSHLQ_N))
2526   ]
2527   "TARGET_HAVE_MVE"
2528   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2529   [(set_attr "type" "mve_move")
2530 ])
2531
2532 ;;
2533 ;; [vshlq_r_s, vshlq_r_u])
2534 ;;
2535 (define_insn "mve_vshlq_r_<supf><mode>"
2536   [
2537    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2538         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2539                        (match_operand:SI 2 "s_register_operand" "r")]
2540          VSHLQ_R))
2541   ]
2542   "TARGET_HAVE_MVE"
2543   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
2544   [(set_attr "type" "mve_move")
2545 ])
2546
2547 ;;
2548 ;; [vsubq_n_s, vsubq_n_u])
2549 ;;
2550 (define_insn "mve_vsubq_n_<supf><mode>"
2551   [
2552    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2553         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2554                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2555          VSUBQ_N))
2556   ]
2557   "TARGET_HAVE_MVE"
2558   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
2559   [(set_attr "type" "mve_move")
2560 ])
2561
2562 ;;
2563 ;; [vsubq_s, vsubq_u])
2564 ;;
2565 (define_insn "mve_vsubq_<supf><mode>"
2566   [
2567    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2568         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2569                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2570          VSUBQ))
2571   ]
2572   "TARGET_HAVE_MVE"
2573   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2574   [(set_attr "type" "mve_move")
2575 ])
2576
2577 ;;
2578 ;; [vabdq_f])
2579 ;;
2580 (define_insn "mve_vabdq_f<mode>"
2581   [
2582    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2583         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2584                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2585          VABDQ_F))
2586   ]
2587   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2588   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2589   [(set_attr "type" "mve_move")
2590 ])
2591
2592 ;;
2593 ;; [vaddlvaq_s vaddlvaq_u])
2594 ;;
2595 (define_insn "mve_vaddlvaq_<supf>v4si"
2596   [
2597    (set (match_operand:DI 0 "s_register_operand" "=r")
2598         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2599                     (match_operand:V4SI 2 "s_register_operand" "w")]
2600          VADDLVAQ))
2601   ]
2602   "TARGET_HAVE_MVE"
2603   "vaddlva.<supf>32 %Q0, %R0, %q2"
2604   [(set_attr "type" "mve_move")
2605 ])
2606
2607 ;;
2608 ;; [vaddq_n_f])
2609 ;;
2610 (define_insn "mve_vaddq_n_f<mode>"
2611   [
2612    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2613         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2614                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2615          VADDQ_N_F))
2616   ]
2617   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2618   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2619   [(set_attr "type" "mve_move")
2620 ])
2621
2622 ;;
2623 ;; [vandq_f])
2624 ;;
2625 (define_insn "mve_vandq_f<mode>"
2626   [
2627    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2628         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2629                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2630          VANDQ_F))
2631   ]
2632   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2633   "vand %q0, %q1, %q2"
2634   [(set_attr "type" "mve_move")
2635 ])
2636
2637 ;;
2638 ;; [vbicq_f])
2639 ;;
2640 (define_insn "mve_vbicq_f<mode>"
2641   [
2642    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2643         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2644                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2645          VBICQ_F))
2646   ]
2647   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2648   "vbic %q0, %q1, %q2"
2649   [(set_attr "type" "mve_move")
2650 ])
2651
2652 ;;
2653 ;; [vbicq_n_s, vbicq_n_u])
2654 ;;
2655 (define_insn "mve_vbicq_n_<supf><mode>"
2656   [
2657    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2658         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2659                        (match_operand:SI 2 "immediate_operand" "i")]
2660          VBICQ_N))
2661   ]
2662   "TARGET_HAVE_MVE"
2663   "vbic.i%#<V_sz_elem>  %q0, %2"
2664   [(set_attr "type" "mve_move")
2665 ])
2666
2667 ;;
2668 ;; [vcaddq_rot270_f])
2669 ;;
2670 (define_insn "mve_vcaddq_rot270_f<mode>"
2671   [
2672    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2673         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2674                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2675          VCADDQ_ROT270_F))
2676   ]
2677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2678   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2679   [(set_attr "type" "mve_move")
2680 ])
2681
2682 ;;
2683 ;; [vcaddq_rot90_f])
2684 ;;
2685 (define_insn "mve_vcaddq_rot90_f<mode>"
2686   [
2687    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2688         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2689                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2690          VCADDQ_ROT90_F))
2691   ]
2692   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2693   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2694   [(set_attr "type" "mve_move")
2695 ])
2696
2697 ;;
2698 ;; [vcmpeqq_f])
2699 ;;
2700 (define_insn "mve_vcmpeqq_f<mode>"
2701   [
2702    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2703         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2704                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2705          VCMPEQQ_F))
2706   ]
2707   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2708   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2709   [(set_attr "type" "mve_move")
2710 ])
2711
2712 ;;
2713 ;; [vcmpeqq_n_f])
2714 ;;
2715 (define_insn "mve_vcmpeqq_n_f<mode>"
2716   [
2717    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2718         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2719                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2720          VCMPEQQ_N_F))
2721   ]
2722   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2723   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2724   [(set_attr "type" "mve_move")
2725 ])
2726
2727 ;;
2728 ;; [vcmpgeq_f])
2729 ;;
2730 (define_insn "mve_vcmpgeq_f<mode>"
2731   [
2732    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2733         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2734                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2735          VCMPGEQ_F))
2736   ]
2737   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2738   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2739   [(set_attr "type" "mve_move")
2740 ])
2741
2742 ;;
2743 ;; [vcmpgeq_n_f])
2744 ;;
2745 (define_insn "mve_vcmpgeq_n_f<mode>"
2746   [
2747    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2748         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2749                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2750          VCMPGEQ_N_F))
2751   ]
2752   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2753   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2754   [(set_attr "type" "mve_move")
2755 ])
2756
2757 ;;
2758 ;; [vcmpgtq_f])
2759 ;;
2760 (define_insn "mve_vcmpgtq_f<mode>"
2761   [
2762    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2763         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2764                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2765          VCMPGTQ_F))
2766   ]
2767   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2768   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2769   [(set_attr "type" "mve_move")
2770 ])
2771
2772 ;;
2773 ;; [vcmpgtq_n_f])
2774 ;;
2775 (define_insn "mve_vcmpgtq_n_f<mode>"
2776   [
2777    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2778         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2779                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2780          VCMPGTQ_N_F))
2781   ]
2782   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2783   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2784   [(set_attr "type" "mve_move")
2785 ])
2786
2787 ;;
2788 ;; [vcmpleq_f])
2789 ;;
2790 (define_insn "mve_vcmpleq_f<mode>"
2791   [
2792    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2793         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2794                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2795          VCMPLEQ_F))
2796   ]
2797   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2798   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2799   [(set_attr "type" "mve_move")
2800 ])
2801
2802 ;;
2803 ;; [vcmpleq_n_f])
2804 ;;
2805 (define_insn "mve_vcmpleq_n_f<mode>"
2806   [
2807    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2808         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2809                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2810          VCMPLEQ_N_F))
2811   ]
2812   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2813   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2814   [(set_attr "type" "mve_move")
2815 ])
2816
2817 ;;
2818 ;; [vcmpltq_f])
2819 ;;
2820 (define_insn "mve_vcmpltq_f<mode>"
2821   [
2822    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2823         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2824                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2825          VCMPLTQ_F))
2826   ]
2827   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2828   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2829   [(set_attr "type" "mve_move")
2830 ])
2831
2832 ;;
2833 ;; [vcmpltq_n_f])
2834 ;;
2835 (define_insn "mve_vcmpltq_n_f<mode>"
2836   [
2837    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2838         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2839                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2840          VCMPLTQ_N_F))
2841   ]
2842   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2843   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2844   [(set_attr "type" "mve_move")
2845 ])
2846
2847 ;;
2848 ;; [vcmpneq_f])
2849 ;;
2850 (define_insn "mve_vcmpneq_f<mode>"
2851   [
2852    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2853         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2854                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2855          VCMPNEQ_F))
2856   ]
2857   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2858   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2859   [(set_attr "type" "mve_move")
2860 ])
2861
2862 ;;
2863 ;; [vcmpneq_n_f])
2864 ;;
2865 (define_insn "mve_vcmpneq_n_f<mode>"
2866   [
2867    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2868         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2869                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2870          VCMPNEQ_N_F))
2871   ]
2872   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2873   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2874   [(set_attr "type" "mve_move")
2875 ])
2876
2877 ;;
2878 ;; [vcmulq_f])
2879 ;;
2880 (define_insn "mve_vcmulq_f<mode>"
2881   [
2882    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2883         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2884                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2885          VCMULQ_F))
2886   ]
2887   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2888   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2889   [(set_attr "type" "mve_move")
2890 ])
2891
2892 ;;
2893 ;; [vcmulq_rot180_f])
2894 ;;
2895 (define_insn "mve_vcmulq_rot180_f<mode>"
2896   [
2897    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2898         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2899                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2900          VCMULQ_ROT180_F))
2901   ]
2902   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2903   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2904   [(set_attr "type" "mve_move")
2905 ])
2906
2907 ;;
2908 ;; [vcmulq_rot270_f])
2909 ;;
2910 (define_insn "mve_vcmulq_rot270_f<mode>"
2911   [
2912    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2913         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2914                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2915          VCMULQ_ROT270_F))
2916   ]
2917   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2918   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2919   [(set_attr "type" "mve_move")
2920 ])
2921
2922 ;;
2923 ;; [vcmulq_rot90_f])
2924 ;;
2925 (define_insn "mve_vcmulq_rot90_f<mode>"
2926   [
2927    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2928         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2929                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2930          VCMULQ_ROT90_F))
2931   ]
2932   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2933   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2934   [(set_attr "type" "mve_move")
2935 ])
2936
2937 ;;
2938 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2939 ;;
2940 (define_insn "mve_vctp<mode1>q_mhi"
2941   [
2942    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2943         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2944                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2945          VCTPQ_M))
2946   ]
2947   "TARGET_HAVE_MVE"
2948   "vpst\;vctpt.<mode1> %1"
2949   [(set_attr "type" "mve_move")
2950    (set_attr "length""8")])
2951
2952 ;;
2953 ;; [vcvtbq_f16_f32])
2954 ;;
2955 (define_insn "mve_vcvtbq_f16_f32v8hf"
2956   [
2957    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2958         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2959                       (match_operand:V4SF 2 "s_register_operand" "w")]
2960          VCVTBQ_F16_F32))
2961   ]
2962   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2963   "vcvtb.f16.f32 %q0, %q2"
2964   [(set_attr "type" "mve_move")
2965 ])
2966
2967 ;;
2968 ;; [vcvttq_f16_f32])
2969 ;;
2970 (define_insn "mve_vcvttq_f16_f32v8hf"
2971   [
2972    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2973         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2974                       (match_operand:V4SF 2 "s_register_operand" "w")]
2975          VCVTTQ_F16_F32))
2976   ]
2977   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2978   "vcvtt.f16.f32 %q0, %q2"
2979   [(set_attr "type" "mve_move")
2980 ])
2981
2982 ;;
2983 ;; [veorq_f])
2984 ;;
2985 (define_insn "mve_veorq_f<mode>"
2986   [
2987    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2988         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2989                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2990          VEORQ_F))
2991   ]
2992   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2993   "veor %q0, %q1, %q2"
2994   [(set_attr "type" "mve_move")
2995 ])
2996
2997 ;;
2998 ;; [vmaxnmaq_f])
2999 ;;
3000 (define_insn "mve_vmaxnmaq_f<mode>"
3001   [
3002    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3003         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3004                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3005          VMAXNMAQ_F))
3006   ]
3007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3008   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
3009   [(set_attr "type" "mve_move")
3010 ])
3011
3012 ;;
3013 ;; [vmaxnmavq_f])
3014 ;;
3015 (define_insn "mve_vmaxnmavq_f<mode>"
3016   [
3017    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3018         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3019                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3020          VMAXNMAVQ_F))
3021   ]
3022   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3023   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
3024   [(set_attr "type" "mve_move")
3025 ])
3026
3027 ;;
3028 ;; [vmaxnmq_f])
3029 ;;
3030 (define_insn "mve_vmaxnmq_f<mode>"
3031   [
3032    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3033         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3034                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3035          VMAXNMQ_F))
3036   ]
3037   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3038   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3039   [(set_attr "type" "mve_move")
3040 ])
3041
3042 ;;
3043 ;; [vmaxnmvq_f])
3044 ;;
3045 (define_insn "mve_vmaxnmvq_f<mode>"
3046   [
3047    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3048         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3049                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3050          VMAXNMVQ_F))
3051   ]
3052   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3053   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
3054   [(set_attr "type" "mve_move")
3055 ])
3056
3057 ;;
3058 ;; [vminnmaq_f])
3059 ;;
3060 (define_insn "mve_vminnmaq_f<mode>"
3061   [
3062    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3063         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3064                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3065          VMINNMAQ_F))
3066   ]
3067   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3068   "vminnma.f%#<V_sz_elem>       %q0, %q2"
3069   [(set_attr "type" "mve_move")
3070 ])
3071
3072 ;;
3073 ;; [vminnmavq_f])
3074 ;;
3075 (define_insn "mve_vminnmavq_f<mode>"
3076   [
3077    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3078         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3079                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3080          VMINNMAVQ_F))
3081   ]
3082   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3083   "vminnmav.f%#<V_sz_elem>      %0, %q2"
3084   [(set_attr "type" "mve_move")
3085 ])
3086
3087 ;;
3088 ;; [vminnmq_f])
3089 ;;
3090 (define_insn "mve_vminnmq_f<mode>"
3091   [
3092    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3093         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3094                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3095          VMINNMQ_F))
3096   ]
3097   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3098   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3099   [(set_attr "type" "mve_move")
3100 ])
3101
3102 ;;
3103 ;; [vminnmvq_f])
3104 ;;
3105 (define_insn "mve_vminnmvq_f<mode>"
3106   [
3107    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3108         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3109                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3110          VMINNMVQ_F))
3111   ]
3112   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3113   "vminnmv.f%#<V_sz_elem>       %0, %q2"
3114   [(set_attr "type" "mve_move")
3115 ])
3116
3117 ;;
3118 ;; [vmlaldavq_u, vmlaldavq_s])
3119 ;;
3120 (define_insn "mve_vmlaldavq_<supf><mode>"
3121   [
3122    (set (match_operand:DI 0 "s_register_operand" "=r")
3123         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3124                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3125          VMLALDAVQ))
3126   ]
3127   "TARGET_HAVE_MVE"
3128   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3129   [(set_attr "type" "mve_move")
3130 ])
3131
3132 ;;
3133 ;; [vmlaldavxq_s])
3134 ;;
3135 (define_insn "mve_vmlaldavxq_s<mode>"
3136   [
3137    (set (match_operand:DI 0 "s_register_operand" "=r")
3138         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3139                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3140          VMLALDAVXQ_S))
3141   ]
3142   "TARGET_HAVE_MVE"
3143   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3144   [(set_attr "type" "mve_move")
3145 ])
3146
3147 ;;
3148 ;; [vmlsldavq_s])
3149 ;;
3150 (define_insn "mve_vmlsldavq_s<mode>"
3151   [
3152    (set (match_operand:DI 0 "s_register_operand" "=r")
3153         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3154                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3155          VMLSLDAVQ_S))
3156   ]
3157   "TARGET_HAVE_MVE"
3158   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3159   [(set_attr "type" "mve_move")
3160 ])
3161
3162 ;;
3163 ;; [vmlsldavxq_s])
3164 ;;
3165 (define_insn "mve_vmlsldavxq_s<mode>"
3166   [
3167    (set (match_operand:DI 0 "s_register_operand" "=r")
3168         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3169                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3170          VMLSLDAVXQ_S))
3171   ]
3172   "TARGET_HAVE_MVE"
3173   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3174   [(set_attr "type" "mve_move")
3175 ])
3176
3177 ;;
3178 ;; [vmovnbq_u, vmovnbq_s])
3179 ;;
3180 (define_insn "mve_vmovnbq_<supf><mode>"
3181   [
3182    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3183         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3184                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3185          VMOVNBQ))
3186   ]
3187   "TARGET_HAVE_MVE"
3188   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
3189   [(set_attr "type" "mve_move")
3190 ])
3191
3192 ;;
3193 ;; [vmovntq_s, vmovntq_u])
3194 ;;
3195 (define_insn "mve_vmovntq_<supf><mode>"
3196   [
3197    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3198         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3199                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3200          VMOVNTQ))
3201   ]
3202   "TARGET_HAVE_MVE"
3203   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
3204   [(set_attr "type" "mve_move")
3205 ])
3206
3207 ;;
3208 ;; [vmulq_f])
3209 ;;
3210 (define_insn "mve_vmulq_f<mode>"
3211   [
3212    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3213         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3214                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3215          VMULQ_F))
3216   ]
3217   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3218   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
3219   [(set_attr "type" "mve_move")
3220 ])
3221
3222 ;;
3223 ;; [vmulq_n_f])
3224 ;;
3225 (define_insn "mve_vmulq_n_f<mode>"
3226   [
3227    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3228         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3229                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
3230          VMULQ_N_F))
3231   ]
3232   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3233   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
3234   [(set_attr "type" "mve_move")
3235 ])
3236
3237 ;;
3238 ;; [vornq_f])
3239 ;;
3240 (define_insn "mve_vornq_f<mode>"
3241   [
3242    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3243         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3244                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3245          VORNQ_F))
3246   ]
3247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3248   "vorn %q0, %q1, %q2"
3249   [(set_attr "type" "mve_move")
3250 ])
3251
3252 ;;
3253 ;; [vorrq_f])
3254 ;;
3255 (define_insn "mve_vorrq_f<mode>"
3256   [
3257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3258         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3259                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3260          VORRQ_F))
3261   ]
3262   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3263   "vorr %q0, %q1, %q2"
3264   [(set_attr "type" "mve_move")
3265 ])
3266
3267 ;;
3268 ;; [vorrq_n_u, vorrq_n_s])
3269 ;;
3270 (define_insn "mve_vorrq_n_<supf><mode>"
3271   [
3272    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3273         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3274                        (match_operand:SI 2 "immediate_operand" "i")]
3275          VORRQ_N))
3276   ]
3277   "TARGET_HAVE_MVE"
3278   "vorr.i%#<V_sz_elem>  %q0, %2"
3279   [(set_attr "type" "mve_move")
3280 ])
3281
3282 ;;
3283 ;; [vqdmullbq_n_s])
3284 ;;
3285 (define_insn "mve_vqdmullbq_n_s<mode>"
3286   [
3287    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3288         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3289                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3290          VQDMULLBQ_N_S))
3291   ]
3292   "TARGET_HAVE_MVE"
3293   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
3294   [(set_attr "type" "mve_move")
3295 ])
3296
3297 ;;
3298 ;; [vqdmullbq_s])
3299 ;;
3300 (define_insn "mve_vqdmullbq_s<mode>"
3301   [
3302    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3303         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3304                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3305          VQDMULLBQ_S))
3306   ]
3307   "TARGET_HAVE_MVE"
3308   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
3309   [(set_attr "type" "mve_move")
3310 ])
3311
3312 ;;
3313 ;; [vqdmulltq_n_s])
3314 ;;
3315 (define_insn "mve_vqdmulltq_n_s<mode>"
3316   [
3317    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3318         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3319                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3320          VQDMULLTQ_N_S))
3321   ]
3322   "TARGET_HAVE_MVE"
3323   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
3324   [(set_attr "type" "mve_move")
3325 ])
3326
3327 ;;
3328 ;; [vqdmulltq_s])
3329 ;;
3330 (define_insn "mve_vqdmulltq_s<mode>"
3331   [
3332    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3333         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3334                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3335          VQDMULLTQ_S))
3336   ]
3337   "TARGET_HAVE_MVE"
3338   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
3339   [(set_attr "type" "mve_move")
3340 ])
3341
3342 ;;
3343 ;; [vqmovnbq_u, vqmovnbq_s])
3344 ;;
3345 (define_insn "mve_vqmovnbq_<supf><mode>"
3346   [
3347    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3348         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3349                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3350          VQMOVNBQ))
3351   ]
3352   "TARGET_HAVE_MVE"
3353   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
3354   [(set_attr "type" "mve_move")
3355 ])
3356
3357 ;;
3358 ;; [vqmovntq_u, vqmovntq_s])
3359 ;;
3360 (define_insn "mve_vqmovntq_<supf><mode>"
3361   [
3362    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3363         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3364                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3365          VQMOVNTQ))
3366   ]
3367   "TARGET_HAVE_MVE"
3368   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
3369   [(set_attr "type" "mve_move")
3370 ])
3371
3372 ;;
3373 ;; [vqmovunbq_s])
3374 ;;
3375 (define_insn "mve_vqmovunbq_s<mode>"
3376   [
3377    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3378         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3379                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3380          VQMOVUNBQ_S))
3381   ]
3382   "TARGET_HAVE_MVE"
3383   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
3384   [(set_attr "type" "mve_move")
3385 ])
3386
3387 ;;
3388 ;; [vqmovuntq_s])
3389 ;;
3390 (define_insn "mve_vqmovuntq_s<mode>"
3391   [
3392    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3393         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3394                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3395          VQMOVUNTQ_S))
3396   ]
3397   "TARGET_HAVE_MVE"
3398   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
3399   [(set_attr "type" "mve_move")
3400 ])
3401
3402 ;;
3403 ;; [vrmlaldavhxq_s])
3404 ;;
3405 (define_insn "mve_vrmlaldavhxq_sv4si"
3406   [
3407    (set (match_operand:DI 0 "s_register_operand" "=r")
3408         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3409                     (match_operand:V4SI 2 "s_register_operand" "w")]
3410          VRMLALDAVHXQ_S))
3411   ]
3412   "TARGET_HAVE_MVE"
3413   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
3414   [(set_attr "type" "mve_move")
3415 ])
3416
3417 ;;
3418 ;; [vrmlsldavhq_s])
3419 ;;
3420 (define_insn "mve_vrmlsldavhq_sv4si"
3421   [
3422    (set (match_operand:DI 0 "s_register_operand" "=r")
3423         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3424                     (match_operand:V4SI 2 "s_register_operand" "w")]
3425          VRMLSLDAVHQ_S))
3426   ]
3427   "TARGET_HAVE_MVE"
3428   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
3429   [(set_attr "type" "mve_move")
3430 ])
3431
3432 ;;
3433 ;; [vrmlsldavhxq_s])
3434 ;;
3435 (define_insn "mve_vrmlsldavhxq_sv4si"
3436   [
3437    (set (match_operand:DI 0 "s_register_operand" "=r")
3438         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3439                     (match_operand:V4SI 2 "s_register_operand" "w")]
3440          VRMLSLDAVHXQ_S))
3441   ]
3442   "TARGET_HAVE_MVE"
3443   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
3444   [(set_attr "type" "mve_move")
3445 ])
3446
3447 ;;
3448 ;; [vshllbq_n_s, vshllbq_n_u])
3449 ;;
3450 (define_insn "mve_vshllbq_n_<supf><mode>"
3451   [
3452    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3453         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3454                                   (match_operand:SI 2 "immediate_operand" "i")]
3455          VSHLLBQ_N))
3456   ]
3457   "TARGET_HAVE_MVE"
3458   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3459   [(set_attr "type" "mve_move")
3460 ])
3461
3462 ;;
3463 ;; [vshlltq_n_u, vshlltq_n_s])
3464 ;;
3465 (define_insn "mve_vshlltq_n_<supf><mode>"
3466   [
3467    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3468         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3469                                   (match_operand:SI 2 "immediate_operand" "i")]
3470          VSHLLTQ_N))
3471   ]
3472   "TARGET_HAVE_MVE"
3473   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3474   [(set_attr "type" "mve_move")
3475 ])
3476
3477 ;;
3478 ;; [vsubq_f])
3479 ;;
3480 (define_insn "mve_vsubq_f<mode>"
3481   [
3482    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3483         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3484                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3485          VSUBQ_F))
3486   ]
3487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3488   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
3489   [(set_attr "type" "mve_move")
3490 ])
3491
3492 ;;
3493 ;; [vmulltq_poly_p])
3494 ;;
3495 (define_insn "mve_vmulltq_poly_p<mode>"
3496   [
3497    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3498         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3499                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3500          VMULLTQ_POLY_P))
3501   ]
3502   "TARGET_HAVE_MVE"
3503   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
3504   [(set_attr "type" "mve_move")
3505 ])
3506
3507 ;;
3508 ;; [vmullbq_poly_p])
3509 ;;
3510 (define_insn "mve_vmullbq_poly_p<mode>"
3511   [
3512    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3513         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3514                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3515          VMULLBQ_POLY_P))
3516   ]
3517   "TARGET_HAVE_MVE"
3518   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
3519   [(set_attr "type" "mve_move")
3520 ])
3521
3522 ;;
3523 ;; [vrmlaldavhq_u vrmlaldavhq_s])
3524 ;;
3525 (define_insn "mve_vrmlaldavhq_<supf>v4si"
3526   [
3527    (set (match_operand:DI 0 "s_register_operand" "=r")
3528         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3529                     (match_operand:V4SI 2 "s_register_operand" "w")]
3530          VRMLALDAVHQ))
3531   ]
3532   "TARGET_HAVE_MVE"
3533   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
3534   [(set_attr "type" "mve_move")
3535 ])
3536
3537 ;;
3538 ;; [vbicq_m_n_s, vbicq_m_n_u])
3539 ;;
3540 (define_insn "mve_vbicq_m_n_<supf><mode>"
3541   [
3542    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3543         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3544                        (match_operand:SI 2 "immediate_operand" "i")
3545                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3546          VBICQ_M_N))
3547   ]
3548   "TARGET_HAVE_MVE"
3549   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
3550   [(set_attr "type" "mve_move")
3551    (set_attr "length""8")])
3552 ;;
3553 ;; [vcmpeqq_m_f])
3554 ;;
3555 (define_insn "mve_vcmpeqq_m_f<mode>"
3556   [
3557    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3558         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3559                     (match_operand:MVE_0 2 "s_register_operand" "w")
3560                     (match_operand:HI 3 "vpr_register_operand" "Up")]
3561          VCMPEQQ_M_F))
3562   ]
3563   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3564   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
3565   [(set_attr "type" "mve_move")
3566    (set_attr "length""8")])
3567 ;;
3568 ;; [vcvtaq_m_u, vcvtaq_m_s])
3569 ;;
3570 (define_insn "mve_vcvtaq_m_<supf><mode>"
3571   [
3572    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3573         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3574                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3575                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3576          VCVTAQ_M))
3577   ]
3578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3579   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3580   [(set_attr "type" "mve_move")
3581    (set_attr "length""8")])
3582 ;;
3583 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3584 ;;
3585 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3586   [
3587    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3588         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3589                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3590                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3591          VCVTQ_M_TO_F))
3592   ]
3593   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3594   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3595   [(set_attr "type" "mve_move")
3596    (set_attr "length""8")])
3597 ;;
3598 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3599 ;;
3600 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3601   [
3602    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3603         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3604                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3605                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3606          VQRSHRNBQ_N))
3607   ]
3608   "TARGET_HAVE_MVE"
3609   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3610   [(set_attr "type" "mve_move")
3611 ])
3612 ;;
3613 ;; [vqrshrunbq_n_s])
3614 ;;
3615 (define_insn "mve_vqrshrunbq_n_s<mode>"
3616   [
3617    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3618         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3619                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3620                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3621          VQRSHRUNBQ_N_S))
3622   ]
3623   "TARGET_HAVE_MVE"
3624   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3625   [(set_attr "type" "mve_move")
3626 ])
3627 ;;
3628 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3629 ;;
3630 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3631   [
3632    (set (match_operand:DI 0 "s_register_operand" "=r")
3633         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3634                     (match_operand:V4SI 2 "s_register_operand" "w")
3635                     (match_operand:V4SI 3 "s_register_operand" "w")]
3636          VRMLALDAVHAQ))
3637   ]
3638   "TARGET_HAVE_MVE"
3639   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3640   [(set_attr "type" "mve_move")
3641 ])
3642
3643 ;;
3644 ;; [vabavq_s, vabavq_u])
3645 ;;
3646 (define_insn "mve_vabavq_<supf><mode>"
3647   [
3648    (set (match_operand:SI 0 "s_register_operand" "=r")
3649         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3650                     (match_operand:MVE_2 2 "s_register_operand" "w")
3651                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3652          VABAVQ))
3653   ]
3654   "TARGET_HAVE_MVE"
3655   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3656   [(set_attr "type" "mve_move")
3657 ])
3658
3659 ;;
3660 ;; [vshlcq_u vshlcq_s]
3661 ;;
3662 (define_expand "mve_vshlcq_vec_<supf><mode>"
3663  [(match_operand:MVE_2 0 "s_register_operand")
3664   (match_operand:MVE_2 1 "s_register_operand")
3665   (match_operand:SI 2 "s_register_operand")
3666   (match_operand:SI 3 "mve_imm_32")
3667   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3668  "TARGET_HAVE_MVE"
3669 {
3670   rtx ignore_wb = gen_reg_rtx (SImode);
3671   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3672                                       operands[2], operands[3]));
3673   DONE;
3674 })
3675
3676 (define_expand "mve_vshlcq_carry_<supf><mode>"
3677  [(match_operand:SI 0 "s_register_operand")
3678   (match_operand:MVE_2 1 "s_register_operand")
3679   (match_operand:SI 2 "s_register_operand")
3680   (match_operand:SI 3 "mve_imm_32")
3681   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3682  "TARGET_HAVE_MVE"
3683 {
3684   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3685   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3686                                       operands[2], operands[3]));
3687   DONE;
3688 })
3689
3690 (define_insn "mve_vshlcq_<supf><mode>"
3691  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3692        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3693                       (match_operand:SI 3 "s_register_operand" "1")
3694                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3695         VSHLCQ))
3696   (set (match_operand:SI  1 "s_register_operand" "=r")
3697        (unspec:SI [(match_dup 2)
3698                    (match_dup 3)
3699                    (match_dup 4)]
3700         VSHLCQ))]
3701  "TARGET_HAVE_MVE"
3702  "vshlc %q0, %1, %4")
3703
3704 ;;
3705 ;; [vabsq_m_s])
3706 ;;
3707 (define_insn "mve_vabsq_m_s<mode>"
3708   [
3709    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3710         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3711                        (match_operand:MVE_2 2 "s_register_operand" "w")
3712                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3713          VABSQ_M_S))
3714   ]
3715   "TARGET_HAVE_MVE"
3716   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3717   [(set_attr "type" "mve_move")
3718    (set_attr "length""8")])
3719
3720 ;;
3721 ;; [vaddvaq_p_u, vaddvaq_p_s])
3722 ;;
3723 (define_insn "mve_vaddvaq_p_<supf><mode>"
3724   [
3725    (set (match_operand:SI 0 "s_register_operand" "=Te")
3726         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3727                        (match_operand:MVE_2 2 "s_register_operand" "w")
3728                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3729          VADDVAQ_P))
3730   ]
3731   "TARGET_HAVE_MVE"
3732   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3733   [(set_attr "type" "mve_move")
3734    (set_attr "length""8")])
3735
3736 ;;
3737 ;; [vclsq_m_s])
3738 ;;
3739 (define_insn "mve_vclsq_m_s<mode>"
3740   [
3741    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3742         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3743                        (match_operand:MVE_2 2 "s_register_operand" "w")
3744                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3745          VCLSQ_M_S))
3746   ]
3747   "TARGET_HAVE_MVE"
3748   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3749   [(set_attr "type" "mve_move")
3750    (set_attr "length""8")])
3751
3752 ;;
3753 ;; [vclzq_m_s, vclzq_m_u])
3754 ;;
3755 (define_insn "mve_vclzq_m_<supf><mode>"
3756   [
3757    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3758         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3759                        (match_operand:MVE_2 2 "s_register_operand" "w")
3760                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3761          VCLZQ_M))
3762   ]
3763   "TARGET_HAVE_MVE"
3764   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3765   [(set_attr "type" "mve_move")
3766    (set_attr "length""8")])
3767
3768 ;;
3769 ;; [vcmpcsq_m_n_u])
3770 ;;
3771 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3772   [
3773    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3774         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3775                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3776                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3777          VCMPCSQ_M_N_U))
3778   ]
3779   "TARGET_HAVE_MVE"
3780   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3781   [(set_attr "type" "mve_move")
3782    (set_attr "length""8")])
3783
3784 ;;
3785 ;; [vcmpcsq_m_u])
3786 ;;
3787 (define_insn "mve_vcmpcsq_m_u<mode>"
3788   [
3789    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3790         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3791                        (match_operand:MVE_2 2 "s_register_operand" "w")
3792                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3793          VCMPCSQ_M_U))
3794   ]
3795   "TARGET_HAVE_MVE"
3796   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3797   [(set_attr "type" "mve_move")
3798    (set_attr "length""8")])
3799
3800 ;;
3801 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3802 ;;
3803 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3804   [
3805    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3806         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3807                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3808                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3809          VCMPEQQ_M_N))
3810   ]
3811   "TARGET_HAVE_MVE"
3812   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3813   [(set_attr "type" "mve_move")
3814    (set_attr "length""8")])
3815
3816 ;;
3817 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3818 ;;
3819 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3820   [
3821    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3822         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3823                        (match_operand:MVE_2 2 "s_register_operand" "w")
3824                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3825          VCMPEQQ_M))
3826   ]
3827   "TARGET_HAVE_MVE"
3828   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3829   [(set_attr "type" "mve_move")
3830    (set_attr "length""8")])
3831
3832 ;;
3833 ;; [vcmpgeq_m_n_s])
3834 ;;
3835 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3836   [
3837    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3838         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3839                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3840                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3841          VCMPGEQ_M_N_S))
3842   ]
3843   "TARGET_HAVE_MVE"
3844   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3845   [(set_attr "type" "mve_move")
3846    (set_attr "length""8")])
3847
3848 ;;
3849 ;; [vcmpgeq_m_s])
3850 ;;
3851 (define_insn "mve_vcmpgeq_m_s<mode>"
3852   [
3853    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3854         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3855                        (match_operand:MVE_2 2 "s_register_operand" "w")
3856                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3857          VCMPGEQ_M_S))
3858   ]
3859   "TARGET_HAVE_MVE"
3860   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3861   [(set_attr "type" "mve_move")
3862    (set_attr "length""8")])
3863
3864 ;;
3865 ;; [vcmpgtq_m_n_s])
3866 ;;
3867 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3868   [
3869    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3870         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3871                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3872                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3873          VCMPGTQ_M_N_S))
3874   ]
3875   "TARGET_HAVE_MVE"
3876   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3877   [(set_attr "type" "mve_move")
3878    (set_attr "length""8")])
3879
3880 ;;
3881 ;; [vcmpgtq_m_s])
3882 ;;
3883 (define_insn "mve_vcmpgtq_m_s<mode>"
3884   [
3885    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3886         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3887                        (match_operand:MVE_2 2 "s_register_operand" "w")
3888                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3889          VCMPGTQ_M_S))
3890   ]
3891   "TARGET_HAVE_MVE"
3892   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3893   [(set_attr "type" "mve_move")
3894    (set_attr "length""8")])
3895
3896 ;;
3897 ;; [vcmphiq_m_n_u])
3898 ;;
3899 (define_insn "mve_vcmphiq_m_n_u<mode>"
3900   [
3901    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3902         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3903                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3904                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3905          VCMPHIQ_M_N_U))
3906   ]
3907   "TARGET_HAVE_MVE"
3908   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3909   [(set_attr "type" "mve_move")
3910    (set_attr "length""8")])
3911
3912 ;;
3913 ;; [vcmphiq_m_u])
3914 ;;
3915 (define_insn "mve_vcmphiq_m_u<mode>"
3916   [
3917    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3918         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3919                        (match_operand:MVE_2 2 "s_register_operand" "w")
3920                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3921          VCMPHIQ_M_U))
3922   ]
3923   "TARGET_HAVE_MVE"
3924   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3925   [(set_attr "type" "mve_move")
3926    (set_attr "length""8")])
3927
3928 ;;
3929 ;; [vcmpleq_m_n_s])
3930 ;;
3931 (define_insn "mve_vcmpleq_m_n_s<mode>"
3932   [
3933    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3934         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3935                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3936                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3937          VCMPLEQ_M_N_S))
3938   ]
3939   "TARGET_HAVE_MVE"
3940   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3941   [(set_attr "type" "mve_move")
3942    (set_attr "length""8")])
3943
3944 ;;
3945 ;; [vcmpleq_m_s])
3946 ;;
3947 (define_insn "mve_vcmpleq_m_s<mode>"
3948   [
3949    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3950         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3951                        (match_operand:MVE_2 2 "s_register_operand" "w")
3952                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3953          VCMPLEQ_M_S))
3954   ]
3955   "TARGET_HAVE_MVE"
3956   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3957   [(set_attr "type" "mve_move")
3958    (set_attr "length""8")])
3959
3960 ;;
3961 ;; [vcmpltq_m_n_s])
3962 ;;
3963 (define_insn "mve_vcmpltq_m_n_s<mode>"
3964   [
3965    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3966         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3967                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3968                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3969          VCMPLTQ_M_N_S))
3970   ]
3971   "TARGET_HAVE_MVE"
3972   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3973   [(set_attr "type" "mve_move")
3974    (set_attr "length""8")])
3975
3976 ;;
3977 ;; [vcmpltq_m_s])
3978 ;;
3979 (define_insn "mve_vcmpltq_m_s<mode>"
3980   [
3981    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3982         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3983                        (match_operand:MVE_2 2 "s_register_operand" "w")
3984                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3985          VCMPLTQ_M_S))
3986   ]
3987   "TARGET_HAVE_MVE"
3988   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3989   [(set_attr "type" "mve_move")
3990    (set_attr "length""8")])
3991
3992 ;;
3993 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3994 ;;
3995 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3996   [
3997    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3998         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3999                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4000                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4001          VCMPNEQ_M_N))
4002   ]
4003   "TARGET_HAVE_MVE"
4004   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
4005   [(set_attr "type" "mve_move")
4006    (set_attr "length""8")])
4007
4008 ;;
4009 ;; [vcmpneq_m_s, vcmpneq_m_u])
4010 ;;
4011 (define_insn "mve_vcmpneq_m_<supf><mode>"
4012   [
4013    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4014         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
4015                        (match_operand:MVE_2 2 "s_register_operand" "w")
4016                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4017          VCMPNEQ_M))
4018   ]
4019   "TARGET_HAVE_MVE"
4020   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
4021   [(set_attr "type" "mve_move")
4022    (set_attr "length""8")])
4023
4024 ;;
4025 ;; [vdupq_m_n_s, vdupq_m_n_u])
4026 ;;
4027 (define_insn "mve_vdupq_m_n_<supf><mode>"
4028   [
4029    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4030         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4031                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4032                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4033          VDUPQ_M_N))
4034   ]
4035   "TARGET_HAVE_MVE"
4036   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4037   [(set_attr "type" "mve_move")
4038    (set_attr "length""8")])
4039
4040 ;;
4041 ;; [vmaxaq_m_s])
4042 ;;
4043 (define_insn "mve_vmaxaq_m_s<mode>"
4044   [
4045    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4046         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4047                        (match_operand:MVE_2 2 "s_register_operand" "w")
4048                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4049          VMAXAQ_M_S))
4050   ]
4051   "TARGET_HAVE_MVE"
4052   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
4053   [(set_attr "type" "mve_move")
4054    (set_attr "length""8")])
4055
4056 ;;
4057 ;; [vmaxavq_p_s])
4058 ;;
4059 (define_insn "mve_vmaxavq_p_s<mode>"
4060   [
4061    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4062         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4063                        (match_operand:MVE_2 2 "s_register_operand" "w")
4064                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4065          VMAXAVQ_P_S))
4066   ]
4067   "TARGET_HAVE_MVE"
4068   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
4069   [(set_attr "type" "mve_move")
4070    (set_attr "length""8")])
4071
4072 ;;
4073 ;; [vmaxvq_p_u, vmaxvq_p_s])
4074 ;;
4075 (define_insn "mve_vmaxvq_p_<supf><mode>"
4076   [
4077    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4078         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4079                        (match_operand:MVE_2 2 "s_register_operand" "w")
4080                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4081          VMAXVQ_P))
4082   ]
4083   "TARGET_HAVE_MVE"
4084   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
4085   [(set_attr "type" "mve_move")
4086    (set_attr "length""8")])
4087
4088 ;;
4089 ;; [vminaq_m_s])
4090 ;;
4091 (define_insn "mve_vminaq_m_s<mode>"
4092   [
4093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4095                        (match_operand:MVE_2 2 "s_register_operand" "w")
4096                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4097          VMINAQ_M_S))
4098   ]
4099   "TARGET_HAVE_MVE"
4100   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
4101   [(set_attr "type" "mve_move")
4102    (set_attr "length""8")])
4103
4104 ;;
4105 ;; [vminavq_p_s])
4106 ;;
4107 (define_insn "mve_vminavq_p_s<mode>"
4108   [
4109    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4110         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4111                        (match_operand:MVE_2 2 "s_register_operand" "w")
4112                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4113          VMINAVQ_P_S))
4114   ]
4115   "TARGET_HAVE_MVE"
4116   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
4117   [(set_attr "type" "mve_move")
4118    (set_attr "length""8")])
4119
4120 ;;
4121 ;; [vminvq_p_s, vminvq_p_u])
4122 ;;
4123 (define_insn "mve_vminvq_p_<supf><mode>"
4124   [
4125    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4126         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4127                        (match_operand:MVE_2 2 "s_register_operand" "w")
4128                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4129          VMINVQ_P))
4130   ]
4131   "TARGET_HAVE_MVE"
4132   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
4133   [(set_attr "type" "mve_move")
4134    (set_attr "length""8")])
4135
4136 ;;
4137 ;; [vmladavaq_u, vmladavaq_s])
4138 ;;
4139 (define_insn "mve_vmladavaq_<supf><mode>"
4140   [
4141    (set (match_operand:SI 0 "s_register_operand" "=Te")
4142         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4143                        (match_operand:MVE_2 2 "s_register_operand" "w")
4144                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4145          VMLADAVAQ))
4146   ]
4147   "TARGET_HAVE_MVE"
4148   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
4149   [(set_attr "type" "mve_move")
4150 ])
4151
4152 ;;
4153 ;; [vmladavq_p_u, vmladavq_p_s])
4154 ;;
4155 (define_insn "mve_vmladavq_p_<supf><mode>"
4156   [
4157    (set (match_operand:SI 0 "s_register_operand" "=Te")
4158         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4159                        (match_operand:MVE_2 2 "s_register_operand" "w")
4160                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4161          VMLADAVQ_P))
4162   ]
4163   "TARGET_HAVE_MVE"
4164   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
4165   [(set_attr "type" "mve_move")
4166    (set_attr "length""8")])
4167
4168 ;;
4169 ;; [vmladavxq_p_s])
4170 ;;
4171 (define_insn "mve_vmladavxq_p_s<mode>"
4172   [
4173    (set (match_operand:SI 0 "s_register_operand" "=Te")
4174         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4175                        (match_operand:MVE_2 2 "s_register_operand" "w")
4176                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4177          VMLADAVXQ_P_S))
4178   ]
4179   "TARGET_HAVE_MVE"
4180   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
4181   [(set_attr "type" "mve_move")
4182    (set_attr "length""8")])
4183
4184 ;;
4185 ;; [vmlaq_n_u, vmlaq_n_s])
4186 ;;
4187 (define_insn "mve_vmlaq_n_<supf><mode>"
4188   [
4189    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4190         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4191                        (match_operand:MVE_2 2 "s_register_operand" "w")
4192                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4193          VMLAQ_N))
4194   ]
4195   "TARGET_HAVE_MVE"
4196   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4197   [(set_attr "type" "mve_move")
4198 ])
4199
4200 ;;
4201 ;; [vmlasq_n_u, vmlasq_n_s])
4202 ;;
4203 (define_insn "mve_vmlasq_n_<supf><mode>"
4204   [
4205    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4206         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4207                        (match_operand:MVE_2 2 "s_register_operand" "w")
4208                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4209          VMLASQ_N))
4210   ]
4211   "TARGET_HAVE_MVE"
4212   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
4213   [(set_attr "type" "mve_move")
4214 ])
4215
4216 ;;
4217 ;; [vmlsdavq_p_s])
4218 ;;
4219 (define_insn "mve_vmlsdavq_p_s<mode>"
4220   [
4221    (set (match_operand:SI 0 "s_register_operand" "=Te")
4222         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4223                        (match_operand:MVE_2 2 "s_register_operand" "w")
4224                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4225          VMLSDAVQ_P_S))
4226   ]
4227   "TARGET_HAVE_MVE"
4228   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
4229   [(set_attr "type" "mve_move")
4230    (set_attr "length""8")])
4231
4232 ;;
4233 ;; [vmlsdavxq_p_s])
4234 ;;
4235 (define_insn "mve_vmlsdavxq_p_s<mode>"
4236   [
4237    (set (match_operand:SI 0 "s_register_operand" "=Te")
4238         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4239                        (match_operand:MVE_2 2 "s_register_operand" "w")
4240                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4241          VMLSDAVXQ_P_S))
4242   ]
4243   "TARGET_HAVE_MVE"
4244   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
4245   [(set_attr "type" "mve_move")
4246    (set_attr "length""8")])
4247
4248 ;;
4249 ;; [vmvnq_m_s, vmvnq_m_u])
4250 ;;
4251 (define_insn "mve_vmvnq_m_<supf><mode>"
4252   [
4253    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4254         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4255                        (match_operand:MVE_2 2 "s_register_operand" "w")
4256                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4257          VMVNQ_M))
4258   ]
4259   "TARGET_HAVE_MVE"
4260   "vpst\;vmvnt %q0, %q2"
4261   [(set_attr "type" "mve_move")
4262    (set_attr "length""8")])
4263
4264 ;;
4265 ;; [vnegq_m_s])
4266 ;;
4267 (define_insn "mve_vnegq_m_s<mode>"
4268   [
4269    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4270         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4271                        (match_operand:MVE_2 2 "s_register_operand" "w")
4272                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4273          VNEGQ_M_S))
4274   ]
4275   "TARGET_HAVE_MVE"
4276   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
4277   [(set_attr "type" "mve_move")
4278    (set_attr "length""8")])
4279
4280 ;;
4281 ;; [vpselq_u, vpselq_s])
4282 ;;
4283 (define_insn "mve_vpselq_<supf><mode>"
4284   [
4285    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
4286         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
4287                        (match_operand:MVE_1 2 "s_register_operand" "w")
4288                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4289          VPSELQ))
4290   ]
4291   "TARGET_HAVE_MVE"
4292   "vpsel %q0, %q1, %q2"
4293   [(set_attr "type" "mve_move")
4294 ])
4295
4296 ;;
4297 ;; [vqabsq_m_s])
4298 ;;
4299 (define_insn "mve_vqabsq_m_s<mode>"
4300   [
4301    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4302         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4303                        (match_operand:MVE_2 2 "s_register_operand" "w")
4304                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4305          VQABSQ_M_S))
4306   ]
4307   "TARGET_HAVE_MVE"
4308   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
4309   [(set_attr "type" "mve_move")
4310    (set_attr "length""8")])
4311
4312 ;;
4313 ;; [vqdmlahq_n_s, vqdmlahq_n_u])
4314 ;;
4315 (define_insn "mve_vqdmlahq_n_<supf><mode>"
4316   [
4317    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4318         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4319                        (match_operand:MVE_2 2 "s_register_operand" "w")
4320                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4321          VQDMLAHQ_N))
4322   ]
4323   "TARGET_HAVE_MVE"
4324   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4325   [(set_attr "type" "mve_move")
4326 ])
4327
4328 ;;
4329 ;; [vqnegq_m_s])
4330 ;;
4331 (define_insn "mve_vqnegq_m_s<mode>"
4332   [
4333    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4334         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4335                        (match_operand:MVE_2 2 "s_register_operand" "w")
4336                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4337          VQNEGQ_M_S))
4338   ]
4339   "TARGET_HAVE_MVE"
4340   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
4341   [(set_attr "type" "mve_move")
4342    (set_attr "length""8")])
4343
4344 ;;
4345 ;; [vqrdmladhq_s])
4346 ;;
4347 (define_insn "mve_vqrdmladhq_s<mode>"
4348   [
4349    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4350         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4351                        (match_operand:MVE_2 2 "s_register_operand" "w")
4352                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4353          VQRDMLADHQ_S))
4354   ]
4355   "TARGET_HAVE_MVE"
4356   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4357   [(set_attr "type" "mve_move")
4358 ])
4359
4360 ;;
4361 ;; [vqrdmladhxq_s])
4362 ;;
4363 (define_insn "mve_vqrdmladhxq_s<mode>"
4364   [
4365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4367                        (match_operand:MVE_2 2 "s_register_operand" "w")
4368                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4369          VQRDMLADHXQ_S))
4370   ]
4371   "TARGET_HAVE_MVE"
4372   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4373   [(set_attr "type" "mve_move")
4374 ])
4375
4376 ;;
4377 ;; [vqrdmlahq_n_s, vqrdmlahq_n_u])
4378 ;;
4379 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
4380   [
4381    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4382         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4383                        (match_operand:MVE_2 2 "s_register_operand" "w")
4384                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4385          VQRDMLAHQ_N))
4386   ]
4387   "TARGET_HAVE_MVE"
4388   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4389   [(set_attr "type" "mve_move")
4390 ])
4391
4392 ;;
4393 ;; [vqrdmlashq_n_s, vqrdmlashq_n_u])
4394 ;;
4395 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
4396   [
4397    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4398         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4399                        (match_operand:MVE_2 2 "s_register_operand" "w")
4400                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4401          VQRDMLASHQ_N))
4402   ]
4403   "TARGET_HAVE_MVE"
4404   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
4405   [(set_attr "type" "mve_move")
4406 ])
4407
4408 ;;
4409 ;; [vqrdmlsdhq_s])
4410 ;;
4411 (define_insn "mve_vqrdmlsdhq_s<mode>"
4412   [
4413    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4414         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4415                        (match_operand:MVE_2 2 "s_register_operand" "w")
4416                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4417          VQRDMLSDHQ_S))
4418   ]
4419   "TARGET_HAVE_MVE"
4420   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4421   [(set_attr "type" "mve_move")
4422 ])
4423
4424 ;;
4425 ;; [vqrdmlsdhxq_s])
4426 ;;
4427 (define_insn "mve_vqrdmlsdhxq_s<mode>"
4428   [
4429    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4430         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4431                        (match_operand:MVE_2 2 "s_register_operand" "w")
4432                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4433          VQRDMLSDHXQ_S))
4434   ]
4435   "TARGET_HAVE_MVE"
4436   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4437   [(set_attr "type" "mve_move")
4438 ])
4439
4440 ;;
4441 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
4442 ;;
4443 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
4444   [
4445    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4446         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4447                        (match_operand:SI 2 "s_register_operand" "r")
4448                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4449          VQRSHLQ_M_N))
4450   ]
4451   "TARGET_HAVE_MVE"
4452   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
4453   [(set_attr "type" "mve_move")
4454    (set_attr "length""8")])
4455
4456 ;;
4457 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
4458 ;;
4459 (define_insn "mve_vqshlq_m_r_<supf><mode>"
4460   [
4461    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4462         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4463                        (match_operand:SI 2 "s_register_operand" "r")
4464                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4465          VQSHLQ_M_R))
4466   ]
4467   "TARGET_HAVE_MVE"
4468   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4469   [(set_attr "type" "mve_move")
4470    (set_attr "length""8")])
4471
4472 ;;
4473 ;; [vrev64q_m_u, vrev64q_m_s])
4474 ;;
4475 (define_insn "mve_vrev64q_m_<supf><mode>"
4476   [
4477    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4478         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4479                        (match_operand:MVE_2 2 "s_register_operand" "w")
4480                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4481          VREV64Q_M))
4482   ]
4483   "TARGET_HAVE_MVE"
4484   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
4485   [(set_attr "type" "mve_move")
4486    (set_attr "length""8")])
4487
4488 ;;
4489 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
4490 ;;
4491 (define_insn "mve_vrshlq_m_n_<supf><mode>"
4492   [
4493    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4494         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4495                        (match_operand:SI 2 "s_register_operand" "r")
4496                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4497          VRSHLQ_M_N))
4498   ]
4499   "TARGET_HAVE_MVE"
4500   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4501   [(set_attr "type" "mve_move")
4502    (set_attr "length""8")])
4503
4504 ;;
4505 ;; [vshlq_m_r_u, vshlq_m_r_s])
4506 ;;
4507 (define_insn "mve_vshlq_m_r_<supf><mode>"
4508   [
4509    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4510         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4511                        (match_operand:SI 2 "s_register_operand" "r")
4512                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4513          VSHLQ_M_R))
4514   ]
4515   "TARGET_HAVE_MVE"
4516   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4517   [(set_attr "type" "mve_move")
4518    (set_attr "length""8")])
4519
4520 ;;
4521 ;; [vsliq_n_u, vsliq_n_s])
4522 ;;
4523 (define_insn "mve_vsliq_n_<supf><mode>"
4524   [
4525    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4526         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4527                        (match_operand:MVE_2 2 "s_register_operand" "w")
4528                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
4529          VSLIQ_N))
4530   ]
4531   "TARGET_HAVE_MVE"
4532   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
4533   [(set_attr "type" "mve_move")
4534 ])
4535
4536 ;;
4537 ;; [vsriq_n_u, vsriq_n_s])
4538 ;;
4539 (define_insn "mve_vsriq_n_<supf><mode>"
4540   [
4541    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4542         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4543                        (match_operand:MVE_2 2 "s_register_operand" "w")
4544                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
4545          VSRIQ_N))
4546   ]
4547   "TARGET_HAVE_MVE"
4548   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
4549   [(set_attr "type" "mve_move")
4550 ])
4551
4552 ;;
4553 ;; [vqdmlsdhxq_s])
4554 ;;
4555 (define_insn "mve_vqdmlsdhxq_s<mode>"
4556   [
4557    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4558         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4559                        (match_operand:MVE_2 2 "s_register_operand" "w")
4560                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4561          VQDMLSDHXQ_S))
4562   ]
4563   "TARGET_HAVE_MVE"
4564   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4565   [(set_attr "type" "mve_move")
4566 ])
4567
4568 ;;
4569 ;; [vqdmlsdhq_s])
4570 ;;
4571 (define_insn "mve_vqdmlsdhq_s<mode>"
4572   [
4573    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4574         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4575                        (match_operand:MVE_2 2 "s_register_operand" "w")
4576                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4577          VQDMLSDHQ_S))
4578   ]
4579   "TARGET_HAVE_MVE"
4580   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4581   [(set_attr "type" "mve_move")
4582 ])
4583
4584 ;;
4585 ;; [vqdmladhxq_s])
4586 ;;
4587 (define_insn "mve_vqdmladhxq_s<mode>"
4588   [
4589    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4590         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4591                        (match_operand:MVE_2 2 "s_register_operand" "w")
4592                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4593          VQDMLADHXQ_S))
4594   ]
4595   "TARGET_HAVE_MVE"
4596   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4597   [(set_attr "type" "mve_move")
4598 ])
4599
4600 ;;
4601 ;; [vqdmladhq_s])
4602 ;;
4603 (define_insn "mve_vqdmladhq_s<mode>"
4604   [
4605    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4606         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4607                        (match_operand:MVE_2 2 "s_register_operand" "w")
4608                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4609          VQDMLADHQ_S))
4610   ]
4611   "TARGET_HAVE_MVE"
4612   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4613   [(set_attr "type" "mve_move")
4614 ])
4615
4616 ;;
4617 ;; [vmlsdavaxq_s])
4618 ;;
4619 (define_insn "mve_vmlsdavaxq_s<mode>"
4620   [
4621    (set (match_operand:SI 0 "s_register_operand" "=Te")
4622         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4623                     (match_operand:MVE_2 2 "s_register_operand" "w")
4624                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4625          VMLSDAVAXQ_S))
4626   ]
4627   "TARGET_HAVE_MVE"
4628   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4629   [(set_attr "type" "mve_move")
4630 ])
4631
4632 ;;
4633 ;; [vmlsdavaq_s])
4634 ;;
4635 (define_insn "mve_vmlsdavaq_s<mode>"
4636   [
4637    (set (match_operand:SI 0 "s_register_operand" "=Te")
4638         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4639                     (match_operand:MVE_2 2 "s_register_operand" "w")
4640                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4641          VMLSDAVAQ_S))
4642   ]
4643   "TARGET_HAVE_MVE"
4644   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4645   [(set_attr "type" "mve_move")
4646 ])
4647
4648 ;;
4649 ;; [vmladavaxq_s])
4650 ;;
4651 (define_insn "mve_vmladavaxq_s<mode>"
4652   [
4653    (set (match_operand:SI 0 "s_register_operand" "=Te")
4654         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4655                     (match_operand:MVE_2 2 "s_register_operand" "w")
4656                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4657          VMLADAVAXQ_S))
4658   ]
4659   "TARGET_HAVE_MVE"
4660   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4661   [(set_attr "type" "mve_move")
4662 ])
4663 ;;
4664 ;; [vabsq_m_f])
4665 ;;
4666 (define_insn "mve_vabsq_m_f<mode>"
4667   [
4668    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4669         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4670                        (match_operand:MVE_0 2 "s_register_operand" "w")
4671                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4672          VABSQ_M_F))
4673   ]
4674   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4675   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4676   [(set_attr "type" "mve_move")
4677    (set_attr "length""8")])
4678
4679 ;;
4680 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4681 ;;
4682 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4683   [
4684    (set (match_operand:DI 0 "s_register_operand" "=r")
4685         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4686                        (match_operand:V4SI 2 "s_register_operand" "w")
4687                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4688          VADDLVAQ_P))
4689   ]
4690   "TARGET_HAVE_MVE"
4691   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4692   [(set_attr "type" "mve_move")
4693    (set_attr "length""8")])
4694 ;;
4695 ;; [vcmlaq_f])
4696 ;;
4697 (define_insn "mve_vcmlaq_f<mode>"
4698   [
4699    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4700         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4701                        (match_operand:MVE_0 2 "s_register_operand" "w")
4702                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4703          VCMLAQ_F))
4704   ]
4705   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4706   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4707   [(set_attr "type" "mve_move")
4708 ])
4709
4710 ;;
4711 ;; [vcmlaq_rot180_f])
4712 ;;
4713 (define_insn "mve_vcmlaq_rot180_f<mode>"
4714   [
4715    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4716         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4717                        (match_operand:MVE_0 2 "s_register_operand" "w")
4718                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4719          VCMLAQ_ROT180_F))
4720   ]
4721   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4722   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4723   [(set_attr "type" "mve_move")
4724 ])
4725
4726 ;;
4727 ;; [vcmlaq_rot270_f])
4728 ;;
4729 (define_insn "mve_vcmlaq_rot270_f<mode>"
4730   [
4731    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4732         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4733                        (match_operand:MVE_0 2 "s_register_operand" "w")
4734                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4735          VCMLAQ_ROT270_F))
4736   ]
4737   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4738   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4739   [(set_attr "type" "mve_move")
4740 ])
4741
4742 ;;
4743 ;; [vcmlaq_rot90_f])
4744 ;;
4745 (define_insn "mve_vcmlaq_rot90_f<mode>"
4746   [
4747    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4748         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4749                        (match_operand:MVE_0 2 "s_register_operand" "w")
4750                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4751          VCMLAQ_ROT90_F))
4752   ]
4753   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4754   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4755   [(set_attr "type" "mve_move")
4756 ])
4757
4758 ;;
4759 ;; [vcmpeqq_m_n_f])
4760 ;;
4761 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4762   [
4763    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4764         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4765                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4766                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4767          VCMPEQQ_M_N_F))
4768   ]
4769   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4770   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4771   [(set_attr "type" "mve_move")
4772    (set_attr "length""8")])
4773
4774 ;;
4775 ;; [vcmpgeq_m_f])
4776 ;;
4777 (define_insn "mve_vcmpgeq_m_f<mode>"
4778   [
4779    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4780         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4781                        (match_operand:MVE_0 2 "s_register_operand" "w")
4782                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4783          VCMPGEQ_M_F))
4784   ]
4785   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4786   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4787   [(set_attr "type" "mve_move")
4788    (set_attr "length""8")])
4789
4790 ;;
4791 ;; [vcmpgeq_m_n_f])
4792 ;;
4793 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4794   [
4795    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4796         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4797                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4798                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4799          VCMPGEQ_M_N_F))
4800   ]
4801   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4802   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4803   [(set_attr "type" "mve_move")
4804    (set_attr "length""8")])
4805
4806 ;;
4807 ;; [vcmpgtq_m_f])
4808 ;;
4809 (define_insn "mve_vcmpgtq_m_f<mode>"
4810   [
4811    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4812         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4813                        (match_operand:MVE_0 2 "s_register_operand" "w")
4814                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4815          VCMPGTQ_M_F))
4816   ]
4817   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4818   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4819   [(set_attr "type" "mve_move")
4820    (set_attr "length""8")])
4821
4822 ;;
4823 ;; [vcmpgtq_m_n_f])
4824 ;;
4825 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4826   [
4827    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4828         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4829                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4830                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4831          VCMPGTQ_M_N_F))
4832   ]
4833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4834   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4835   [(set_attr "type" "mve_move")
4836    (set_attr "length""8")])
4837
4838 ;;
4839 ;; [vcmpleq_m_f])
4840 ;;
4841 (define_insn "mve_vcmpleq_m_f<mode>"
4842   [
4843    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4844         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4845                        (match_operand:MVE_0 2 "s_register_operand" "w")
4846                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4847          VCMPLEQ_M_F))
4848   ]
4849   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4850   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4851   [(set_attr "type" "mve_move")
4852    (set_attr "length""8")])
4853
4854 ;;
4855 ;; [vcmpleq_m_n_f])
4856 ;;
4857 (define_insn "mve_vcmpleq_m_n_f<mode>"
4858   [
4859    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4860         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4861                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4862                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4863          VCMPLEQ_M_N_F))
4864   ]
4865   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4866   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4867   [(set_attr "type" "mve_move")
4868    (set_attr "length""8")])
4869
4870 ;;
4871 ;; [vcmpltq_m_f])
4872 ;;
4873 (define_insn "mve_vcmpltq_m_f<mode>"
4874   [
4875    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4876         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4877                        (match_operand:MVE_0 2 "s_register_operand" "w")
4878                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4879          VCMPLTQ_M_F))
4880   ]
4881   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4882   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4883   [(set_attr "type" "mve_move")
4884    (set_attr "length""8")])
4885
4886 ;;
4887 ;; [vcmpltq_m_n_f])
4888 ;;
4889 (define_insn "mve_vcmpltq_m_n_f<mode>"
4890   [
4891    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4892         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4893                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4894                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4895          VCMPLTQ_M_N_F))
4896   ]
4897   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4898   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4899   [(set_attr "type" "mve_move")
4900    (set_attr "length""8")])
4901
4902 ;;
4903 ;; [vcmpneq_m_f])
4904 ;;
4905 (define_insn "mve_vcmpneq_m_f<mode>"
4906   [
4907    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4908         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4909                        (match_operand:MVE_0 2 "s_register_operand" "w")
4910                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4911          VCMPNEQ_M_F))
4912   ]
4913   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4914   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4915   [(set_attr "type" "mve_move")
4916    (set_attr "length""8")])
4917
4918 ;;
4919 ;; [vcmpneq_m_n_f])
4920 ;;
4921 (define_insn "mve_vcmpneq_m_n_f<mode>"
4922   [
4923    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4924         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4925                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4926                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4927          VCMPNEQ_M_N_F))
4928   ]
4929   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4930   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4931   [(set_attr "type" "mve_move")
4932    (set_attr "length""8")])
4933
4934 ;;
4935 ;; [vcvtbq_m_f16_f32])
4936 ;;
4937 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4938   [
4939    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4940         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4941                        (match_operand:V4SF 2 "s_register_operand" "w")
4942                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4943          VCVTBQ_M_F16_F32))
4944   ]
4945   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4946   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4947   [(set_attr "type" "mve_move")
4948    (set_attr "length""8")])
4949
4950 ;;
4951 ;; [vcvtbq_m_f32_f16])
4952 ;;
4953 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4954   [
4955    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4956         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4957                        (match_operand:V8HF 2 "s_register_operand" "w")
4958                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4959          VCVTBQ_M_F32_F16))
4960   ]
4961   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4962   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4963   [(set_attr "type" "mve_move")
4964    (set_attr "length""8")])
4965
4966 ;;
4967 ;; [vcvttq_m_f16_f32])
4968 ;;
4969 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4970   [
4971    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4972         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4973                        (match_operand:V4SF 2 "s_register_operand" "w")
4974                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4975          VCVTTQ_M_F16_F32))
4976   ]
4977   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4978   "vpst\;vcvttt.f16.f32 %q0, %q2"
4979   [(set_attr "type" "mve_move")
4980    (set_attr "length""8")])
4981
4982 ;;
4983 ;; [vcvttq_m_f32_f16])
4984 ;;
4985 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4986   [
4987    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4988         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4989                        (match_operand:V8HF 2 "s_register_operand" "w")
4990                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4991          VCVTTQ_M_F32_F16))
4992   ]
4993   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4994   "vpst\;vcvttt.f32.f16 %q0, %q2"
4995   [(set_attr "type" "mve_move")
4996    (set_attr "length""8")])
4997
4998 ;;
4999 ;; [vdupq_m_n_f])
5000 ;;
5001 (define_insn "mve_vdupq_m_n_f<mode>"
5002   [
5003    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5004         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5005                        (match_operand:<V_elem> 2 "s_register_operand" "r")
5006                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5007          VDUPQ_M_N_F))
5008   ]
5009   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5010   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
5011   [(set_attr "type" "mve_move")
5012    (set_attr "length""8")])
5013
5014 ;;
5015 ;; [vfmaq_f])
5016 ;;
5017 (define_insn "mve_vfmaq_f<mode>"
5018   [
5019    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5020         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5021                        (match_operand:MVE_0 2 "s_register_operand" "w")
5022                        (match_operand:MVE_0 3 "s_register_operand" "w")]
5023          VFMAQ_F))
5024   ]
5025   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5026   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
5027   [(set_attr "type" "mve_move")
5028 ])
5029
5030 ;;
5031 ;; [vfmaq_n_f])
5032 ;;
5033 (define_insn "mve_vfmaq_n_f<mode>"
5034   [
5035    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5036         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5037                        (match_operand:MVE_0 2 "s_register_operand" "w")
5038                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5039          VFMAQ_N_F))
5040   ]
5041   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5042   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
5043   [(set_attr "type" "mve_move")
5044 ])
5045
5046 ;;
5047 ;; [vfmasq_n_f])
5048 ;;
5049 (define_insn "mve_vfmasq_n_f<mode>"
5050   [
5051    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5052         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5053                        (match_operand:MVE_0 2 "s_register_operand" "w")
5054                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5055          VFMASQ_N_F))
5056   ]
5057   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5058   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
5059   [(set_attr "type" "mve_move")
5060 ])
5061 ;;
5062 ;; [vfmsq_f])
5063 ;;
5064 (define_insn "mve_vfmsq_f<mode>"
5065   [
5066    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5067         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5068                        (match_operand:MVE_0 2 "s_register_operand" "w")
5069                        (match_operand:MVE_0 3 "s_register_operand" "w")]
5070          VFMSQ_F))
5071   ]
5072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5073   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
5074   [(set_attr "type" "mve_move")
5075 ])
5076
5077 ;;
5078 ;; [vmaxnmaq_m_f])
5079 ;;
5080 (define_insn "mve_vmaxnmaq_m_f<mode>"
5081   [
5082    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5083         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5084                        (match_operand:MVE_0 2 "s_register_operand" "w")
5085                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5086          VMAXNMAQ_M_F))
5087   ]
5088   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5089   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
5090   [(set_attr "type" "mve_move")
5091    (set_attr "length""8")])
5092 ;;
5093 ;; [vmaxnmavq_p_f])
5094 ;;
5095 (define_insn "mve_vmaxnmavq_p_f<mode>"
5096   [
5097    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5098         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5099                        (match_operand:MVE_0 2 "s_register_operand" "w")
5100                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5101          VMAXNMAVQ_P_F))
5102   ]
5103   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5104   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
5105   [(set_attr "type" "mve_move")
5106    (set_attr "length""8")])
5107
5108 ;;
5109 ;; [vmaxnmvq_p_f])
5110 ;;
5111 (define_insn "mve_vmaxnmvq_p_f<mode>"
5112   [
5113    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5114         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5115                        (match_operand:MVE_0 2 "s_register_operand" "w")
5116                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5117          VMAXNMVQ_P_F))
5118   ]
5119   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5120   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
5121   [(set_attr "type" "mve_move")
5122    (set_attr "length""8")])
5123 ;;
5124 ;; [vminnmaq_m_f])
5125 ;;
5126 (define_insn "mve_vminnmaq_m_f<mode>"
5127   [
5128    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5129         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5130                        (match_operand:MVE_0 2 "s_register_operand" "w")
5131                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5132          VMINNMAQ_M_F))
5133   ]
5134   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5135   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
5136   [(set_attr "type" "mve_move")
5137    (set_attr "length""8")])
5138
5139 ;;
5140 ;; [vminnmavq_p_f])
5141 ;;
5142 (define_insn "mve_vminnmavq_p_f<mode>"
5143   [
5144    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5145         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5146                        (match_operand:MVE_0 2 "s_register_operand" "w")
5147                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5148          VMINNMAVQ_P_F))
5149   ]
5150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5151   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
5152   [(set_attr "type" "mve_move")
5153    (set_attr "length""8")])
5154 ;;
5155 ;; [vminnmvq_p_f])
5156 ;;
5157 (define_insn "mve_vminnmvq_p_f<mode>"
5158   [
5159    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5160         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5161                        (match_operand:MVE_0 2 "s_register_operand" "w")
5162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5163          VMINNMVQ_P_F))
5164   ]
5165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5166   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
5167   [(set_attr "type" "mve_move")
5168    (set_attr "length""8")])
5169
5170 ;;
5171 ;; [vmlaldavaq_s, vmlaldavaq_u])
5172 ;;
5173 (define_insn "mve_vmlaldavaq_<supf><mode>"
5174   [
5175    (set (match_operand:DI 0 "s_register_operand" "=r")
5176         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5177                        (match_operand:MVE_5 2 "s_register_operand" "w")
5178                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5179          VMLALDAVAQ))
5180   ]
5181   "TARGET_HAVE_MVE"
5182   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5183   [(set_attr "type" "mve_move")
5184 ])
5185
5186 ;;
5187 ;; [vmlaldavaxq_s])
5188 ;;
5189 (define_insn "mve_vmlaldavaxq_s<mode>"
5190   [
5191    (set (match_operand:DI 0 "s_register_operand" "=r")
5192         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5193                        (match_operand:MVE_5 2 "s_register_operand" "w")
5194                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5195          VMLALDAVAXQ_S))
5196   ]
5197   "TARGET_HAVE_MVE"
5198   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5199   [(set_attr "type" "mve_move")
5200 ])
5201
5202 ;;
5203 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
5204 ;;
5205 (define_insn "mve_vmlaldavq_p_<supf><mode>"
5206   [
5207    (set (match_operand:DI 0 "s_register_operand" "=r")
5208         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5209                        (match_operand:MVE_5 2 "s_register_operand" "w")
5210                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5211          VMLALDAVQ_P))
5212   ]
5213   "TARGET_HAVE_MVE"
5214   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5215   [(set_attr "type" "mve_move")
5216    (set_attr "length""8")])
5217
5218 ;;
5219 ;; [vmlaldavxq_p_s])
5220 ;;
5221 (define_insn "mve_vmlaldavxq_p_s<mode>"
5222   [
5223    (set (match_operand:DI 0 "s_register_operand" "=r")
5224         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5225                        (match_operand:MVE_5 2 "s_register_operand" "w")
5226                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5227          VMLALDAVXQ_P_S))
5228   ]
5229   "TARGET_HAVE_MVE"
5230   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
5231   [(set_attr "type" "mve_move")
5232    (set_attr "length""8")])
5233 ;;
5234 ;; [vmlsldavaq_s])
5235 ;;
5236 (define_insn "mve_vmlsldavaq_s<mode>"
5237   [
5238    (set (match_operand:DI 0 "s_register_operand" "=r")
5239         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5240                        (match_operand:MVE_5 2 "s_register_operand" "w")
5241                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5242          VMLSLDAVAQ_S))
5243   ]
5244   "TARGET_HAVE_MVE"
5245   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5246   [(set_attr "type" "mve_move")
5247 ])
5248
5249 ;;
5250 ;; [vmlsldavaxq_s])
5251 ;;
5252 (define_insn "mve_vmlsldavaxq_s<mode>"
5253   [
5254    (set (match_operand:DI 0 "s_register_operand" "=r")
5255         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5256                        (match_operand:MVE_5 2 "s_register_operand" "w")
5257                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5258          VMLSLDAVAXQ_S))
5259   ]
5260   "TARGET_HAVE_MVE"
5261   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5262   [(set_attr "type" "mve_move")
5263 ])
5264
5265 ;;
5266 ;; [vmlsldavq_p_s])
5267 ;;
5268 (define_insn "mve_vmlsldavq_p_s<mode>"
5269   [
5270    (set (match_operand:DI 0 "s_register_operand" "=r")
5271         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5272                        (match_operand:MVE_5 2 "s_register_operand" "w")
5273                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5274          VMLSLDAVQ_P_S))
5275   ]
5276   "TARGET_HAVE_MVE"
5277   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5278   [(set_attr "type" "mve_move")
5279    (set_attr "length""8")])
5280
5281 ;;
5282 ;; [vmlsldavxq_p_s])
5283 ;;
5284 (define_insn "mve_vmlsldavxq_p_s<mode>"
5285   [
5286    (set (match_operand:DI 0 "s_register_operand" "=r")
5287         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5288                        (match_operand:MVE_5 2 "s_register_operand" "w")
5289                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5290          VMLSLDAVXQ_P_S))
5291   ]
5292   "TARGET_HAVE_MVE"
5293   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5294   [(set_attr "type" "mve_move")
5295    (set_attr "length""8")])
5296 ;;
5297 ;; [vmovlbq_m_u, vmovlbq_m_s])
5298 ;;
5299 (define_insn "mve_vmovlbq_m_<supf><mode>"
5300   [
5301    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5302         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5303                        (match_operand:MVE_3 2 "s_register_operand" "w")
5304                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5305          VMOVLBQ_M))
5306   ]
5307   "TARGET_HAVE_MVE"
5308   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
5309   [(set_attr "type" "mve_move")
5310    (set_attr "length""8")])
5311 ;;
5312 ;; [vmovltq_m_u, vmovltq_m_s])
5313 ;;
5314 (define_insn "mve_vmovltq_m_<supf><mode>"
5315   [
5316    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5317         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5318                        (match_operand:MVE_3 2 "s_register_operand" "w")
5319                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5320          VMOVLTQ_M))
5321   ]
5322   "TARGET_HAVE_MVE"
5323   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
5324   [(set_attr "type" "mve_move")
5325    (set_attr "length""8")])
5326 ;;
5327 ;; [vmovnbq_m_u, vmovnbq_m_s])
5328 ;;
5329 (define_insn "mve_vmovnbq_m_<supf><mode>"
5330   [
5331    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5332         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5333                        (match_operand:MVE_5 2 "s_register_operand" "w")
5334                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5335          VMOVNBQ_M))
5336   ]
5337   "TARGET_HAVE_MVE"
5338   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
5339   [(set_attr "type" "mve_move")
5340    (set_attr "length""8")])
5341
5342 ;;
5343 ;; [vmovntq_m_u, vmovntq_m_s])
5344 ;;
5345 (define_insn "mve_vmovntq_m_<supf><mode>"
5346   [
5347    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5348         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5349                        (match_operand:MVE_5 2 "s_register_operand" "w")
5350                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5351          VMOVNTQ_M))
5352   ]
5353   "TARGET_HAVE_MVE"
5354   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
5355   [(set_attr "type" "mve_move")
5356    (set_attr "length""8")])
5357
5358 ;;
5359 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
5360 ;;
5361 (define_insn "mve_vmvnq_m_n_<supf><mode>"
5362   [
5363    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5364         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5365                        (match_operand:SI 2 "immediate_operand" "i")
5366                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5367          VMVNQ_M_N))
5368   ]
5369   "TARGET_HAVE_MVE"
5370   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
5371   [(set_attr "type" "mve_move")
5372    (set_attr "length""8")])
5373 ;;
5374 ;; [vnegq_m_f])
5375 ;;
5376 (define_insn "mve_vnegq_m_f<mode>"
5377   [
5378    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5379         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5380                        (match_operand:MVE_0 2 "s_register_operand" "w")
5381                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5382          VNEGQ_M_F))
5383   ]
5384   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5385   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
5386   [(set_attr "type" "mve_move")
5387    (set_attr "length""8")])
5388
5389 ;;
5390 ;; [vorrq_m_n_s, vorrq_m_n_u])
5391 ;;
5392 (define_insn "mve_vorrq_m_n_<supf><mode>"
5393   [
5394    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5395         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5396                        (match_operand:SI 2 "immediate_operand" "i")
5397                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5398          VORRQ_M_N))
5399   ]
5400   "TARGET_HAVE_MVE"
5401   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
5402   [(set_attr "type" "mve_move")
5403    (set_attr "length""8")])
5404 ;;
5405 ;; [vpselq_f])
5406 ;;
5407 (define_insn "mve_vpselq_f<mode>"
5408   [
5409    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5410         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
5411                        (match_operand:MVE_0 2 "s_register_operand" "w")
5412                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5413          VPSELQ_F))
5414   ]
5415   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5416   "vpsel %q0, %q1, %q2"
5417   [(set_attr "type" "mve_move")
5418 ])
5419
5420 ;;
5421 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
5422 ;;
5423 (define_insn "mve_vqmovnbq_m_<supf><mode>"
5424   [
5425    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5426         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5427                        (match_operand:MVE_5 2 "s_register_operand" "w")
5428                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5429          VQMOVNBQ_M))
5430   ]
5431   "TARGET_HAVE_MVE"
5432   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
5433   [(set_attr "type" "mve_move")
5434    (set_attr "length""8")])
5435
5436 ;;
5437 ;; [vqmovntq_m_u, vqmovntq_m_s])
5438 ;;
5439 (define_insn "mve_vqmovntq_m_<supf><mode>"
5440   [
5441    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5442         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5443                        (match_operand:MVE_5 2 "s_register_operand" "w")
5444                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5445          VQMOVNTQ_M))
5446   ]
5447   "TARGET_HAVE_MVE"
5448   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
5449   [(set_attr "type" "mve_move")
5450    (set_attr "length""8")])
5451
5452 ;;
5453 ;; [vqmovunbq_m_s])
5454 ;;
5455 (define_insn "mve_vqmovunbq_m_s<mode>"
5456   [
5457    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5458         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5459                        (match_operand:MVE_5 2 "s_register_operand" "w")
5460                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5461          VQMOVUNBQ_M_S))
5462   ]
5463   "TARGET_HAVE_MVE"
5464   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
5465   [(set_attr "type" "mve_move")
5466    (set_attr "length""8")])
5467
5468 ;;
5469 ;; [vqmovuntq_m_s])
5470 ;;
5471 (define_insn "mve_vqmovuntq_m_s<mode>"
5472   [
5473    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5474         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5475                        (match_operand:MVE_5 2 "s_register_operand" "w")
5476                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5477          VQMOVUNTQ_M_S))
5478   ]
5479   "TARGET_HAVE_MVE"
5480   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
5481   [(set_attr "type" "mve_move")
5482    (set_attr "length""8")])
5483
5484 ;;
5485 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
5486 ;;
5487 (define_insn "mve_vqrshrntq_n_<supf><mode>"
5488   [
5489    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5490         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5491                        (match_operand:MVE_5 2 "s_register_operand" "w")
5492                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5493          VQRSHRNTQ_N))
5494   ]
5495   "TARGET_HAVE_MVE"
5496   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
5497   [(set_attr "type" "mve_move")
5498 ])
5499
5500 ;;
5501 ;; [vqrshruntq_n_s])
5502 ;;
5503 (define_insn "mve_vqrshruntq_n_s<mode>"
5504   [
5505    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5506         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5507                        (match_operand:MVE_5 2 "s_register_operand" "w")
5508                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5509          VQRSHRUNTQ_N_S))
5510   ]
5511   "TARGET_HAVE_MVE"
5512   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
5513   [(set_attr "type" "mve_move")
5514 ])
5515
5516 ;;
5517 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
5518 ;;
5519 (define_insn "mve_vqshrnbq_n_<supf><mode>"
5520   [
5521    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5522         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5523                        (match_operand:MVE_5 2 "s_register_operand" "w")
5524                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5525          VQSHRNBQ_N))
5526   ]
5527   "TARGET_HAVE_MVE"
5528   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5529   [(set_attr "type" "mve_move")
5530 ])
5531
5532 ;;
5533 ;; [vqshrntq_n_u, vqshrntq_n_s])
5534 ;;
5535 (define_insn "mve_vqshrntq_n_<supf><mode>"
5536   [
5537    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5538         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5539                        (match_operand:MVE_5 2 "s_register_operand" "w")
5540                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5541          VQSHRNTQ_N))
5542   ]
5543   "TARGET_HAVE_MVE"
5544   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
5545   [(set_attr "type" "mve_move")
5546 ])
5547
5548 ;;
5549 ;; [vqshrunbq_n_s])
5550 ;;
5551 (define_insn "mve_vqshrunbq_n_s<mode>"
5552   [
5553    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5554         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5555                        (match_operand:MVE_5 2 "s_register_operand" "w")
5556                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5557          VQSHRUNBQ_N_S))
5558   ]
5559   "TARGET_HAVE_MVE"
5560   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5561   [(set_attr "type" "mve_move")
5562 ])
5563
5564 ;;
5565 ;; [vqshruntq_n_s])
5566 ;;
5567 (define_insn "mve_vqshruntq_n_s<mode>"
5568   [
5569    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5570         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5571                        (match_operand:MVE_5 2 "s_register_operand" "w")
5572                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5573          VQSHRUNTQ_N_S))
5574   ]
5575   "TARGET_HAVE_MVE"
5576   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5577   [(set_attr "type" "mve_move")
5578 ])
5579
5580 ;;
5581 ;; [vrev32q_m_f])
5582 ;;
5583 (define_insn "mve_vrev32q_m_fv8hf"
5584   [
5585    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5586         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5587                        (match_operand:V8HF 2 "s_register_operand" "w")
5588                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5589          VREV32Q_M_F))
5590   ]
5591   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5592   "vpst\;vrev32t.16 %q0, %q2"
5593   [(set_attr "type" "mve_move")
5594    (set_attr "length""8")])
5595
5596 ;;
5597 ;; [vrev32q_m_s, vrev32q_m_u])
5598 ;;
5599 (define_insn "mve_vrev32q_m_<supf><mode>"
5600   [
5601    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5602         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5603                        (match_operand:MVE_3 2 "s_register_operand" "w")
5604                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5605          VREV32Q_M))
5606   ]
5607   "TARGET_HAVE_MVE"
5608   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5609   [(set_attr "type" "mve_move")
5610    (set_attr "length""8")])
5611
5612 ;;
5613 ;; [vrev64q_m_f])
5614 ;;
5615 (define_insn "mve_vrev64q_m_f<mode>"
5616   [
5617    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5618         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5619                        (match_operand:MVE_0 2 "s_register_operand" "w")
5620                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5621          VREV64Q_M_F))
5622   ]
5623   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5624   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5625   [(set_attr "type" "mve_move")
5626    (set_attr "length""8")])
5627
5628 ;;
5629 ;; [vrmlaldavhaxq_s])
5630 ;;
5631 (define_insn "mve_vrmlaldavhaxq_sv4si"
5632   [
5633    (set (match_operand:DI 0 "s_register_operand" "=r")
5634         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5635                        (match_operand:V4SI 2 "s_register_operand" "w")
5636                        (match_operand:V4SI 3 "s_register_operand" "w")]
5637          VRMLALDAVHAXQ_S))
5638   ]
5639   "TARGET_HAVE_MVE"
5640   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5641   [(set_attr "type" "mve_move")
5642 ])
5643
5644 ;;
5645 ;; [vrmlaldavhxq_p_s])
5646 ;;
5647 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5648   [
5649    (set (match_operand:DI 0 "s_register_operand" "=r")
5650         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5651                        (match_operand:V4SI 2 "s_register_operand" "w")
5652                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5653          VRMLALDAVHXQ_P_S))
5654   ]
5655   "TARGET_HAVE_MVE"
5656   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5657   [(set_attr "type" "mve_move")
5658    (set_attr "length""8")])
5659
5660 ;;
5661 ;; [vrmlsldavhaxq_s])
5662 ;;
5663 (define_insn "mve_vrmlsldavhaxq_sv4si"
5664   [
5665    (set (match_operand:DI 0 "s_register_operand" "=r")
5666         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5667                        (match_operand:V4SI 2 "s_register_operand" "w")
5668                        (match_operand:V4SI 3 "s_register_operand" "w")]
5669          VRMLSLDAVHAXQ_S))
5670   ]
5671   "TARGET_HAVE_MVE"
5672   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5673   [(set_attr "type" "mve_move")
5674 ])
5675
5676 ;;
5677 ;; [vrmlsldavhq_p_s])
5678 ;;
5679 (define_insn "mve_vrmlsldavhq_p_sv4si"
5680   [
5681    (set (match_operand:DI 0 "s_register_operand" "=r")
5682         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5683                        (match_operand:V4SI 2 "s_register_operand" "w")
5684                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5685          VRMLSLDAVHQ_P_S))
5686   ]
5687   "TARGET_HAVE_MVE"
5688   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5689   [(set_attr "type" "mve_move")
5690    (set_attr "length""8")])
5691
5692 ;;
5693 ;; [vrmlsldavhxq_p_s])
5694 ;;
5695 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5696   [
5697    (set (match_operand:DI 0 "s_register_operand" "=r")
5698         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5699                        (match_operand:V4SI 2 "s_register_operand" "w")
5700                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5701          VRMLSLDAVHXQ_P_S))
5702   ]
5703   "TARGET_HAVE_MVE"
5704   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5705   [(set_attr "type" "mve_move")
5706    (set_attr "length""8")])
5707
5708 ;;
5709 ;; [vrndaq_m_f])
5710 ;;
5711 (define_insn "mve_vrndaq_m_f<mode>"
5712   [
5713    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5714         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5715                        (match_operand:MVE_0 2 "s_register_operand" "w")
5716                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5717          VRNDAQ_M_F))
5718   ]
5719   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5720   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5721   [(set_attr "type" "mve_move")
5722    (set_attr "length""8")])
5723
5724 ;;
5725 ;; [vrndmq_m_f])
5726 ;;
5727 (define_insn "mve_vrndmq_m_f<mode>"
5728   [
5729    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5730         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5731                        (match_operand:MVE_0 2 "s_register_operand" "w")
5732                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5733          VRNDMQ_M_F))
5734   ]
5735   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5736   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5737   [(set_attr "type" "mve_move")
5738    (set_attr "length""8")])
5739
5740 ;;
5741 ;; [vrndnq_m_f])
5742 ;;
5743 (define_insn "mve_vrndnq_m_f<mode>"
5744   [
5745    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5746         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5747                        (match_operand:MVE_0 2 "s_register_operand" "w")
5748                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5749          VRNDNQ_M_F))
5750   ]
5751   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5752   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5753   [(set_attr "type" "mve_move")
5754    (set_attr "length""8")])
5755
5756 ;;
5757 ;; [vrndpq_m_f])
5758 ;;
5759 (define_insn "mve_vrndpq_m_f<mode>"
5760   [
5761    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5762         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5763                        (match_operand:MVE_0 2 "s_register_operand" "w")
5764                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5765          VRNDPQ_M_F))
5766   ]
5767   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5768   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5769   [(set_attr "type" "mve_move")
5770    (set_attr "length""8")])
5771
5772 ;;
5773 ;; [vrndxq_m_f])
5774 ;;
5775 (define_insn "mve_vrndxq_m_f<mode>"
5776   [
5777    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5778         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5779                        (match_operand:MVE_0 2 "s_register_operand" "w")
5780                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5781          VRNDXQ_M_F))
5782   ]
5783   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5784   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5785   [(set_attr "type" "mve_move")
5786    (set_attr "length""8")])
5787
5788 ;;
5789 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5790 ;;
5791 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5792   [
5793    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5794         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5795                        (match_operand:MVE_5 2 "s_register_operand" "w")
5796                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5797          VRSHRNBQ_N))
5798   ]
5799   "TARGET_HAVE_MVE"
5800   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5801   [(set_attr "type" "mve_move")
5802 ])
5803
5804 ;;
5805 ;; [vrshrntq_n_u, vrshrntq_n_s])
5806 ;;
5807 (define_insn "mve_vrshrntq_n_<supf><mode>"
5808   [
5809    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5810         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5811                        (match_operand:MVE_5 2 "s_register_operand" "w")
5812                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5813          VRSHRNTQ_N))
5814   ]
5815   "TARGET_HAVE_MVE"
5816   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5817   [(set_attr "type" "mve_move")
5818 ])
5819
5820 ;;
5821 ;; [vshrnbq_n_u, vshrnbq_n_s])
5822 ;;
5823 (define_insn "mve_vshrnbq_n_<supf><mode>"
5824   [
5825    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5826         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5827                        (match_operand:MVE_5 2 "s_register_operand" "w")
5828                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5829          VSHRNBQ_N))
5830   ]
5831   "TARGET_HAVE_MVE"
5832   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5833   [(set_attr "type" "mve_move")
5834 ])
5835
5836 ;;
5837 ;; [vshrntq_n_s, vshrntq_n_u])
5838 ;;
5839 (define_insn "mve_vshrntq_n_<supf><mode>"
5840   [
5841    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5842         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5843                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5844                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5845          VSHRNTQ_N))
5846   ]
5847   "TARGET_HAVE_MVE"
5848   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5849   [(set_attr "type" "mve_move")
5850 ])
5851
5852 ;;
5853 ;; [vcvtmq_m_s, vcvtmq_m_u])
5854 ;;
5855 (define_insn "mve_vcvtmq_m_<supf><mode>"
5856   [
5857    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5858         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5859                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5860                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5861          VCVTMQ_M))
5862   ]
5863   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5864   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5865   [(set_attr "type" "mve_move")
5866    (set_attr "length""8")])
5867
5868 ;;
5869 ;; [vcvtpq_m_u, vcvtpq_m_s])
5870 ;;
5871 (define_insn "mve_vcvtpq_m_<supf><mode>"
5872   [
5873    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5874         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5875                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5876                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5877          VCVTPQ_M))
5878   ]
5879   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5880   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5881   [(set_attr "type" "mve_move")
5882    (set_attr "length""8")])
5883
5884 ;;
5885 ;; [vcvtnq_m_s, vcvtnq_m_u])
5886 ;;
5887 (define_insn "mve_vcvtnq_m_<supf><mode>"
5888   [
5889    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5890         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5891                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5892                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5893          VCVTNQ_M))
5894   ]
5895   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5896   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5897   [(set_attr "type" "mve_move")
5898    (set_attr "length""8")])
5899
5900 ;;
5901 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5902 ;;
5903 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5904   [
5905    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5906         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5907                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5908                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5909                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5910          VCVTQ_M_N_FROM_F))
5911   ]
5912   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5913   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5914   [(set_attr "type" "mve_move")
5915    (set_attr "length""8")])
5916
5917 ;;
5918 ;; [vrev16q_m_u, vrev16q_m_s])
5919 ;;
5920 (define_insn "mve_vrev16q_m_<supf>v16qi"
5921   [
5922    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5923         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5924                        (match_operand:V16QI 2 "s_register_operand" "w")
5925                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5926          VREV16Q_M))
5927   ]
5928   "TARGET_HAVE_MVE"
5929   "vpst\;vrev16t.8 %q0, %q2"
5930   [(set_attr "type" "mve_move")
5931    (set_attr "length""8")])
5932
5933 ;;
5934 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5935 ;;
5936 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5937   [
5938    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5939         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5940                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5941                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5942          VCVTQ_M_FROM_F))
5943   ]
5944   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5945   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5946   [(set_attr "type" "mve_move")
5947    (set_attr "length""8")])
5948
5949 ;;
5950 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5951 ;;
5952 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5953   [
5954    (set (match_operand:DI 0 "s_register_operand" "=r")
5955         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5956                     (match_operand:V4SI 2 "s_register_operand" "w")
5957                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5958          VRMLALDAVHQ_P))
5959   ]
5960   "TARGET_HAVE_MVE"
5961   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5962   [(set_attr "type" "mve_move")
5963    (set_attr "length""8")])
5964
5965 ;;
5966 ;; [vrmlsldavhaq_s])
5967 ;;
5968 (define_insn "mve_vrmlsldavhaq_sv4si"
5969   [
5970    (set (match_operand:DI 0 "s_register_operand" "=r")
5971         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5972                     (match_operand:V4SI 2 "s_register_operand" "w")
5973                     (match_operand:V4SI 3 "s_register_operand" "w")]
5974          VRMLSLDAVHAQ_S))
5975   ]
5976   "TARGET_HAVE_MVE"
5977   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5978   [(set_attr "type" "mve_move")
5979 ])
5980
5981 ;;
5982 ;; [vabavq_p_s, vabavq_p_u])
5983 ;;
5984 (define_insn "mve_vabavq_p_<supf><mode>"
5985   [
5986    (set (match_operand:SI 0 "s_register_operand" "=r")
5987         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5988                     (match_operand:MVE_2 2 "s_register_operand" "w")
5989                     (match_operand:MVE_2 3 "s_register_operand" "w")
5990                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5991          VABAVQ_P))
5992   ]
5993   "TARGET_HAVE_MVE"
5994   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5995   [(set_attr "type" "mve_move")
5996 ])
5997
5998 ;;
5999 ;; [vqshluq_m_n_s])
6000 ;;
6001 (define_insn "mve_vqshluq_m_n_s<mode>"
6002   [
6003    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6004         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6005                        (match_operand:MVE_2 2 "s_register_operand" "w")
6006                        (match_operand:SI 3 "mve_imm_7" "Ra")
6007                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6008          VQSHLUQ_M_N_S))
6009   ]
6010   "TARGET_HAVE_MVE"
6011   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
6012   [(set_attr "type" "mve_move")])
6013
6014 ;;
6015 ;; [vshlq_m_s, vshlq_m_u])
6016 ;;
6017 (define_insn "mve_vshlq_m_<supf><mode>"
6018   [
6019    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6020         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6021                        (match_operand:MVE_2 2 "s_register_operand" "w")
6022                        (match_operand:MVE_2 3 "s_register_operand" "w")
6023                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6024          VSHLQ_M))
6025   ]
6026   "TARGET_HAVE_MVE"
6027   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6028   [(set_attr "type" "mve_move")])
6029
6030 ;;
6031 ;; [vsriq_m_n_s, vsriq_m_n_u])
6032 ;;
6033 (define_insn "mve_vsriq_m_n_<supf><mode>"
6034   [
6035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6037                        (match_operand:MVE_2 2 "s_register_operand" "w")
6038                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
6039                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6040          VSRIQ_M_N))
6041   ]
6042   "TARGET_HAVE_MVE"
6043   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
6044   [(set_attr "type" "mve_move")])
6045
6046 ;;
6047 ;; [vsubq_m_u, vsubq_m_s])
6048 ;;
6049 (define_insn "mve_vsubq_m_<supf><mode>"
6050   [
6051    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6052         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6053                        (match_operand:MVE_2 2 "s_register_operand" "w")
6054                        (match_operand:MVE_2 3 "s_register_operand" "w")
6055                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6056          VSUBQ_M))
6057   ]
6058   "TARGET_HAVE_MVE"
6059   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
6060   [(set_attr "type" "mve_move")])
6061
6062 ;;
6063 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
6064 ;;
6065 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
6066   [
6067    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6068         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6069                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
6070                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6071                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6072          VCVTQ_M_N_TO_F))
6073   ]
6074   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6075   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6076   [(set_attr "type" "mve_move")
6077    (set_attr "length""8")])
6078 ;;
6079 ;; [vabdq_m_s, vabdq_m_u])
6080 ;;
6081 (define_insn "mve_vabdq_m_<supf><mode>"
6082   [
6083    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6084         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6085                        (match_operand:MVE_2 2 "s_register_operand" "w")
6086                        (match_operand:MVE_2 3 "s_register_operand" "w")
6087                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6088          VABDQ_M))
6089   ]
6090   "TARGET_HAVE_MVE"
6091   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6092   [(set_attr "type" "mve_move")
6093    (set_attr "length""8")])
6094
6095 ;;
6096 ;; [vaddq_m_n_s, vaddq_m_n_u])
6097 ;;
6098 (define_insn "mve_vaddq_m_n_<supf><mode>"
6099   [
6100    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6101         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6102                        (match_operand:MVE_2 2 "s_register_operand" "w")
6103                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6104                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6105          VADDQ_M_N))
6106   ]
6107   "TARGET_HAVE_MVE"
6108   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
6109   [(set_attr "type" "mve_move")
6110    (set_attr "length""8")])
6111
6112 ;;
6113 ;; [vaddq_m_u, vaddq_m_s])
6114 ;;
6115 (define_insn "mve_vaddq_m_<supf><mode>"
6116   [
6117    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6118         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6119                        (match_operand:MVE_2 2 "s_register_operand" "w")
6120                        (match_operand:MVE_2 3 "s_register_operand" "w")
6121                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6122          VADDQ_M))
6123   ]
6124   "TARGET_HAVE_MVE"
6125   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
6126   [(set_attr "type" "mve_move")
6127    (set_attr "length""8")])
6128
6129 ;;
6130 ;; [vandq_m_u, vandq_m_s])
6131 ;;
6132 (define_insn "mve_vandq_m_<supf><mode>"
6133   [
6134    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6135         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6136                        (match_operand:MVE_2 2 "s_register_operand" "w")
6137                        (match_operand:MVE_2 3 "s_register_operand" "w")
6138                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6139          VANDQ_M))
6140   ]
6141   "TARGET_HAVE_MVE"
6142   "vpst\;vandt %q0, %q2, %q3"
6143   [(set_attr "type" "mve_move")
6144    (set_attr "length""8")])
6145
6146 ;;
6147 ;; [vbicq_m_u, vbicq_m_s])
6148 ;;
6149 (define_insn "mve_vbicq_m_<supf><mode>"
6150   [
6151    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6152         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6153                        (match_operand:MVE_2 2 "s_register_operand" "w")
6154                        (match_operand:MVE_2 3 "s_register_operand" "w")
6155                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6156          VBICQ_M))
6157   ]
6158   "TARGET_HAVE_MVE"
6159   "vpst\;vbict %q0, %q2, %q3"
6160   [(set_attr "type" "mve_move")
6161    (set_attr "length""8")])
6162
6163 ;;
6164 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
6165 ;;
6166 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
6167   [
6168    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6169         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6170                        (match_operand:MVE_2 2 "s_register_operand" "w")
6171                        (match_operand:SI 3 "s_register_operand" "r")
6172                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6173          VBRSRQ_M_N))
6174   ]
6175   "TARGET_HAVE_MVE"
6176   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6177   [(set_attr "type" "mve_move")
6178    (set_attr "length""8")])
6179
6180 ;;
6181 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
6182 ;;
6183 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
6184   [
6185    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6186         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6187                        (match_operand:MVE_2 2 "s_register_operand" "w")
6188                        (match_operand:MVE_2 3 "s_register_operand" "w")
6189                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6190          VCADDQ_ROT270_M))
6191   ]
6192   "TARGET_HAVE_MVE"
6193   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
6194   [(set_attr "type" "mve_move")
6195    (set_attr "length""8")])
6196
6197 ;;
6198 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
6199 ;;
6200 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
6201   [
6202    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6203         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6204                        (match_operand:MVE_2 2 "s_register_operand" "w")
6205                        (match_operand:MVE_2 3 "s_register_operand" "w")
6206                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6207          VCADDQ_ROT90_M))
6208   ]
6209   "TARGET_HAVE_MVE"
6210   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
6211   [(set_attr "type" "mve_move")
6212    (set_attr "length""8")])
6213
6214 ;;
6215 ;; [veorq_m_s, veorq_m_u])
6216 ;;
6217 (define_insn "mve_veorq_m_<supf><mode>"
6218   [
6219    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6220         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6221                        (match_operand:MVE_2 2 "s_register_operand" "w")
6222                        (match_operand:MVE_2 3 "s_register_operand" "w")
6223                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6224          VEORQ_M))
6225   ]
6226   "TARGET_HAVE_MVE"
6227   "vpst\;veort %q0, %q2, %q3"
6228   [(set_attr "type" "mve_move")
6229    (set_attr "length""8")])
6230
6231 ;;
6232 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
6233 ;;
6234 (define_insn "mve_vhaddq_m_n_<supf><mode>"
6235   [
6236    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6237         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6238                        (match_operand:MVE_2 2 "s_register_operand" "w")
6239                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6240                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6241          VHADDQ_M_N))
6242   ]
6243   "TARGET_HAVE_MVE"
6244   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6245   [(set_attr "type" "mve_move")
6246    (set_attr "length""8")])
6247
6248 ;;
6249 ;; [vhaddq_m_s, vhaddq_m_u])
6250 ;;
6251 (define_insn "mve_vhaddq_m_<supf><mode>"
6252   [
6253    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6254         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6255                        (match_operand:MVE_2 2 "s_register_operand" "w")
6256                        (match_operand:MVE_2 3 "s_register_operand" "w")
6257                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6258          VHADDQ_M))
6259   ]
6260   "TARGET_HAVE_MVE"
6261   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6262   [(set_attr "type" "mve_move")
6263    (set_attr "length""8")])
6264
6265 ;;
6266 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
6267 ;;
6268 (define_insn "mve_vhsubq_m_n_<supf><mode>"
6269   [
6270    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6271         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6272                        (match_operand:MVE_2 2 "s_register_operand" "w")
6273                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6274                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6275          VHSUBQ_M_N))
6276   ]
6277   "TARGET_HAVE_MVE"
6278   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6279   [(set_attr "type" "mve_move")
6280    (set_attr "length""8")])
6281
6282 ;;
6283 ;; [vhsubq_m_s, vhsubq_m_u])
6284 ;;
6285 (define_insn "mve_vhsubq_m_<supf><mode>"
6286   [
6287    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6288         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6289                        (match_operand:MVE_2 2 "s_register_operand" "w")
6290                        (match_operand:MVE_2 3 "s_register_operand" "w")
6291                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6292          VHSUBQ_M))
6293   ]
6294   "TARGET_HAVE_MVE"
6295   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6296   [(set_attr "type" "mve_move")
6297    (set_attr "length""8")])
6298
6299 ;;
6300 ;; [vmaxq_m_s, vmaxq_m_u])
6301 ;;
6302 (define_insn "mve_vmaxq_m_<supf><mode>"
6303   [
6304    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6305         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6306                        (match_operand:MVE_2 2 "s_register_operand" "w")
6307                        (match_operand:MVE_2 3 "s_register_operand" "w")
6308                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6309          VMAXQ_M))
6310   ]
6311   "TARGET_HAVE_MVE"
6312   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6313   [(set_attr "type" "mve_move")
6314    (set_attr "length""8")])
6315
6316 ;;
6317 ;; [vminq_m_s, vminq_m_u])
6318 ;;
6319 (define_insn "mve_vminq_m_<supf><mode>"
6320   [
6321    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6322         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6323                        (match_operand:MVE_2 2 "s_register_operand" "w")
6324                        (match_operand:MVE_2 3 "s_register_operand" "w")
6325                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6326          VMINQ_M))
6327   ]
6328   "TARGET_HAVE_MVE"
6329   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6330   [(set_attr "type" "mve_move")
6331    (set_attr "length""8")])
6332
6333 ;;
6334 ;; [vmladavaq_p_u, vmladavaq_p_s])
6335 ;;
6336 (define_insn "mve_vmladavaq_p_<supf><mode>"
6337   [
6338    (set (match_operand:SI 0 "s_register_operand" "=Te")
6339         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6340                     (match_operand:MVE_2 2 "s_register_operand" "w")
6341                     (match_operand:MVE_2 3 "s_register_operand" "w")
6342                     (match_operand:HI 4 "vpr_register_operand" "Up")]
6343          VMLADAVAQ_P))
6344   ]
6345   "TARGET_HAVE_MVE"
6346   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
6347   [(set_attr "type" "mve_move")
6348    (set_attr "length""8")])
6349
6350 ;;
6351 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
6352 ;;
6353 (define_insn "mve_vmlaq_m_n_<supf><mode>"
6354   [
6355    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6356         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6357                        (match_operand:MVE_2 2 "s_register_operand" "w")
6358                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6359                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6360          VMLAQ_M_N))
6361   ]
6362   "TARGET_HAVE_MVE"
6363   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
6364   [(set_attr "type" "mve_move")
6365    (set_attr "length""8")])
6366
6367 ;;
6368 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
6369 ;;
6370 (define_insn "mve_vmlasq_m_n_<supf><mode>"
6371   [
6372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6373         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6374                        (match_operand:MVE_2 2 "s_register_operand" "w")
6375                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6376                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6377          VMLASQ_M_N))
6378   ]
6379   "TARGET_HAVE_MVE"
6380   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6381   [(set_attr "type" "mve_move")
6382    (set_attr "length""8")])
6383
6384 ;;
6385 ;; [vmulhq_m_s, vmulhq_m_u])
6386 ;;
6387 (define_insn "mve_vmulhq_m_<supf><mode>"
6388   [
6389    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6390         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6391                        (match_operand:MVE_2 2 "s_register_operand" "w")
6392                        (match_operand:MVE_2 3 "s_register_operand" "w")
6393                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6394          VMULHQ_M))
6395   ]
6396   "TARGET_HAVE_MVE"
6397   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6398   [(set_attr "type" "mve_move")
6399    (set_attr "length""8")])
6400
6401 ;;
6402 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
6403 ;;
6404 (define_insn "mve_vmullbq_int_m_<supf><mode>"
6405   [
6406    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6407         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6408                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6409                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6410                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6411          VMULLBQ_INT_M))
6412   ]
6413   "TARGET_HAVE_MVE"
6414   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6415   [(set_attr "type" "mve_move")
6416    (set_attr "length""8")])
6417
6418 ;;
6419 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
6420 ;;
6421 (define_insn "mve_vmulltq_int_m_<supf><mode>"
6422   [
6423    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6424         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6425                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6426                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6427                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6428          VMULLTQ_INT_M))
6429   ]
6430   "TARGET_HAVE_MVE"
6431   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6432   [(set_attr "type" "mve_move")
6433    (set_attr "length""8")])
6434
6435 ;;
6436 ;; [vmulq_m_n_u, vmulq_m_n_s])
6437 ;;
6438 (define_insn "mve_vmulq_m_n_<supf><mode>"
6439   [
6440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6441         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6442                        (match_operand:MVE_2 2 "s_register_operand" "w")
6443                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6444                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6445          VMULQ_M_N))
6446   ]
6447   "TARGET_HAVE_MVE"
6448   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
6449   [(set_attr "type" "mve_move")
6450    (set_attr "length""8")])
6451
6452 ;;
6453 ;; [vmulq_m_s, vmulq_m_u])
6454 ;;
6455 (define_insn "mve_vmulq_m_<supf><mode>"
6456   [
6457    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6458         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6459                        (match_operand:MVE_2 2 "s_register_operand" "w")
6460                        (match_operand:MVE_2 3 "s_register_operand" "w")
6461                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6462          VMULQ_M))
6463   ]
6464   "TARGET_HAVE_MVE"
6465   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
6466   [(set_attr "type" "mve_move")
6467    (set_attr "length""8")])
6468
6469 ;;
6470 ;; [vornq_m_u, vornq_m_s])
6471 ;;
6472 (define_insn "mve_vornq_m_<supf><mode>"
6473   [
6474    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6475         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6476                        (match_operand:MVE_2 2 "s_register_operand" "w")
6477                        (match_operand:MVE_2 3 "s_register_operand" "w")
6478                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6479          VORNQ_M))
6480   ]
6481   "TARGET_HAVE_MVE"
6482   "vpst\;vornt %q0, %q2, %q3"
6483   [(set_attr "type" "mve_move")
6484    (set_attr "length""8")])
6485
6486 ;;
6487 ;; [vorrq_m_s, vorrq_m_u])
6488 ;;
6489 (define_insn "mve_vorrq_m_<supf><mode>"
6490   [
6491    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6492         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6493                        (match_operand:MVE_2 2 "s_register_operand" "w")
6494                        (match_operand:MVE_2 3 "s_register_operand" "w")
6495                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6496          VORRQ_M))
6497   ]
6498   "TARGET_HAVE_MVE"
6499   "vpst\;vorrt %q0, %q2, %q3"
6500   [(set_attr "type" "mve_move")
6501    (set_attr "length""8")])
6502
6503 ;;
6504 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
6505 ;;
6506 (define_insn "mve_vqaddq_m_n_<supf><mode>"
6507   [
6508    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6509         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6510                        (match_operand:MVE_2 2 "s_register_operand" "w")
6511                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6512                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6513          VQADDQ_M_N))
6514   ]
6515   "TARGET_HAVE_MVE"
6516   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6517   [(set_attr "type" "mve_move")
6518    (set_attr "length""8")])
6519
6520 ;;
6521 ;; [vqaddq_m_u, vqaddq_m_s])
6522 ;;
6523 (define_insn "mve_vqaddq_m_<supf><mode>"
6524   [
6525    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6526         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6527                        (match_operand:MVE_2 2 "s_register_operand" "w")
6528                        (match_operand:MVE_2 3 "s_register_operand" "w")
6529                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6530          VQADDQ_M))
6531   ]
6532   "TARGET_HAVE_MVE"
6533   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6534   [(set_attr "type" "mve_move")
6535    (set_attr "length""8")])
6536
6537 ;;
6538 ;; [vqdmlahq_m_n_s])
6539 ;;
6540 (define_insn "mve_vqdmlahq_m_n_s<mode>"
6541   [
6542    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6543         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6544                        (match_operand:MVE_2 2 "s_register_operand" "w")
6545                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6546                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6547          VQDMLAHQ_M_N_S))
6548   ]
6549   "TARGET_HAVE_MVE"
6550   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6551   [(set_attr "type" "mve_move")
6552    (set_attr "length""8")])
6553
6554 ;;
6555 ;; [vqrdmlahq_m_n_s])
6556 ;;
6557 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6558   [
6559    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6560         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6561                        (match_operand:MVE_2 2 "s_register_operand" "w")
6562                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6563                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6564          VQRDMLAHQ_M_N_S))
6565   ]
6566   "TARGET_HAVE_MVE"
6567   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6568   [(set_attr "type" "mve_move")
6569    (set_attr "length""8")])
6570
6571 ;;
6572 ;; [vqrdmlashq_m_n_s])
6573 ;;
6574 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6575   [
6576    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6577         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6578                        (match_operand:MVE_2 2 "s_register_operand" "w")
6579                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6580                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6581          VQRDMLASHQ_M_N_S))
6582   ]
6583   "TARGET_HAVE_MVE"
6584   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6585   [(set_attr "type" "mve_move")
6586    (set_attr "length""8")])
6587
6588 ;;
6589 ;; [vqrshlq_m_u, vqrshlq_m_s])
6590 ;;
6591 (define_insn "mve_vqrshlq_m_<supf><mode>"
6592   [
6593    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6594         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6595                        (match_operand:MVE_2 2 "s_register_operand" "w")
6596                        (match_operand:MVE_2 3 "s_register_operand" "w")
6597                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6598          VQRSHLQ_M))
6599   ]
6600   "TARGET_HAVE_MVE"
6601   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6602   [(set_attr "type" "mve_move")
6603    (set_attr "length""8")])
6604
6605 ;;
6606 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6607 ;;
6608 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6609   [
6610    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6611         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6612                        (match_operand:MVE_2 2 "s_register_operand" "w")
6613                        (match_operand:SI 3 "immediate_operand" "i")
6614                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6615          VQSHLQ_M_N))
6616   ]
6617   "TARGET_HAVE_MVE"
6618   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6619   [(set_attr "type" "mve_move")
6620    (set_attr "length""8")])
6621
6622 ;;
6623 ;; [vqshlq_m_u, vqshlq_m_s])
6624 ;;
6625 (define_insn "mve_vqshlq_m_<supf><mode>"
6626   [
6627    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6628         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6629                        (match_operand:MVE_2 2 "s_register_operand" "w")
6630                        (match_operand:MVE_2 3 "s_register_operand" "w")
6631                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6632          VQSHLQ_M))
6633   ]
6634   "TARGET_HAVE_MVE"
6635   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6636   [(set_attr "type" "mve_move")
6637    (set_attr "length""8")])
6638
6639 ;;
6640 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6641 ;;
6642 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6643   [
6644    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6645         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6646                        (match_operand:MVE_2 2 "s_register_operand" "w")
6647                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6648                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6649          VQSUBQ_M_N))
6650   ]
6651   "TARGET_HAVE_MVE"
6652   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6653   [(set_attr "type" "mve_move")
6654    (set_attr "length""8")])
6655
6656 ;;
6657 ;; [vqsubq_m_u, vqsubq_m_s])
6658 ;;
6659 (define_insn "mve_vqsubq_m_<supf><mode>"
6660   [
6661    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6662         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6663                        (match_operand:MVE_2 2 "s_register_operand" "w")
6664                        (match_operand:MVE_2 3 "s_register_operand" "w")
6665                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6666          VQSUBQ_M))
6667   ]
6668   "TARGET_HAVE_MVE"
6669   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6670   [(set_attr "type" "mve_move")
6671    (set_attr "length""8")])
6672
6673 ;;
6674 ;; [vrhaddq_m_u, vrhaddq_m_s])
6675 ;;
6676 (define_insn "mve_vrhaddq_m_<supf><mode>"
6677   [
6678    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6679         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6680                        (match_operand:MVE_2 2 "s_register_operand" "w")
6681                        (match_operand:MVE_2 3 "s_register_operand" "w")
6682                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6683          VRHADDQ_M))
6684   ]
6685   "TARGET_HAVE_MVE"
6686   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6687   [(set_attr "type" "mve_move")
6688    (set_attr "length""8")])
6689
6690 ;;
6691 ;; [vrmulhq_m_u, vrmulhq_m_s])
6692 ;;
6693 (define_insn "mve_vrmulhq_m_<supf><mode>"
6694   [
6695    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6696         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6697                        (match_operand:MVE_2 2 "s_register_operand" "w")
6698                        (match_operand:MVE_2 3 "s_register_operand" "w")
6699                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6700          VRMULHQ_M))
6701   ]
6702   "TARGET_HAVE_MVE"
6703   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6704   [(set_attr "type" "mve_move")
6705    (set_attr "length""8")])
6706
6707 ;;
6708 ;; [vrshlq_m_s, vrshlq_m_u])
6709 ;;
6710 (define_insn "mve_vrshlq_m_<supf><mode>"
6711   [
6712    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6713         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6714                        (match_operand:MVE_2 2 "s_register_operand" "w")
6715                        (match_operand:MVE_2 3 "s_register_operand" "w")
6716                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6717          VRSHLQ_M))
6718   ]
6719   "TARGET_HAVE_MVE"
6720   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6721   [(set_attr "type" "mve_move")
6722    (set_attr "length""8")])
6723
6724 ;;
6725 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6726 ;;
6727 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6728   [
6729    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6730         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6731                        (match_operand:MVE_2 2 "s_register_operand" "w")
6732                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6733                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6734          VRSHRQ_M_N))
6735   ]
6736   "TARGET_HAVE_MVE"
6737   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6738   [(set_attr "type" "mve_move")
6739    (set_attr "length""8")])
6740
6741 ;;
6742 ;; [vshlq_m_n_s, vshlq_m_n_u])
6743 ;;
6744 (define_insn "mve_vshlq_m_n_<supf><mode>"
6745   [
6746    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6747         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6748                        (match_operand:MVE_2 2 "s_register_operand" "w")
6749                        (match_operand:SI 3 "immediate_operand" "i")
6750                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6751          VSHLQ_M_N))
6752   ]
6753   "TARGET_HAVE_MVE"
6754   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6755   [(set_attr "type" "mve_move")
6756    (set_attr "length""8")])
6757
6758 ;;
6759 ;; [vshrq_m_n_s, vshrq_m_n_u])
6760 ;;
6761 (define_insn "mve_vshrq_m_n_<supf><mode>"
6762   [
6763    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6764         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6765                        (match_operand:MVE_2 2 "s_register_operand" "w")
6766                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6767                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6768          VSHRQ_M_N))
6769   ]
6770   "TARGET_HAVE_MVE"
6771   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6772   [(set_attr "type" "mve_move")
6773    (set_attr "length""8")])
6774
6775 ;;
6776 ;; [vsliq_m_n_u, vsliq_m_n_s])
6777 ;;
6778 (define_insn "mve_vsliq_m_n_<supf><mode>"
6779    [
6780    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6781        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6782                        (match_operand:MVE_2 2 "s_register_operand" "w")
6783                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6784                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6785          VSLIQ_M_N))
6786   ]
6787   "TARGET_HAVE_MVE"
6788   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6789   [(set_attr "type" "mve_move")
6790    (set_attr "length""8")])
6791
6792 ;;
6793 ;; [vsubq_m_n_s, vsubq_m_n_u])
6794 ;;
6795 (define_insn "mve_vsubq_m_n_<supf><mode>"
6796   [
6797    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6798         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6799                        (match_operand:MVE_2 2 "s_register_operand" "w")
6800                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6801                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6802          VSUBQ_M_N))
6803   ]
6804   "TARGET_HAVE_MVE"
6805   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6806   [(set_attr "type" "mve_move")
6807    (set_attr "length""8")])
6808
6809 ;;
6810 ;; [vhcaddq_rot270_m_s])
6811 ;;
6812 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6813   [
6814    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6815         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6816                        (match_operand:MVE_2 2 "s_register_operand" "w")
6817                        (match_operand:MVE_2 3 "s_register_operand" "w")
6818                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6819          VHCADDQ_ROT270_M_S))
6820   ]
6821   "TARGET_HAVE_MVE"
6822   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6823   [(set_attr "type" "mve_move")
6824    (set_attr "length""8")])
6825
6826 ;;
6827 ;; [vhcaddq_rot90_m_s])
6828 ;;
6829 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6830   [
6831    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6832         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6833                        (match_operand:MVE_2 2 "s_register_operand" "w")
6834                        (match_operand:MVE_2 3 "s_register_operand" "w")
6835                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6836          VHCADDQ_ROT90_M_S))
6837   ]
6838   "TARGET_HAVE_MVE"
6839   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6840   [(set_attr "type" "mve_move")
6841    (set_attr "length""8")])
6842
6843 ;;
6844 ;; [vmladavaxq_p_s])
6845 ;;
6846 (define_insn "mve_vmladavaxq_p_s<mode>"
6847   [
6848    (set (match_operand:SI 0 "s_register_operand" "=Te")
6849         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6850                        (match_operand:MVE_2 2 "s_register_operand" "w")
6851                        (match_operand:MVE_2 3 "s_register_operand" "w")
6852                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6853          VMLADAVAXQ_P_S))
6854   ]
6855   "TARGET_HAVE_MVE"
6856   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6857   [(set_attr "type" "mve_move")
6858    (set_attr "length""8")])
6859
6860 ;;
6861 ;; [vmlsdavaq_p_s])
6862 ;;
6863 (define_insn "mve_vmlsdavaq_p_s<mode>"
6864   [
6865    (set (match_operand:SI 0 "s_register_operand" "=Te")
6866         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6867                        (match_operand:MVE_2 2 "s_register_operand" "w")
6868                        (match_operand:MVE_2 3 "s_register_operand" "w")
6869                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6870          VMLSDAVAQ_P_S))
6871   ]
6872   "TARGET_HAVE_MVE"
6873   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6874   [(set_attr "type" "mve_move")
6875    (set_attr "length""8")])
6876
6877 ;;
6878 ;; [vmlsdavaxq_p_s])
6879 ;;
6880 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6881   [
6882    (set (match_operand:SI 0 "s_register_operand" "=Te")
6883         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6884                        (match_operand:MVE_2 2 "s_register_operand" "w")
6885                        (match_operand:MVE_2 3 "s_register_operand" "w")
6886                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6887          VMLSDAVAXQ_P_S))
6888   ]
6889   "TARGET_HAVE_MVE"
6890   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6891   [(set_attr "type" "mve_move")
6892    (set_attr "length""8")])
6893
6894 ;;
6895 ;; [vqdmladhq_m_s])
6896 ;;
6897 (define_insn "mve_vqdmladhq_m_s<mode>"
6898   [
6899    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6900         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6901                        (match_operand:MVE_2 2 "s_register_operand" "w")
6902                        (match_operand:MVE_2 3 "s_register_operand" "w")
6903                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6904          VQDMLADHQ_M_S))
6905   ]
6906   "TARGET_HAVE_MVE"
6907   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6908   [(set_attr "type" "mve_move")
6909    (set_attr "length""8")])
6910
6911 ;;
6912 ;; [vqdmladhxq_m_s])
6913 ;;
6914 (define_insn "mve_vqdmladhxq_m_s<mode>"
6915   [
6916    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6917         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6918                        (match_operand:MVE_2 2 "s_register_operand" "w")
6919                        (match_operand:MVE_2 3 "s_register_operand" "w")
6920                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6921          VQDMLADHXQ_M_S))
6922   ]
6923   "TARGET_HAVE_MVE"
6924   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6925   [(set_attr "type" "mve_move")
6926    (set_attr "length""8")])
6927
6928 ;;
6929 ;; [vqdmlsdhq_m_s])
6930 ;;
6931 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6932   [
6933    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6934         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6935                        (match_operand:MVE_2 2 "s_register_operand" "w")
6936                        (match_operand:MVE_2 3 "s_register_operand" "w")
6937                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6938          VQDMLSDHQ_M_S))
6939   ]
6940   "TARGET_HAVE_MVE"
6941   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6942   [(set_attr "type" "mve_move")
6943    (set_attr "length""8")])
6944
6945 ;;
6946 ;; [vqdmlsdhxq_m_s])
6947 ;;
6948 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6949   [
6950    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6951         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6952                        (match_operand:MVE_2 2 "s_register_operand" "w")
6953                        (match_operand:MVE_2 3 "s_register_operand" "w")
6954                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6955          VQDMLSDHXQ_M_S))
6956   ]
6957   "TARGET_HAVE_MVE"
6958   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6959   [(set_attr "type" "mve_move")
6960    (set_attr "length""8")])
6961
6962 ;;
6963 ;; [vqdmulhq_m_n_s])
6964 ;;
6965 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6966   [
6967    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6968         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6969                        (match_operand:MVE_2 2 "s_register_operand" "w")
6970                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6971                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6972          VQDMULHQ_M_N_S))
6973   ]
6974   "TARGET_HAVE_MVE"
6975   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6976   [(set_attr "type" "mve_move")
6977    (set_attr "length""8")])
6978
6979 ;;
6980 ;; [vqdmulhq_m_s])
6981 ;;
6982 (define_insn "mve_vqdmulhq_m_s<mode>"
6983   [
6984    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6985         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6986                        (match_operand:MVE_2 2 "s_register_operand" "w")
6987                        (match_operand:MVE_2 3 "s_register_operand" "w")
6988                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6989          VQDMULHQ_M_S))
6990   ]
6991   "TARGET_HAVE_MVE"
6992   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6993   [(set_attr "type" "mve_move")
6994    (set_attr "length""8")])
6995
6996 ;;
6997 ;; [vqrdmladhq_m_s])
6998 ;;
6999 (define_insn "mve_vqrdmladhq_m_s<mode>"
7000   [
7001    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7002         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7003                        (match_operand:MVE_2 2 "s_register_operand" "w")
7004                        (match_operand:MVE_2 3 "s_register_operand" "w")
7005                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7006          VQRDMLADHQ_M_S))
7007   ]
7008   "TARGET_HAVE_MVE"
7009   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7010   [(set_attr "type" "mve_move")
7011    (set_attr "length""8")])
7012
7013 ;;
7014 ;; [vqrdmladhxq_m_s])
7015 ;;
7016 (define_insn "mve_vqrdmladhxq_m_s<mode>"
7017   [
7018    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7019         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7020                        (match_operand:MVE_2 2 "s_register_operand" "w")
7021                        (match_operand:MVE_2 3 "s_register_operand" "w")
7022                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7023          VQRDMLADHXQ_M_S))
7024   ]
7025   "TARGET_HAVE_MVE"
7026   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7027   [(set_attr "type" "mve_move")
7028    (set_attr "length""8")])
7029
7030 ;;
7031 ;; [vqrdmlsdhq_m_s])
7032 ;;
7033 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
7034   [
7035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7037                        (match_operand:MVE_2 2 "s_register_operand" "w")
7038                        (match_operand:MVE_2 3 "s_register_operand" "w")
7039                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7040          VQRDMLSDHQ_M_S))
7041   ]
7042   "TARGET_HAVE_MVE"
7043   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7044   [(set_attr "type" "mve_move")
7045    (set_attr "length""8")])
7046
7047 ;;
7048 ;; [vqrdmlsdhxq_m_s])
7049 ;;
7050 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
7051   [
7052    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7053         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7054                        (match_operand:MVE_2 2 "s_register_operand" "w")
7055                        (match_operand:MVE_2 3 "s_register_operand" "w")
7056                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7057          VQRDMLSDHXQ_M_S))
7058   ]
7059   "TARGET_HAVE_MVE"
7060   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7061   [(set_attr "type" "mve_move")
7062    (set_attr "length""8")])
7063
7064 ;;
7065 ;; [vqrdmulhq_m_n_s])
7066 ;;
7067 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
7068   [
7069    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7070         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7071                        (match_operand:MVE_2 2 "s_register_operand" "w")
7072                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7073                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7074          VQRDMULHQ_M_N_S))
7075   ]
7076   "TARGET_HAVE_MVE"
7077   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
7078   [(set_attr "type" "mve_move")
7079    (set_attr "length""8")])
7080
7081 ;;
7082 ;; [vqrdmulhq_m_s])
7083 ;;
7084 (define_insn "mve_vqrdmulhq_m_s<mode>"
7085   [
7086    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7087         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7088                        (match_operand:MVE_2 2 "s_register_operand" "w")
7089                        (match_operand:MVE_2 3 "s_register_operand" "w")
7090                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7091          VQRDMULHQ_M_S))
7092   ]
7093   "TARGET_HAVE_MVE"
7094   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7095   [(set_attr "type" "mve_move")
7096    (set_attr "length""8")])
7097
7098 ;;
7099 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
7100 ;;
7101 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
7102   [
7103    (set (match_operand:DI 0 "s_register_operand" "=r")
7104         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7105                        (match_operand:MVE_5 2 "s_register_operand" "w")
7106                        (match_operand:MVE_5 3 "s_register_operand" "w")
7107                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7108          VMLALDAVAQ_P))
7109   ]
7110   "TARGET_HAVE_MVE"
7111   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7112   [(set_attr "type" "mve_move")
7113    (set_attr "length""8")])
7114
7115 ;;
7116 ;; [vmlaldavaxq_p_u, vmlaldavaxq_p_s])
7117 ;;
7118 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
7119   [
7120    (set (match_operand:DI 0 "s_register_operand" "=r")
7121         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7122                        (match_operand:MVE_5 2 "s_register_operand" "w")
7123                        (match_operand:MVE_5 3 "s_register_operand" "w")
7124                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7125          VMLALDAVAXQ_P))
7126   ]
7127   "TARGET_HAVE_MVE"
7128   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7129   [(set_attr "type" "mve_move")
7130    (set_attr "length""8")])
7131
7132 ;;
7133 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
7134 ;;
7135 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
7136   [
7137    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7138         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7139                        (match_operand:MVE_5 2 "s_register_operand" "w")
7140                        (match_operand:SI 3 "mve_imm_8" "Rb")
7141                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7142          VQRSHRNBQ_M_N))
7143   ]
7144   "TARGET_HAVE_MVE"
7145   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7146   [(set_attr "type" "mve_move")
7147    (set_attr "length""8")])
7148
7149 ;;
7150 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
7151 ;;
7152 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
7153   [
7154    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7155         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7156                        (match_operand:MVE_5 2 "s_register_operand" "w")
7157                        (match_operand:SI 3 "mve_imm_8" "Rb")
7158                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7159          VQRSHRNTQ_M_N))
7160   ]
7161   "TARGET_HAVE_MVE"
7162   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7163   [(set_attr "type" "mve_move")
7164    (set_attr "length""8")])
7165
7166 ;;
7167 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
7168 ;;
7169 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
7170   [
7171    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7172         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7173                        (match_operand:MVE_5 2 "s_register_operand" "w")
7174                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7175                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7176          VQSHRNBQ_M_N))
7177   ]
7178   "TARGET_HAVE_MVE"
7179   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7180   [(set_attr "type" "mve_move")
7181    (set_attr "length""8")])
7182
7183 ;;
7184 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
7185 ;;
7186 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
7187   [
7188    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7189         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7190                        (match_operand:MVE_5 2 "s_register_operand" "w")
7191                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7192                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7193          VQSHRNTQ_M_N))
7194   ]
7195   "TARGET_HAVE_MVE"
7196   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7197   [(set_attr "type" "mve_move")
7198    (set_attr "length""8")])
7199
7200 ;;
7201 ;; [vrmlaldavhaq_p_s])
7202 ;;
7203 (define_insn "mve_vrmlaldavhaq_p_sv4si"
7204   [
7205    (set (match_operand:DI 0 "s_register_operand" "=r")
7206         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7207                        (match_operand:V4SI 2 "s_register_operand" "w")
7208                        (match_operand:V4SI 3 "s_register_operand" "w")
7209                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7210          VRMLALDAVHAQ_P_S))
7211   ]
7212   "TARGET_HAVE_MVE"
7213   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
7214   [(set_attr "type" "mve_move")
7215    (set_attr "length""8")])
7216
7217 ;;
7218 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
7219 ;;
7220 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
7221   [
7222    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7223         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7224                        (match_operand:MVE_5 2 "s_register_operand" "w")
7225                        (match_operand:SI 3 "mve_imm_8" "Rb")
7226                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7227          VRSHRNBQ_M_N))
7228   ]
7229   "TARGET_HAVE_MVE"
7230   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7231   [(set_attr "type" "mve_move")
7232    (set_attr "length""8")])
7233
7234 ;;
7235 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
7236 ;;
7237 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
7238   [
7239    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7240         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7241                        (match_operand:MVE_5 2 "s_register_operand" "w")
7242                        (match_operand:SI 3 "mve_imm_8" "Rb")
7243                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7244          VRSHRNTQ_M_N))
7245   ]
7246   "TARGET_HAVE_MVE"
7247   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7248   [(set_attr "type" "mve_move")
7249    (set_attr "length""8")])
7250
7251 ;;
7252 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
7253 ;;
7254 (define_insn "mve_vshllbq_m_n_<supf><mode>"
7255   [
7256    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7257         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7258                        (match_operand:MVE_3 2 "s_register_operand" "w")
7259                        (match_operand:SI 3 "immediate_operand" "i")
7260                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7261          VSHLLBQ_M_N))
7262   ]
7263   "TARGET_HAVE_MVE"
7264   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7265   [(set_attr "type" "mve_move")
7266    (set_attr "length""8")])
7267
7268 ;;
7269 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
7270 ;;
7271 (define_insn "mve_vshlltq_m_n_<supf><mode>"
7272   [
7273    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7274         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7275                        (match_operand:MVE_3 2 "s_register_operand" "w")
7276                        (match_operand:SI 3 "immediate_operand" "i")
7277                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7278          VSHLLTQ_M_N))
7279   ]
7280   "TARGET_HAVE_MVE"
7281   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7282   [(set_attr "type" "mve_move")
7283    (set_attr "length""8")])
7284
7285 ;;
7286 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
7287 ;;
7288 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
7289   [
7290    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7291         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7292                        (match_operand:MVE_5 2 "s_register_operand" "w")
7293                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7294                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7295          VSHRNBQ_M_N))
7296   ]
7297   "TARGET_HAVE_MVE"
7298   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7299   [(set_attr "type" "mve_move")
7300    (set_attr "length""8")])
7301
7302 ;;
7303 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
7304 ;;
7305 (define_insn "mve_vshrntq_m_n_<supf><mode>"
7306   [
7307    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7308         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7309                        (match_operand:MVE_5 2 "s_register_operand" "w")
7310                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7311                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7312          VSHRNTQ_M_N))
7313   ]
7314   "TARGET_HAVE_MVE"
7315   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7316   [(set_attr "type" "mve_move")
7317    (set_attr "length""8")])
7318
7319 ;;
7320 ;; [vmlsldavaq_p_s])
7321 ;;
7322 (define_insn "mve_vmlsldavaq_p_s<mode>"
7323   [
7324    (set (match_operand:DI 0 "s_register_operand" "=r")
7325         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7326                        (match_operand:MVE_5 2 "s_register_operand" "w")
7327                        (match_operand:MVE_5 3 "s_register_operand" "w")
7328                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7329          VMLSLDAVAQ_P_S))
7330   ]
7331   "TARGET_HAVE_MVE"
7332   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7333   [(set_attr "type" "mve_move")
7334    (set_attr "length""8")])
7335
7336 ;;
7337 ;; [vmlsldavaxq_p_s])
7338 ;;
7339 (define_insn "mve_vmlsldavaxq_p_s<mode>"
7340   [
7341    (set (match_operand:DI 0 "s_register_operand" "=r")
7342         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7343                        (match_operand:MVE_5 2 "s_register_operand" "w")
7344                        (match_operand:MVE_5 3 "s_register_operand" "w")
7345                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7346          VMLSLDAVAXQ_P_S))
7347   ]
7348   "TARGET_HAVE_MVE"
7349   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7350   [(set_attr "type" "mve_move")
7351    (set_attr "length""8")])
7352
7353 ;;
7354 ;; [vmullbq_poly_m_p])
7355 ;;
7356 (define_insn "mve_vmullbq_poly_m_p<mode>"
7357   [
7358    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7359         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7360                        (match_operand:MVE_3 2 "s_register_operand" "w")
7361                        (match_operand:MVE_3 3 "s_register_operand" "w")
7362                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7363          VMULLBQ_POLY_M_P))
7364   ]
7365   "TARGET_HAVE_MVE"
7366   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7367   [(set_attr "type" "mve_move")
7368    (set_attr "length""8")])
7369
7370 ;;
7371 ;; [vmulltq_poly_m_p])
7372 ;;
7373 (define_insn "mve_vmulltq_poly_m_p<mode>"
7374   [
7375    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7376         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7377                        (match_operand:MVE_3 2 "s_register_operand" "w")
7378                        (match_operand:MVE_3 3 "s_register_operand" "w")
7379                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7380          VMULLTQ_POLY_M_P))
7381   ]
7382   "TARGET_HAVE_MVE"
7383   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7384   [(set_attr "type" "mve_move")
7385    (set_attr "length""8")])
7386
7387 ;;
7388 ;; [vqdmullbq_m_n_s])
7389 ;;
7390 (define_insn "mve_vqdmullbq_m_n_s<mode>"
7391   [
7392    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7393         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7394                        (match_operand:MVE_5 2 "s_register_operand" "w")
7395                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7396                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7397          VQDMULLBQ_M_N_S))
7398   ]
7399   "TARGET_HAVE_MVE"
7400   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7401   [(set_attr "type" "mve_move")
7402    (set_attr "length""8")])
7403
7404 ;;
7405 ;; [vqdmullbq_m_s])
7406 ;;
7407 (define_insn "mve_vqdmullbq_m_s<mode>"
7408   [
7409    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7410         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7411                        (match_operand:MVE_5 2 "s_register_operand" "w")
7412                        (match_operand:MVE_5 3 "s_register_operand" "w")
7413                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7414          VQDMULLBQ_M_S))
7415   ]
7416   "TARGET_HAVE_MVE"
7417   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7418   [(set_attr "type" "mve_move")
7419    (set_attr "length""8")])
7420
7421 ;;
7422 ;; [vqdmulltq_m_n_s])
7423 ;;
7424 (define_insn "mve_vqdmulltq_m_n_s<mode>"
7425   [
7426    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7427         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7428                        (match_operand:MVE_5 2 "s_register_operand" "w")
7429                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7430                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7431          VQDMULLTQ_M_N_S))
7432   ]
7433   "TARGET_HAVE_MVE"
7434   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
7435   [(set_attr "type" "mve_move")
7436    (set_attr "length""8")])
7437
7438 ;;
7439 ;; [vqdmulltq_m_s])
7440 ;;
7441 (define_insn "mve_vqdmulltq_m_s<mode>"
7442   [
7443    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7444         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7445                        (match_operand:MVE_5 2 "s_register_operand" "w")
7446                        (match_operand:MVE_5 3 "s_register_operand" "w")
7447                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7448          VQDMULLTQ_M_S))
7449   ]
7450   "TARGET_HAVE_MVE"
7451   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7452   [(set_attr "type" "mve_move")
7453    (set_attr "length""8")])
7454
7455 ;;
7456 ;; [vqrshrunbq_m_n_s])
7457 ;;
7458 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
7459   [
7460    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7461         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7462                        (match_operand:MVE_5 2 "s_register_operand" "w")
7463                        (match_operand:SI 3 "mve_imm_8" "Rb")
7464                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7465          VQRSHRUNBQ_M_N_S))
7466   ]
7467   "TARGET_HAVE_MVE"
7468   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7469   [(set_attr "type" "mve_move")
7470    (set_attr "length""8")])
7471
7472 ;;
7473 ;; [vqrshruntq_m_n_s])
7474 ;;
7475 (define_insn "mve_vqrshruntq_m_n_s<mode>"
7476   [
7477    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7478         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7479                        (match_operand:MVE_5 2 "s_register_operand" "w")
7480                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7481                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7482          VQRSHRUNTQ_M_N_S))
7483   ]
7484   "TARGET_HAVE_MVE"
7485   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7486   [(set_attr "type" "mve_move")
7487    (set_attr "length""8")])
7488
7489 ;;
7490 ;; [vqshrunbq_m_n_s])
7491 ;;
7492 (define_insn "mve_vqshrunbq_m_n_s<mode>"
7493   [
7494    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7495         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7496                        (match_operand:MVE_5 2 "s_register_operand" "w")
7497                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7498                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7499          VQSHRUNBQ_M_N_S))
7500   ]
7501   "TARGET_HAVE_MVE"
7502   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7503   [(set_attr "type" "mve_move")
7504    (set_attr "length""8")])
7505
7506 ;;
7507 ;; [vqshruntq_m_n_s])
7508 ;;
7509 (define_insn "mve_vqshruntq_m_n_s<mode>"
7510   [
7511    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7512         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7513                        (match_operand:MVE_5 2 "s_register_operand" "w")
7514                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7515                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7516          VQSHRUNTQ_M_N_S))
7517   ]
7518   "TARGET_HAVE_MVE"
7519   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7520   [(set_attr "type" "mve_move")
7521    (set_attr "length""8")])
7522
7523 ;;
7524 ;; [vrmlaldavhaq_p_u])
7525 ;;
7526 (define_insn "mve_vrmlaldavhaq_p_uv4si"
7527   [
7528    (set (match_operand:DI 0 "s_register_operand" "=r")
7529         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7530                        (match_operand:V4SI 2 "s_register_operand" "w")
7531                        (match_operand:V4SI 3 "s_register_operand" "w")
7532                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7533          VRMLALDAVHAQ_P_U))
7534   ]
7535   "TARGET_HAVE_MVE"
7536   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7537   [(set_attr "type" "mve_move")
7538    (set_attr "length""8")])
7539
7540 ;;
7541 ;; [vrmlaldavhaxq_p_s])
7542 ;;
7543 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7544   [
7545    (set (match_operand:DI 0 "s_register_operand" "=r")
7546         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7547                        (match_operand:V4SI 2 "s_register_operand" "w")
7548                        (match_operand:V4SI 3 "s_register_operand" "w")
7549                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7550          VRMLALDAVHAXQ_P_S))
7551   ]
7552   "TARGET_HAVE_MVE"
7553   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7554   [(set_attr "type" "mve_move")
7555    (set_attr "length""8")])
7556
7557 ;;
7558 ;; [vrmlsldavhaq_p_s])
7559 ;;
7560 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7561   [
7562    (set (match_operand:DI 0 "s_register_operand" "=r")
7563         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7564                        (match_operand:V4SI 2 "s_register_operand" "w")
7565                        (match_operand:V4SI 3 "s_register_operand" "w")
7566                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7567          VRMLSLDAVHAQ_P_S))
7568   ]
7569   "TARGET_HAVE_MVE"
7570   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7571   [(set_attr "type" "mve_move")
7572    (set_attr "length""8")])
7573
7574 ;;
7575 ;; [vrmlsldavhaxq_p_s])
7576 ;;
7577 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7578   [
7579    (set (match_operand:DI 0 "s_register_operand" "=r")
7580         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7581                        (match_operand:V4SI 2 "s_register_operand" "w")
7582                        (match_operand:V4SI 3 "s_register_operand" "w")
7583                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7584          VRMLSLDAVHAXQ_P_S))
7585   ]
7586   "TARGET_HAVE_MVE"
7587   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7588   [(set_attr "type" "mve_move")
7589    (set_attr "length""8")])
7590 ;;
7591 ;; [vabdq_m_f])
7592 ;;
7593 (define_insn "mve_vabdq_m_f<mode>"
7594   [
7595    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7596         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7597                        (match_operand:MVE_0 2 "s_register_operand" "w")
7598                        (match_operand:MVE_0 3 "s_register_operand" "w")
7599                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7600          VABDQ_M_F))
7601   ]
7602   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7603   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7604   [(set_attr "type" "mve_move")
7605    (set_attr "length""8")])
7606
7607 ;;
7608 ;; [vaddq_m_f])
7609 ;;
7610 (define_insn "mve_vaddq_m_f<mode>"
7611   [
7612    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7613         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7614                        (match_operand:MVE_0 2 "s_register_operand" "w")
7615                        (match_operand:MVE_0 3 "s_register_operand" "w")
7616                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7617          VADDQ_M_F))
7618   ]
7619   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7620   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7621   [(set_attr "type" "mve_move")
7622    (set_attr "length""8")])
7623
7624 ;;
7625 ;; [vaddq_m_n_f])
7626 ;;
7627 (define_insn "mve_vaddq_m_n_f<mode>"
7628   [
7629    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7630         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7631                        (match_operand:MVE_0 2 "s_register_operand" "w")
7632                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7633                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7634          VADDQ_M_N_F))
7635   ]
7636   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7637   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7638   [(set_attr "type" "mve_move")
7639    (set_attr "length""8")])
7640
7641 ;;
7642 ;; [vandq_m_f])
7643 ;;
7644 (define_insn "mve_vandq_m_f<mode>"
7645   [
7646    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7647         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7648                        (match_operand:MVE_0 2 "s_register_operand" "w")
7649                        (match_operand:MVE_0 3 "s_register_operand" "w")
7650                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7651          VANDQ_M_F))
7652   ]
7653   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7654   "vpst\;vandt %q0, %q2, %q3"
7655   [(set_attr "type" "mve_move")
7656    (set_attr "length""8")])
7657
7658 ;;
7659 ;; [vbicq_m_f])
7660 ;;
7661 (define_insn "mve_vbicq_m_f<mode>"
7662   [
7663    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7664         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7665                        (match_operand:MVE_0 2 "s_register_operand" "w")
7666                        (match_operand:MVE_0 3 "s_register_operand" "w")
7667                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7668          VBICQ_M_F))
7669   ]
7670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7671   "vpst\;vbict %q0, %q2, %q3"
7672   [(set_attr "type" "mve_move")
7673    (set_attr "length""8")])
7674
7675 ;;
7676 ;; [vbrsrq_m_n_f])
7677 ;;
7678 (define_insn "mve_vbrsrq_m_n_f<mode>"
7679   [
7680    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7681         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7682                        (match_operand:MVE_0 2 "s_register_operand" "w")
7683                        (match_operand:SI 3 "s_register_operand" "r")
7684                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7685          VBRSRQ_M_N_F))
7686   ]
7687   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7688   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7689   [(set_attr "type" "mve_move")
7690    (set_attr "length""8")])
7691
7692 ;;
7693 ;; [vcaddq_rot270_m_f])
7694 ;;
7695 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7696   [
7697    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7698         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7699                        (match_operand:MVE_0 2 "s_register_operand" "w")
7700                        (match_operand:MVE_0 3 "s_register_operand" "w")
7701                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7702          VCADDQ_ROT270_M_F))
7703   ]
7704   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7705   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7706   [(set_attr "type" "mve_move")
7707    (set_attr "length""8")])
7708
7709 ;;
7710 ;; [vcaddq_rot90_m_f])
7711 ;;
7712 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7713   [
7714    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7715         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7716                        (match_operand:MVE_0 2 "s_register_operand" "w")
7717                        (match_operand:MVE_0 3 "s_register_operand" "w")
7718                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7719          VCADDQ_ROT90_M_F))
7720   ]
7721   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7722   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7723   [(set_attr "type" "mve_move")
7724    (set_attr "length""8")])
7725
7726 ;;
7727 ;; [vcmlaq_m_f])
7728 ;;
7729 (define_insn "mve_vcmlaq_m_f<mode>"
7730   [
7731    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7732         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7733                        (match_operand:MVE_0 2 "s_register_operand" "w")
7734                        (match_operand:MVE_0 3 "s_register_operand" "w")
7735                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7736          VCMLAQ_M_F))
7737   ]
7738   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7739   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7740   [(set_attr "type" "mve_move")
7741    (set_attr "length""8")])
7742
7743 ;;
7744 ;; [vcmlaq_rot180_m_f])
7745 ;;
7746 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7747   [
7748    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7749         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7750                        (match_operand:MVE_0 2 "s_register_operand" "w")
7751                        (match_operand:MVE_0 3 "s_register_operand" "w")
7752                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7753          VCMLAQ_ROT180_M_F))
7754   ]
7755   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7756   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7757   [(set_attr "type" "mve_move")
7758    (set_attr "length""8")])
7759
7760 ;;
7761 ;; [vcmlaq_rot270_m_f])
7762 ;;
7763 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7764   [
7765    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7766         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7767                        (match_operand:MVE_0 2 "s_register_operand" "w")
7768                        (match_operand:MVE_0 3 "s_register_operand" "w")
7769                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7770          VCMLAQ_ROT270_M_F))
7771   ]
7772   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7773   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7774   [(set_attr "type" "mve_move")
7775    (set_attr "length""8")])
7776
7777 ;;
7778 ;; [vcmlaq_rot90_m_f])
7779 ;;
7780 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7781   [
7782    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7783         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7784                        (match_operand:MVE_0 2 "s_register_operand" "w")
7785                        (match_operand:MVE_0 3 "s_register_operand" "w")
7786                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7787          VCMLAQ_ROT90_M_F))
7788   ]
7789   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7790   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7791   [(set_attr "type" "mve_move")
7792    (set_attr "length""8")])
7793
7794 ;;
7795 ;; [vcmulq_m_f])
7796 ;;
7797 (define_insn "mve_vcmulq_m_f<mode>"
7798   [
7799    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7800         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7801                        (match_operand:MVE_0 2 "s_register_operand" "w")
7802                        (match_operand:MVE_0 3 "s_register_operand" "w")
7803                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7804          VCMULQ_M_F))
7805   ]
7806   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7807   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7808   [(set_attr "type" "mve_move")
7809    (set_attr "length""8")])
7810
7811 ;;
7812 ;; [vcmulq_rot180_m_f])
7813 ;;
7814 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7815   [
7816    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7817         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7818                        (match_operand:MVE_0 2 "s_register_operand" "w")
7819                        (match_operand:MVE_0 3 "s_register_operand" "w")
7820                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7821          VCMULQ_ROT180_M_F))
7822   ]
7823   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7824   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7825   [(set_attr "type" "mve_move")
7826    (set_attr "length""8")])
7827
7828 ;;
7829 ;; [vcmulq_rot270_m_f])
7830 ;;
7831 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7832   [
7833    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7834         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7835                        (match_operand:MVE_0 2 "s_register_operand" "w")
7836                        (match_operand:MVE_0 3 "s_register_operand" "w")
7837                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7838          VCMULQ_ROT270_M_F))
7839   ]
7840   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7841   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7842   [(set_attr "type" "mve_move")
7843    (set_attr "length""8")])
7844
7845 ;;
7846 ;; [vcmulq_rot90_m_f])
7847 ;;
7848 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7849   [
7850    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7851         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7852                        (match_operand:MVE_0 2 "s_register_operand" "w")
7853                        (match_operand:MVE_0 3 "s_register_operand" "w")
7854                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7855          VCMULQ_ROT90_M_F))
7856   ]
7857   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7858   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7859   [(set_attr "type" "mve_move")
7860    (set_attr "length""8")])
7861
7862 ;;
7863 ;; [veorq_m_f])
7864 ;;
7865 (define_insn "mve_veorq_m_f<mode>"
7866   [
7867    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7868         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7869                        (match_operand:MVE_0 2 "s_register_operand" "w")
7870                        (match_operand:MVE_0 3 "s_register_operand" "w")
7871                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7872          VEORQ_M_F))
7873   ]
7874   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7875   "vpst\;veort %q0, %q2, %q3"
7876   [(set_attr "type" "mve_move")
7877    (set_attr "length""8")])
7878
7879 ;;
7880 ;; [vfmaq_m_f])
7881 ;;
7882 (define_insn "mve_vfmaq_m_f<mode>"
7883   [
7884    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7885         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7886                        (match_operand:MVE_0 2 "s_register_operand" "w")
7887                        (match_operand:MVE_0 3 "s_register_operand" "w")
7888                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7889          VFMAQ_M_F))
7890   ]
7891   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7892   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7893   [(set_attr "type" "mve_move")
7894    (set_attr "length""8")])
7895
7896 ;;
7897 ;; [vfmaq_m_n_f])
7898 ;;
7899 (define_insn "mve_vfmaq_m_n_f<mode>"
7900   [
7901    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7902         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7903                        (match_operand:MVE_0 2 "s_register_operand" "w")
7904                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7905                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7906          VFMAQ_M_N_F))
7907   ]
7908   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7909   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7910   [(set_attr "type" "mve_move")
7911    (set_attr "length""8")])
7912
7913 ;;
7914 ;; [vfmasq_m_n_f])
7915 ;;
7916 (define_insn "mve_vfmasq_m_n_f<mode>"
7917   [
7918    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7919         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7920                        (match_operand:MVE_0 2 "s_register_operand" "w")
7921                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7922                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7923          VFMASQ_M_N_F))
7924   ]
7925   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7926   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7927   [(set_attr "type" "mve_move")
7928    (set_attr "length""8")])
7929
7930 ;;
7931 ;; [vfmsq_m_f])
7932 ;;
7933 (define_insn "mve_vfmsq_m_f<mode>"
7934   [
7935    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7936         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7937                        (match_operand:MVE_0 2 "s_register_operand" "w")
7938                        (match_operand:MVE_0 3 "s_register_operand" "w")
7939                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7940          VFMSQ_M_F))
7941   ]
7942   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7943   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7944   [(set_attr "type" "mve_move")
7945    (set_attr "length""8")])
7946
7947 ;;
7948 ;; [vmaxnmq_m_f])
7949 ;;
7950 (define_insn "mve_vmaxnmq_m_f<mode>"
7951   [
7952    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7953         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7954                        (match_operand:MVE_0 2 "s_register_operand" "w")
7955                        (match_operand:MVE_0 3 "s_register_operand" "w")
7956                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7957          VMAXNMQ_M_F))
7958   ]
7959   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7960   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7961   [(set_attr "type" "mve_move")
7962    (set_attr "length""8")])
7963
7964 ;;
7965 ;; [vminnmq_m_f])
7966 ;;
7967 (define_insn "mve_vminnmq_m_f<mode>"
7968   [
7969    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7970         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7971                        (match_operand:MVE_0 2 "s_register_operand" "w")
7972                        (match_operand:MVE_0 3 "s_register_operand" "w")
7973                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7974          VMINNMQ_M_F))
7975   ]
7976   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7977   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7978   [(set_attr "type" "mve_move")
7979    (set_attr "length""8")])
7980
7981 ;;
7982 ;; [vmulq_m_f])
7983 ;;
7984 (define_insn "mve_vmulq_m_f<mode>"
7985   [
7986    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7987         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7988                        (match_operand:MVE_0 2 "s_register_operand" "w")
7989                        (match_operand:MVE_0 3 "s_register_operand" "w")
7990                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7991          VMULQ_M_F))
7992   ]
7993   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7994   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7995   [(set_attr "type" "mve_move")
7996    (set_attr "length""8")])
7997
7998 ;;
7999 ;; [vmulq_m_n_f])
8000 ;;
8001 (define_insn "mve_vmulq_m_n_f<mode>"
8002   [
8003    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8004         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8005                        (match_operand:MVE_0 2 "s_register_operand" "w")
8006                        (match_operand:<V_elem> 3 "s_register_operand" "r")
8007                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8008          VMULQ_M_N_F))
8009   ]
8010   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8011   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
8012   [(set_attr "type" "mve_move")
8013    (set_attr "length""8")])
8014
8015 ;;
8016 ;; [vornq_m_f])
8017 ;;
8018 (define_insn "mve_vornq_m_f<mode>"
8019   [
8020    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8021         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8022                        (match_operand:MVE_0 2 "s_register_operand" "w")
8023                        (match_operand:MVE_0 3 "s_register_operand" "w")
8024                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8025          VORNQ_M_F))
8026   ]
8027   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8028   "vpst\;vornt %q0, %q2, %q3"
8029   [(set_attr "type" "mve_move")
8030    (set_attr "length""8")])
8031
8032 ;;
8033 ;; [vorrq_m_f])
8034 ;;
8035 (define_insn "mve_vorrq_m_f<mode>"
8036   [
8037    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8038         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8039                        (match_operand:MVE_0 2 "s_register_operand" "w")
8040                        (match_operand:MVE_0 3 "s_register_operand" "w")
8041                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8042          VORRQ_M_F))
8043   ]
8044   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8045   "vpst\;vorrt %q0, %q2, %q3"
8046   [(set_attr "type" "mve_move")
8047    (set_attr "length""8")])
8048
8049 ;;
8050 ;; [vsubq_m_f])
8051 ;;
8052 (define_insn "mve_vsubq_m_f<mode>"
8053   [
8054    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8055         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8056                        (match_operand:MVE_0 2 "s_register_operand" "w")
8057                        (match_operand:MVE_0 3 "s_register_operand" "w")
8058                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8059          VSUBQ_M_F))
8060   ]
8061   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8062   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
8063   [(set_attr "type" "mve_move")
8064    (set_attr "length""8")])
8065
8066 ;;
8067 ;; [vsubq_m_n_f])
8068 ;;
8069 (define_insn "mve_vsubq_m_n_f<mode>"
8070   [
8071    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8072         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8073                        (match_operand:MVE_0 2 "s_register_operand" "w")
8074                        (match_operand:<V_elem> 3 "s_register_operand" "r")
8075                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8076          VSUBQ_M_N_F))
8077   ]
8078   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8079   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
8080   [(set_attr "type" "mve_move")
8081    (set_attr "length""8")])
8082
8083 ;;
8084 ;; [vstrbq_s vstrbq_u]
8085 ;;
8086 (define_insn "mve_vstrbq_<supf><mode>"
8087   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
8088         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
8089          VSTRBQ))
8090   ]
8091   "TARGET_HAVE_MVE"
8092 {
8093    rtx ops[2];
8094    int regno = REGNO (operands[1]);
8095    ops[1] = gen_rtx_REG (TImode, regno);
8096    ops[0]  = operands[0];
8097    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
8098    return "";
8099 }
8100   [(set_attr "length" "4")])
8101
8102 ;;
8103 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
8104 ;;
8105 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
8106   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
8107    (match_operand:MVE_2 1 "s_register_operand")
8108    (match_operand:MVE_2 2 "s_register_operand")
8109    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
8110   "TARGET_HAVE_MVE"
8111 {
8112   rtx ind = XEXP (operands[0], 0);
8113   gcc_assert (REG_P (ind));
8114   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8115                                                               operands[2]));
8116   DONE;
8117 })
8118
8119 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
8120   [(set (mem:BLK (scratch))
8121         (unspec:BLK
8122           [(match_operand:SI 0 "register_operand" "r")
8123            (match_operand:MVE_2 1 "s_register_operand" "w")
8124            (match_operand:MVE_2 2 "s_register_operand" "w")]
8125           VSTRBSOQ))]
8126   "TARGET_HAVE_MVE"
8127   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
8128   [(set_attr "length" "4")])
8129
8130 ;;
8131 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
8132 ;;
8133 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
8134   [(set (mem:BLK (scratch))
8135         (unspec:BLK
8136                 [(match_operand:V4SI 0 "s_register_operand" "w")
8137                  (match_operand:SI 1 "immediate_operand" "i")
8138                  (match_operand:V4SI 2 "s_register_operand" "w")]
8139          VSTRWSBQ))
8140   ]
8141   "TARGET_HAVE_MVE"
8142 {
8143    rtx ops[3];
8144    ops[0] = operands[0];
8145    ops[1] = operands[1];
8146    ops[2] = operands[2];
8147    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
8148    return "";
8149 }
8150   [(set_attr "length" "4")])
8151
8152 ;;
8153 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
8154 ;;
8155 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
8156   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8157         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8158                        (match_operand:MVE_2 2 "s_register_operand" "w")]
8159          VLDRBGOQ))
8160   ]
8161   "TARGET_HAVE_MVE"
8162 {
8163    rtx ops[3];
8164    ops[0] = operands[0];
8165    ops[1] = operands[1];
8166    ops[2] = operands[2];
8167    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8168      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
8169    else
8170      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8171    return "";
8172 }
8173   [(set_attr "length" "4")])
8174
8175 ;;
8176 ;; [vldrbq_s vldrbq_u]
8177 ;;
8178 (define_insn "mve_vldrbq_<supf><mode>"
8179   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8180         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
8181          VLDRBQ))
8182   ]
8183   "TARGET_HAVE_MVE"
8184 {
8185    rtx ops[2];
8186    int regno = REGNO (operands[0]);
8187    ops[0] = gen_rtx_REG (TImode, regno);
8188    ops[1]  = operands[1];
8189    if (<V_sz_elem> == 8)
8190      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
8191    else
8192      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
8193    return "";
8194 }
8195   [(set_attr "length" "4")])
8196
8197 ;;
8198 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
8199 ;;
8200 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
8201   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8202         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8203                       (match_operand:SI 2 "immediate_operand" "i")]
8204          VLDRWGBQ))
8205   ]
8206   "TARGET_HAVE_MVE"
8207 {
8208    rtx ops[3];
8209    ops[0] = operands[0];
8210    ops[1] = operands[1];
8211    ops[2] = operands[2];
8212    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8213    return "";
8214 }
8215   [(set_attr "length" "4")])
8216
8217 ;;
8218 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
8219 ;;
8220 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
8221   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
8222    (match_operand:MVE_2 1 "s_register_operand")
8223    (match_operand:MVE_2 2 "s_register_operand")
8224    (match_operand:HI 3 "vpr_register_operand" "Up")
8225    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
8226   "TARGET_HAVE_MVE"
8227 {
8228   rtx ind = XEXP (operands[0], 0);
8229   gcc_assert (REG_P (ind));
8230   emit_insn (
8231     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8232                                                        operands[2],
8233                                                        operands[3]));
8234   DONE;
8235 })
8236
8237 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
8238   [(set (mem:BLK (scratch))
8239         (unspec:BLK
8240           [(match_operand:SI 0 "register_operand" "r")
8241            (match_operand:MVE_2 1 "s_register_operand" "w")
8242            (match_operand:MVE_2 2 "s_register_operand" "w")
8243            (match_operand:HI 3 "vpr_register_operand" "Up")]
8244           VSTRBSOQ))]
8245   "TARGET_HAVE_MVE"
8246   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
8247   [(set_attr "length" "8")])
8248
8249 ;;
8250 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
8251 ;;
8252 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
8253   [(set (mem:BLK (scratch))
8254         (unspec:BLK
8255                 [(match_operand:V4SI 0 "s_register_operand" "w")
8256                  (match_operand:SI 1 "immediate_operand" "i")
8257                  (match_operand:V4SI 2 "s_register_operand" "w")
8258                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8259          VSTRWSBQ))
8260   ]
8261   "TARGET_HAVE_MVE"
8262 {
8263    rtx ops[3];
8264    ops[0] = operands[0];
8265    ops[1] = operands[1];
8266    ops[2] = operands[2];
8267    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8268    return "";
8269 }
8270   [(set_attr "length" "8")])
8271
8272 ;;
8273 ;; [vstrbq_p_s vstrbq_p_u]
8274 ;;
8275 (define_insn "mve_vstrbq_p_<supf><mode>"
8276   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
8277         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
8278                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8279          VSTRBQ))
8280   ]
8281   "TARGET_HAVE_MVE"
8282 {
8283    rtx ops[2];
8284    int regno = REGNO (operands[1]);
8285    ops[1] = gen_rtx_REG (TImode, regno);
8286    ops[0]  = operands[0];
8287    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
8288    return "";
8289 }
8290   [(set_attr "length" "8")])
8291
8292 ;;
8293 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
8294 ;;
8295 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
8296   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8297         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8298                        (match_operand:MVE_2 2 "s_register_operand" "w")
8299                        (match_operand:HI 3 "vpr_register_operand" "Up")]
8300          VLDRBGOQ))
8301   ]
8302   "TARGET_HAVE_MVE"
8303 {
8304    rtx ops[4];
8305    ops[0] = operands[0];
8306    ops[1] = operands[1];
8307    ops[2] = operands[2];
8308    ops[3] = operands[3];
8309    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8310      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
8311    else
8312      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8313    return "";
8314 }
8315   [(set_attr "length" "8")])
8316
8317 ;;
8318 ;; [vldrbq_z_s vldrbq_z_u]
8319 ;;
8320 (define_insn "mve_vldrbq_z_<supf><mode>"
8321   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8322         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
8323                        (match_operand:HI 2 "vpr_register_operand" "Up")]
8324          VLDRBQ))
8325   ]
8326   "TARGET_HAVE_MVE"
8327 {
8328    rtx ops[2];
8329    int regno = REGNO (operands[0]);
8330    ops[0] = gen_rtx_REG (TImode, regno);
8331    ops[1]  = operands[1];
8332    if (<V_sz_elem> == 8)
8333      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
8334    else
8335      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
8336    return "";
8337 }
8338   [(set_attr "length" "8")])
8339
8340 ;;
8341 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
8342 ;;
8343 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
8344   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8345         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8346                       (match_operand:SI 2 "immediate_operand" "i")
8347                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8348          VLDRWGBQ))
8349   ]
8350   "TARGET_HAVE_MVE"
8351 {
8352    rtx ops[3];
8353    ops[0] = operands[0];
8354    ops[1] = operands[1];
8355    ops[2] = operands[2];
8356    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8357    return "";
8358 }
8359   [(set_attr "length" "8")])
8360
8361 ;;
8362 ;; [vldrhq_f]
8363 ;;
8364 (define_insn "mve_vldrhq_fv8hf"
8365   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8366         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
8367          VLDRHQ_F))
8368   ]
8369   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8370 {
8371    rtx ops[2];
8372    int regno = REGNO (operands[0]);
8373    ops[0] = gen_rtx_REG (TImode, regno);
8374    ops[1]  = operands[1];
8375    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
8376    return "";
8377 }
8378   [(set_attr "length" "4")])
8379
8380 ;;
8381 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
8382 ;;
8383 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
8384   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8385         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8386                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8387         VLDRHGOQ))
8388   ]
8389   "TARGET_HAVE_MVE"
8390 {
8391    rtx ops[3];
8392    ops[0] = operands[0];
8393    ops[1] = operands[1];
8394    ops[2] = operands[2];
8395    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8396      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
8397    else
8398      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8399    return "";
8400 }
8401   [(set_attr "length" "4")])
8402
8403 ;;
8404 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
8405 ;;
8406 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
8407   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8408         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8409                        (match_operand:MVE_6 2 "s_register_operand" "w")
8410                        (match_operand:HI 3 "vpr_register_operand" "Up")
8411         ]VLDRHGOQ))
8412   ]
8413   "TARGET_HAVE_MVE"
8414 {
8415    rtx ops[4];
8416    ops[0] = operands[0];
8417    ops[1] = operands[1];
8418    ops[2] = operands[2];
8419    ops[3] = operands[3];
8420    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8421      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
8422    else
8423      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8424    return "";
8425 }
8426  [(set_attr "length" "8")])
8427
8428 ;;
8429 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
8430 ;;
8431 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
8432   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8433         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8434                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8435         VLDRHGSOQ))
8436   ]
8437   "TARGET_HAVE_MVE"
8438 {
8439    rtx ops[3];
8440    ops[0] = operands[0];
8441    ops[1] = operands[1];
8442    ops[2] = operands[2];
8443       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8444      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8445    else
8446      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8447    return "";
8448 }
8449   [(set_attr "length" "4")])
8450
8451 ;;
8452 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
8453 ;;
8454 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
8455   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8456         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8457                        (match_operand:MVE_6 2 "s_register_operand" "w")
8458                        (match_operand:HI 3 "vpr_register_operand" "Up")
8459         ]VLDRHGSOQ))
8460   ]
8461   "TARGET_HAVE_MVE"
8462 {
8463    rtx ops[4];
8464    ops[0] = operands[0];
8465    ops[1] = operands[1];
8466    ops[2] = operands[2];
8467    ops[3] = operands[3];
8468    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8469      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8470    else
8471      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8472    return "";
8473 }
8474   [(set_attr "length" "8")])
8475
8476 ;;
8477 ;; [vldrhq_s, vldrhq_u]
8478 ;;
8479 (define_insn "mve_vldrhq_<supf><mode>"
8480   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8481         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
8482          VLDRHQ))
8483   ]
8484   "TARGET_HAVE_MVE"
8485 {
8486    rtx ops[2];
8487    int regno = REGNO (operands[0]);
8488    ops[0] = gen_rtx_REG (TImode, regno);
8489    ops[1]  = operands[1];
8490    if (<V_sz_elem> == 16)
8491      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
8492    else
8493      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
8494    return "";
8495 }
8496   [(set_attr "length" "4")])
8497
8498 ;;
8499 ;; [vldrhq_z_f]
8500 ;;
8501 (define_insn "mve_vldrhq_z_fv8hf"
8502   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8503         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
8504         (match_operand:HI 2 "vpr_register_operand" "Up")]
8505          VLDRHQ_F))
8506   ]
8507   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8508 {
8509    rtx ops[2];
8510    int regno = REGNO (operands[0]);
8511    ops[0] = gen_rtx_REG (TImode, regno);
8512    ops[1]  = operands[1];
8513    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
8514    return "";
8515 }
8516   [(set_attr "length" "8")])
8517
8518 ;;
8519 ;; [vldrhq_z_s vldrhq_z_u]
8520 ;;
8521 (define_insn "mve_vldrhq_z_<supf><mode>"
8522   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8523         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
8524         (match_operand:HI 2 "vpr_register_operand" "Up")]
8525          VLDRHQ))
8526   ]
8527   "TARGET_HAVE_MVE"
8528 {
8529    rtx ops[2];
8530    int regno = REGNO (operands[0]);
8531    ops[0] = gen_rtx_REG (TImode, regno);
8532    ops[1]  = operands[1];
8533    if (<V_sz_elem> == 16)
8534      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
8535    else
8536      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8537    return "";
8538 }
8539   [(set_attr "length" "8")])
8540
8541 ;;
8542 ;; [vldrwq_f]
8543 ;;
8544 (define_insn "mve_vldrwq_fv4sf"
8545   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8546         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
8547          VLDRWQ_F))
8548   ]
8549   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8550 {
8551    rtx ops[2];
8552    int regno = REGNO (operands[0]);
8553    ops[0] = gen_rtx_REG (TImode, regno);
8554    ops[1]  = operands[1];
8555    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8556    return "";
8557 }
8558   [(set_attr "length" "4")])
8559
8560 ;;
8561 ;; [vldrwq_s vldrwq_u]
8562 ;;
8563 (define_insn "mve_vldrwq_<supf>v4si"
8564   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8565         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
8566          VLDRWQ))
8567   ]
8568   "TARGET_HAVE_MVE"
8569 {
8570    rtx ops[2];
8571    int regno = REGNO (operands[0]);
8572    ops[0] = gen_rtx_REG (TImode, regno);
8573    ops[1]  = operands[1];
8574    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8575    return "";
8576 }
8577   [(set_attr "length" "4")])
8578
8579 ;;
8580 ;; [vldrwq_z_f]
8581 ;;
8582 (define_insn "mve_vldrwq_z_fv4sf"
8583   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8584         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
8585         (match_operand:HI 2 "vpr_register_operand" "Up")]
8586          VLDRWQ_F))
8587   ]
8588   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8589 {
8590    rtx ops[2];
8591    int regno = REGNO (operands[0]);
8592    ops[0] = gen_rtx_REG (TImode, regno);
8593    ops[1]  = operands[1];
8594    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8595    return "";
8596 }
8597   [(set_attr "length" "8")])
8598
8599 ;;
8600 ;; [vldrwq_z_s vldrwq_z_u]
8601 ;;
8602 (define_insn "mve_vldrwq_z_<supf>v4si"
8603   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8604         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
8605         (match_operand:HI 2 "vpr_register_operand" "Up")]
8606          VLDRWQ))
8607   ]
8608   "TARGET_HAVE_MVE"
8609 {
8610    rtx ops[2];
8611    int regno = REGNO (operands[0]);
8612    ops[0] = gen_rtx_REG (TImode, regno);
8613    ops[1]  = operands[1];
8614    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8615    return "";
8616 }
8617   [(set_attr "length" "8")])
8618
8619 (define_expand "mve_vld1q_f<mode>"
8620   [(match_operand:MVE_0 0 "s_register_operand")
8621    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
8622   ]
8623   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8624 {
8625   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8626   DONE;
8627 })
8628
8629 (define_expand "mve_vld1q_<supf><mode>"
8630   [(match_operand:MVE_2 0 "s_register_operand")
8631    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
8632   ]
8633   "TARGET_HAVE_MVE"
8634 {
8635   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8636   DONE;
8637 })
8638
8639 ;;
8640 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8641 ;;
8642 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8643   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8644         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8645                       (match_operand:SI 2 "immediate_operand" "i")]
8646          VLDRDGBQ))
8647   ]
8648   "TARGET_HAVE_MVE"
8649 {
8650    rtx ops[3];
8651    ops[0] = operands[0];
8652    ops[1] = operands[1];
8653    ops[2] = operands[2];
8654    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8655    return "";
8656 }
8657   [(set_attr "length" "4")])
8658
8659 ;;
8660 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8661 ;;
8662 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8663   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8664         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8665                       (match_operand:SI 2 "immediate_operand" "i")
8666                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8667          VLDRDGBQ))
8668   ]
8669   "TARGET_HAVE_MVE"
8670 {
8671    rtx ops[3];
8672    ops[0] = operands[0];
8673    ops[1] = operands[1];
8674    ops[2] = operands[2];
8675    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8676    return "";
8677 }
8678   [(set_attr "length" "8")])
8679
8680 ;;
8681 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8682 ;;
8683 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8684  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8685        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8686                      (match_operand:V2DI 2 "s_register_operand" "w")]
8687         VLDRDGOQ))
8688  ]
8689  "TARGET_HAVE_MVE"
8690 {
8691   rtx ops[3];
8692   ops[0] = operands[0];
8693   ops[1] = operands[1];
8694   ops[2] = operands[2];
8695   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8696   return "";
8697 }
8698  [(set_attr "length" "4")])
8699
8700 ;;
8701 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8702 ;;
8703 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8704  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8705        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8706                      (match_operand:V2DI 2 "s_register_operand" "w")
8707                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8708         VLDRDGOQ))
8709  ]
8710  "TARGET_HAVE_MVE"
8711 {
8712   rtx ops[3];
8713   ops[0] = operands[0];
8714   ops[1] = operands[1];
8715   ops[2] = operands[2];
8716   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8717   return "";
8718 }
8719  [(set_attr "length" "8")])
8720
8721 ;;
8722 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8723 ;;
8724 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8725   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8726         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8727                       (match_operand:V2DI 2 "s_register_operand" "w")]
8728          VLDRDGSOQ))
8729   ]
8730   "TARGET_HAVE_MVE"
8731 {
8732    rtx ops[3];
8733    ops[0] = operands[0];
8734    ops[1] = operands[1];
8735    ops[2] = operands[2];
8736    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8737    return "";
8738 }
8739   [(set_attr "length" "4")])
8740
8741 ;;
8742 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8743 ;;
8744 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8745   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8746         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8747                       (match_operand:V2DI 2 "s_register_operand" "w")
8748                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8749          VLDRDGSOQ))
8750   ]
8751   "TARGET_HAVE_MVE"
8752 {
8753    rtx ops[3];
8754    ops[0] = operands[0];
8755    ops[1] = operands[1];
8756    ops[2] = operands[2];
8757    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8758    return "";
8759 }
8760   [(set_attr "length" "8")])
8761
8762 ;;
8763 ;; [vldrhq_gather_offset_f]
8764 ;;
8765 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8766   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8767         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8768                       (match_operand:V8HI 2 "s_register_operand" "w")]
8769          VLDRHQGO_F))
8770   ]
8771   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8772 {
8773    rtx ops[3];
8774    ops[0] = operands[0];
8775    ops[1] = operands[1];
8776    ops[2] = operands[2];
8777    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8778    return "";
8779 }
8780   [(set_attr "length" "4")])
8781
8782 ;;
8783 ;; [vldrhq_gather_offset_z_f]
8784 ;;
8785 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8786   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8787         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8788                       (match_operand:V8HI 2 "s_register_operand" "w")
8789                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8790          VLDRHQGO_F))
8791   ]
8792   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8793 {
8794    rtx ops[4];
8795    ops[0] = operands[0];
8796    ops[1] = operands[1];
8797    ops[2] = operands[2];
8798    ops[3] = operands[3];
8799    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8800    return "";
8801 }
8802   [(set_attr "length" "8")])
8803
8804 ;;
8805 ;; [vldrhq_gather_shifted_offset_f]
8806 ;;
8807 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8808   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8809         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8810                       (match_operand:V8HI 2 "s_register_operand" "w")]
8811          VLDRHQGSO_F))
8812   ]
8813   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8814 {
8815    rtx ops[3];
8816    ops[0] = operands[0];
8817    ops[1] = operands[1];
8818    ops[2] = operands[2];
8819    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8820    return "";
8821 }
8822   [(set_attr "length" "4")])
8823
8824 ;;
8825 ;; [vldrhq_gather_shifted_offset_z_f]
8826 ;;
8827 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8828   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8829         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8830                       (match_operand:V8HI 2 "s_register_operand" "w")
8831                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8832          VLDRHQGSO_F))
8833   ]
8834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8835 {
8836    rtx ops[4];
8837    ops[0] = operands[0];
8838    ops[1] = operands[1];
8839    ops[2] = operands[2];
8840    ops[3] = operands[3];
8841    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8842    return "";
8843 }
8844   [(set_attr "length" "8")])
8845
8846 ;;
8847 ;; [vldrwq_gather_base_f]
8848 ;;
8849 (define_insn "mve_vldrwq_gather_base_fv4sf"
8850   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8851         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8852                       (match_operand:SI 2 "immediate_operand" "i")]
8853          VLDRWQGB_F))
8854   ]
8855   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8856 {
8857    rtx ops[3];
8858    ops[0] = operands[0];
8859    ops[1] = operands[1];
8860    ops[2] = operands[2];
8861    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8862    return "";
8863 }
8864   [(set_attr "length" "4")])
8865
8866 ;;
8867 ;; [vldrwq_gather_base_z_f]
8868 ;;
8869 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8870   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8871         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8872                       (match_operand:SI 2 "immediate_operand" "i")
8873                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8874          VLDRWQGB_F))
8875   ]
8876   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8877 {
8878    rtx ops[3];
8879    ops[0] = operands[0];
8880    ops[1] = operands[1];
8881    ops[2] = operands[2];
8882    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8883    return "";
8884 }
8885   [(set_attr "length" "8")])
8886
8887 ;;
8888 ;; [vldrwq_gather_offset_f]
8889 ;;
8890 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8891   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8892         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8893                        (match_operand:V4SI 2 "s_register_operand" "w")]
8894          VLDRWQGO_F))
8895   ]
8896   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8897 {
8898    rtx ops[3];
8899    ops[0] = operands[0];
8900    ops[1] = operands[1];
8901    ops[2] = operands[2];
8902    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8903    return "";
8904 }
8905   [(set_attr "length" "4")])
8906
8907 ;;
8908 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8909 ;;
8910 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8911   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8912         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8913                        (match_operand:V4SI 2 "s_register_operand" "w")]
8914          VLDRWGOQ))
8915   ]
8916   "TARGET_HAVE_MVE"
8917 {
8918    rtx ops[3];
8919    ops[0] = operands[0];
8920    ops[1] = operands[1];
8921    ops[2] = operands[2];
8922    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8923    return "";
8924 }
8925   [(set_attr "length" "4")])
8926
8927 ;;
8928 ;; [vldrwq_gather_offset_z_f]
8929 ;;
8930 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8931   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8932         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8933                       (match_operand:V4SI 2 "s_register_operand" "w")
8934                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8935          VLDRWQGO_F))
8936   ]
8937   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8938 {
8939    rtx ops[4];
8940    ops[0] = operands[0];
8941    ops[1] = operands[1];
8942    ops[2] = operands[2];
8943    ops[3] = operands[3];
8944    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8945    return "";
8946 }
8947   [(set_attr "length" "8")])
8948
8949 ;;
8950 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8951 ;;
8952 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8953   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8954         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8955                       (match_operand:V4SI 2 "s_register_operand" "w")
8956                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8957          VLDRWGOQ))
8958   ]
8959   "TARGET_HAVE_MVE"
8960 {
8961    rtx ops[4];
8962    ops[0] = operands[0];
8963    ops[1] = operands[1];
8964    ops[2] = operands[2];
8965    ops[3] = operands[3];
8966    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8967    return "";
8968 }
8969   [(set_attr "length" "8")])
8970
8971 ;;
8972 ;; [vldrwq_gather_shifted_offset_f]
8973 ;;
8974 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8975   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8976         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8977                       (match_operand:V4SI 2 "s_register_operand" "w")]
8978          VLDRWQGSO_F))
8979   ]
8980   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8981 {
8982    rtx ops[3];
8983    ops[0] = operands[0];
8984    ops[1] = operands[1];
8985    ops[2] = operands[2];
8986    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8987    return "";
8988 }
8989   [(set_attr "length" "4")])
8990
8991 ;;
8992 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8993 ;;
8994 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8995   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8996         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8997                       (match_operand:V4SI 2 "s_register_operand" "w")]
8998          VLDRWGSOQ))
8999   ]
9000   "TARGET_HAVE_MVE"
9001 {
9002    rtx ops[3];
9003    ops[0] = operands[0];
9004    ops[1] = operands[1];
9005    ops[2] = operands[2];
9006    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
9007    return "";
9008 }
9009   [(set_attr "length" "4")])
9010
9011 ;;
9012 ;; [vldrwq_gather_shifted_offset_z_f]
9013 ;;
9014 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
9015   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9016         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
9017                       (match_operand:V4SI 2 "s_register_operand" "w")
9018                       (match_operand:HI 3 "vpr_register_operand" "Up")]
9019          VLDRWQGSO_F))
9020   ]
9021   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9022 {
9023    rtx ops[4];
9024    ops[0] = operands[0];
9025    ops[1] = operands[1];
9026    ops[2] = operands[2];
9027    ops[3] = operands[3];
9028    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
9029    return "";
9030 }
9031   [(set_attr "length" "8")])
9032
9033 ;;
9034 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
9035 ;;
9036 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
9037   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9038         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
9039                       (match_operand:V4SI 2 "s_register_operand" "w")
9040                       (match_operand:HI 3 "vpr_register_operand" "Up")]
9041          VLDRWGSOQ))
9042   ]
9043   "TARGET_HAVE_MVE"
9044 {
9045    rtx ops[4];
9046    ops[0] = operands[0];
9047    ops[1] = operands[1];
9048    ops[2] = operands[2];
9049    ops[3] = operands[3];
9050    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
9051    return "";
9052 }
9053   [(set_attr "length" "8")])
9054
9055 ;;
9056 ;; [vstrhq_f]
9057 ;;
9058 (define_insn "mve_vstrhq_fv8hf"
9059   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
9060         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
9061          VSTRHQ_F))
9062   ]
9063   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9064 {
9065    rtx ops[2];
9066    int regno = REGNO (operands[1]);
9067    ops[1] = gen_rtx_REG (TImode, regno);
9068    ops[0]  = operands[0];
9069    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
9070    return "";
9071 }
9072   [(set_attr "length" "4")])
9073
9074 ;;
9075 ;; [vstrhq_p_f]
9076 ;;
9077 (define_insn "mve_vstrhq_p_fv8hf"
9078   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
9079         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
9080                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9081          VSTRHQ_F))
9082   ]
9083   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9084 {
9085    rtx ops[2];
9086    int regno = REGNO (operands[1]);
9087    ops[1] = gen_rtx_REG (TImode, regno);
9088    ops[0]  = operands[0];
9089    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
9090    return "";
9091 }
9092   [(set_attr "length" "8")])
9093
9094 ;;
9095 ;; [vstrhq_p_s vstrhq_p_u]
9096 ;;
9097 (define_insn "mve_vstrhq_p_<supf><mode>"
9098   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
9099         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
9100                               (match_operand:HI 2 "vpr_register_operand" "Up")]
9101          VSTRHQ))
9102   ]
9103   "TARGET_HAVE_MVE"
9104 {
9105    rtx ops[2];
9106    int regno = REGNO (operands[1]);
9107    ops[1] = gen_rtx_REG (TImode, regno);
9108    ops[0]  = operands[0];
9109    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
9110    return "";
9111 }
9112   [(set_attr "length" "8")])
9113
9114 ;;
9115 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
9116 ;;
9117 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
9118   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
9119    (match_operand:MVE_6 1 "s_register_operand")
9120    (match_operand:MVE_6 2 "s_register_operand")
9121    (match_operand:HI 3 "vpr_register_operand")
9122    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
9123   "TARGET_HAVE_MVE"
9124 {
9125   rtx ind = XEXP (operands[0], 0);
9126   gcc_assert (REG_P (ind));
9127   emit_insn (
9128     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
9129                                                        operands[2],
9130                                                        operands[3]));
9131   DONE;
9132 })
9133
9134 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
9135   [(set (mem:BLK (scratch))
9136         (unspec:BLK
9137           [(match_operand:SI 0 "register_operand" "r")
9138            (match_operand:MVE_6 1 "s_register_operand" "w")
9139            (match_operand:MVE_6 2 "s_register_operand" "w")
9140            (match_operand:HI 3 "vpr_register_operand" "Up")]
9141           VSTRHSOQ))]
9142   "TARGET_HAVE_MVE"
9143   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
9144   [(set_attr "length" "8")])
9145
9146 ;;
9147 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
9148 ;;
9149 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
9150   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
9151    (match_operand:MVE_6 1 "s_register_operand")
9152    (match_operand:MVE_6 2 "s_register_operand")
9153    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
9154   "TARGET_HAVE_MVE"
9155 {
9156   rtx ind = XEXP (operands[0], 0);
9157   gcc_assert (REG_P (ind));
9158   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
9159                                                               operands[2]));
9160   DONE;
9161 })
9162
9163 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
9164   [(set (mem:BLK (scratch))
9165         (unspec:BLK
9166           [(match_operand:SI 0 "register_operand" "r")
9167            (match_operand:MVE_6 1 "s_register_operand" "w")
9168            (match_operand:MVE_6 2 "s_register_operand" "w")]
9169           VSTRHSOQ))]
9170   "TARGET_HAVE_MVE"
9171   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
9172   [(set_attr "length" "4")])
9173
9174 ;;
9175 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
9176 ;;
9177 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
9178   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
9179    (match_operand:MVE_6 1 "s_register_operand")
9180    (match_operand:MVE_6 2 "s_register_operand")
9181    (match_operand:HI 3 "vpr_register_operand")
9182    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
9183   "TARGET_HAVE_MVE"
9184 {
9185   rtx ind = XEXP (operands[0], 0);
9186   gcc_assert (REG_P (ind));
9187   emit_insn (
9188     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
9189                                                                operands[2],
9190                                                                operands[3]));
9191   DONE;
9192 })
9193
9194 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
9195   [(set (mem:BLK (scratch))
9196         (unspec:BLK
9197           [(match_operand:SI 0 "register_operand" "r")
9198            (match_operand:MVE_6 1 "s_register_operand" "w")
9199            (match_operand:MVE_6 2 "s_register_operand" "w")
9200            (match_operand:HI 3 "vpr_register_operand" "Up")]
9201           VSTRHSSOQ))]
9202   "TARGET_HAVE_MVE"
9203   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
9204   [(set_attr "length" "8")])
9205
9206 ;;
9207 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
9208 ;;
9209 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
9210   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
9211    (match_operand:MVE_6 1 "s_register_operand")
9212    (match_operand:MVE_6 2 "s_register_operand")
9213    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
9214   "TARGET_HAVE_MVE"
9215 {
9216   rtx ind = XEXP (operands[0], 0);
9217   gcc_assert (REG_P (ind));
9218   emit_insn (
9219     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
9220                                                              operands[2]));
9221   DONE;
9222 })
9223
9224 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
9225   [(set (mem:BLK (scratch))
9226         (unspec:BLK
9227           [(match_operand:SI 0 "register_operand" "r")
9228            (match_operand:MVE_6 1 "s_register_operand" "w")
9229            (match_operand:MVE_6 2 "s_register_operand" "w")]
9230           VSTRHSSOQ))]
9231   "TARGET_HAVE_MVE"
9232   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
9233   [(set_attr "length" "4")])
9234
9235 ;;
9236 ;; [vstrhq_s, vstrhq_u]
9237 ;;
9238 (define_insn "mve_vstrhq_<supf><mode>"
9239   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
9240         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
9241          VSTRHQ))
9242   ]
9243   "TARGET_HAVE_MVE"
9244 {
9245    rtx ops[2];
9246    int regno = REGNO (operands[1]);
9247    ops[1] = gen_rtx_REG (TImode, regno);
9248    ops[0]  = operands[0];
9249    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
9250    return "";
9251 }
9252   [(set_attr "length" "4")])
9253
9254 ;;
9255 ;; [vstrwq_f]
9256 ;;
9257 (define_insn "mve_vstrwq_fv4sf"
9258   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
9259         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
9260          VSTRWQ_F))
9261   ]
9262   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9263 {
9264    rtx ops[2];
9265    int regno = REGNO (operands[1]);
9266    ops[1] = gen_rtx_REG (TImode, regno);
9267    ops[0]  = operands[0];
9268    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9269    return "";
9270 }
9271   [(set_attr "length" "4")])
9272
9273 ;;
9274 ;; [vstrwq_p_f]
9275 ;;
9276 (define_insn "mve_vstrwq_p_fv4sf"
9277   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
9278         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
9279                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9280          VSTRWQ_F))
9281   ]
9282   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9283 {
9284    rtx ops[2];
9285    int regno = REGNO (operands[1]);
9286    ops[1] = gen_rtx_REG (TImode, regno);
9287    ops[0]  = operands[0];
9288    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
9289    return "";
9290 }
9291   [(set_attr "length" "8")])
9292
9293 ;;
9294 ;; [vstrwq_p_s vstrwq_p_u]
9295 ;;
9296 (define_insn "mve_vstrwq_p_<supf>v4si"
9297   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
9298         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9299                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9300          VSTRWQ))
9301   ]
9302   "TARGET_HAVE_MVE"
9303 {
9304    rtx ops[2];
9305    int regno = REGNO (operands[1]);
9306    ops[1] = gen_rtx_REG (TImode, regno);
9307    ops[0]  = operands[0];
9308    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
9309    return "";
9310 }
9311   [(set_attr "length" "8")])
9312
9313 ;;
9314 ;; [vstrwq_s vstrwq_u]
9315 ;;
9316 (define_insn "mve_vstrwq_<supf>v4si"
9317   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
9318         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
9319          VSTRWQ))
9320   ]
9321   "TARGET_HAVE_MVE"
9322 {
9323    rtx ops[2];
9324    int regno = REGNO (operands[1]);
9325    ops[1] = gen_rtx_REG (TImode, regno);
9326    ops[0]  = operands[0];
9327    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9328    return "";
9329 }
9330   [(set_attr "length" "4")])
9331
9332 (define_expand "mve_vst1q_f<mode>"
9333   [(match_operand:<MVE_CNVT> 0 "memory_operand")
9334    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
9335   ]
9336   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
9337 {
9338   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
9339   DONE;
9340 })
9341
9342 (define_expand "mve_vst1q_<supf><mode>"
9343   [(match_operand:MVE_2 0 "memory_operand")
9344    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
9345   ]
9346   "TARGET_HAVE_MVE"
9347 {
9348   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
9349   DONE;
9350 })
9351
9352 ;;
9353 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
9354 ;;
9355 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
9356   [(set (mem:BLK (scratch))
9357         (unspec:BLK
9358                 [(match_operand:V2DI 0 "s_register_operand" "w")
9359                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9360                  (match_operand:V2DI 2 "s_register_operand" "w")
9361                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9362          VSTRDSBQ))
9363   ]
9364   "TARGET_HAVE_MVE"
9365 {
9366    rtx ops[3];
9367    ops[0] = operands[0];
9368    ops[1] = operands[1];
9369    ops[2] = operands[2];
9370    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
9371    return "";
9372 }
9373   [(set_attr "length" "8")])
9374
9375 ;;
9376 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
9377 ;;
9378 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
9379   [(set (mem:BLK (scratch))
9380         (unspec:BLK
9381                 [(match_operand:V2DI 0 "s_register_operand" "=w")
9382                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9383                  (match_operand:V2DI 2 "s_register_operand" "w")]
9384          VSTRDSBQ))
9385   ]
9386   "TARGET_HAVE_MVE"
9387 {
9388    rtx ops[3];
9389    ops[0] = operands[0];
9390    ops[1] = operands[1];
9391    ops[2] = operands[2];
9392    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
9393    return "";
9394 }
9395   [(set_attr "length" "4")])
9396
9397 ;;
9398 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
9399 ;;
9400 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
9401   [(match_operand:V2DI 0 "mve_scatter_memory")
9402    (match_operand:V2DI 1 "s_register_operand")
9403    (match_operand:V2DI 2 "s_register_operand")
9404    (match_operand:HI 3 "vpr_register_operand")
9405    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
9406   "TARGET_HAVE_MVE"
9407 {
9408   rtx ind = XEXP (operands[0], 0);
9409   gcc_assert (REG_P (ind));
9410   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
9411                                                               operands[2],
9412                                                               operands[3]));
9413   DONE;
9414 })
9415
9416 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
9417   [(set (mem:BLK (scratch))
9418         (unspec:BLK
9419           [(match_operand:SI 0 "register_operand" "r")
9420            (match_operand:V2DI 1 "s_register_operand" "w")
9421            (match_operand:V2DI 2 "s_register_operand" "w")
9422            (match_operand:HI 3 "vpr_register_operand" "Up")]
9423           VSTRDSOQ))]
9424   "TARGET_HAVE_MVE"
9425   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
9426   [(set_attr "length" "8")])
9427
9428 ;;
9429 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
9430 ;;
9431 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
9432   [(match_operand:V2DI 0 "mve_scatter_memory")
9433    (match_operand:V2DI 1 "s_register_operand")
9434    (match_operand:V2DI 2 "s_register_operand")
9435    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
9436   "TARGET_HAVE_MVE"
9437 {
9438   rtx ind = XEXP (operands[0], 0);
9439   gcc_assert (REG_P (ind));
9440   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
9441                                                             operands[2]));
9442   DONE;
9443 })
9444
9445 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
9446   [(set (mem:BLK (scratch))
9447         (unspec:BLK
9448           [(match_operand:SI 0 "register_operand" "r")
9449            (match_operand:V2DI 1 "s_register_operand" "w")
9450            (match_operand:V2DI 2 "s_register_operand" "w")]
9451           VSTRDSOQ))]
9452   "TARGET_HAVE_MVE"
9453   "vstrd.64\t%q2, [%0, %q1]"
9454   [(set_attr "length" "4")])
9455
9456 ;;
9457 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
9458 ;;
9459 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
9460   [(match_operand:V2DI 0 "mve_scatter_memory")
9461    (match_operand:V2DI 1 "s_register_operand")
9462    (match_operand:V2DI 2 "s_register_operand")
9463    (match_operand:HI 3 "vpr_register_operand")
9464    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
9465   "TARGET_HAVE_MVE"
9466 {
9467   rtx ind = XEXP (operands[0], 0);
9468   gcc_assert (REG_P (ind));
9469   emit_insn (
9470     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
9471                                                              operands[2],
9472                                                              operands[3]));
9473   DONE;
9474 })
9475
9476 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
9477   [(set (mem:BLK (scratch))
9478         (unspec:BLK
9479           [(match_operand:SI 0 "register_operand" "r")
9480            (match_operand:V2DI 1 "s_register_operand" "w")
9481            (match_operand:V2DI 2 "s_register_operand" "w")
9482            (match_operand:HI 3 "vpr_register_operand" "Up")]
9483           VSTRDSSOQ))]
9484   "TARGET_HAVE_MVE"
9485   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
9486   [(set_attr "length" "8")])
9487
9488 ;;
9489 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
9490 ;;
9491 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
9492   [(match_operand:V2DI 0 "mve_scatter_memory")
9493    (match_operand:V2DI 1 "s_register_operand")
9494    (match_operand:V2DI 2 "s_register_operand")
9495    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
9496   "TARGET_HAVE_MVE"
9497 {
9498   rtx ind = XEXP (operands[0], 0);
9499   gcc_assert (REG_P (ind));
9500   emit_insn (
9501     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
9502                                                            operands[2]));
9503   DONE;
9504 })
9505
9506 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
9507   [(set (mem:BLK (scratch))
9508         (unspec:BLK
9509           [(match_operand:SI 0 "register_operand" "r")
9510            (match_operand:V2DI 1 "s_register_operand" "w")
9511            (match_operand:V2DI 2 "s_register_operand" "w")]
9512           VSTRDSSOQ))]
9513   "TARGET_HAVE_MVE"
9514   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
9515   [(set_attr "length" "4")])
9516
9517 ;;
9518 ;; [vstrhq_scatter_offset_f]
9519 ;;
9520 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
9521   [(match_operand:V8HI 0 "mve_scatter_memory")
9522    (match_operand:V8HI 1 "s_register_operand")
9523    (match_operand:V8HF 2 "s_register_operand")
9524    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
9525   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9526 {
9527   rtx ind = XEXP (operands[0], 0);
9528   gcc_assert (REG_P (ind));
9529   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
9530                                                        operands[2]));
9531   DONE;
9532 })
9533
9534 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
9535   [(set (mem:BLK (scratch))
9536         (unspec:BLK
9537           [(match_operand:SI 0 "register_operand" "r")
9538            (match_operand:V8HI 1 "s_register_operand" "w")
9539            (match_operand:V8HF 2 "s_register_operand" "w")]
9540           VSTRHQSO_F))]
9541   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9542   "vstrh.16\t%q2, [%0, %q1]"
9543   [(set_attr "length" "4")])
9544
9545 ;;
9546 ;; [vstrhq_scatter_offset_p_f]
9547 ;;
9548 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
9549   [(match_operand:V8HI 0 "mve_scatter_memory")
9550    (match_operand:V8HI 1 "s_register_operand")
9551    (match_operand:V8HF 2 "s_register_operand")
9552    (match_operand:HI 3 "vpr_register_operand")
9553    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
9554   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9555 {
9556   rtx ind = XEXP (operands[0], 0);
9557   gcc_assert (REG_P (ind));
9558   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
9559                                                          operands[2],
9560                                                          operands[3]));
9561   DONE;
9562 })
9563
9564 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
9565   [(set (mem:BLK (scratch))
9566         (unspec:BLK
9567           [(match_operand:SI 0 "register_operand" "r")
9568            (match_operand:V8HI 1 "s_register_operand" "w")
9569            (match_operand:V8HF 2 "s_register_operand" "w")
9570            (match_operand:HI 3 "vpr_register_operand" "Up")]
9571           VSTRHQSO_F))]
9572   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9573   "vpst\;vstrht.16\t%q2, [%0, %q1]"
9574   [(set_attr "length" "8")])
9575
9576 ;;
9577 ;; [vstrhq_scatter_shifted_offset_f]
9578 ;;
9579 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
9580   [(match_operand:V8HI 0 "memory_operand" "=Us")
9581    (match_operand:V8HI 1 "s_register_operand" "w")
9582    (match_operand:V8HF 2 "s_register_operand" "w")
9583    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9584   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9585 {
9586   rtx ind = XEXP (operands[0], 0);
9587   gcc_assert (REG_P (ind));
9588   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
9589                                                                operands[2]));
9590   DONE;
9591 })
9592
9593 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
9594   [(set (mem:BLK (scratch))
9595         (unspec:BLK
9596           [(match_operand:SI 0 "register_operand" "r")
9597            (match_operand:V8HI 1 "s_register_operand" "w")
9598            (match_operand:V8HF 2 "s_register_operand" "w")]
9599           VSTRHQSSO_F))]
9600   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9601   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
9602   [(set_attr "length" "4")])
9603
9604 ;;
9605 ;; [vstrhq_scatter_shifted_offset_p_f]
9606 ;;
9607 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9608   [(match_operand:V8HI 0 "memory_operand" "=Us")
9609    (match_operand:V8HI 1 "s_register_operand" "w")
9610    (match_operand:V8HF 2 "s_register_operand" "w")
9611    (match_operand:HI 3 "vpr_register_operand" "Up")
9612    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9613   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9614 {
9615   rtx ind = XEXP (operands[0], 0);
9616   gcc_assert (REG_P (ind));
9617   emit_insn (
9618     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
9619                                                         operands[2],
9620                                                         operands[3]));
9621   DONE;
9622 })
9623
9624 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
9625   [(set (mem:BLK (scratch))
9626         (unspec:BLK
9627           [(match_operand:SI 0 "register_operand" "r")
9628            (match_operand:V8HI 1 "s_register_operand" "w")
9629            (match_operand:V8HF 2 "s_register_operand" "w")
9630            (match_operand:HI 3 "vpr_register_operand" "Up")]
9631           VSTRHQSSO_F))]
9632   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9633   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
9634   [(set_attr "length" "8")])
9635
9636 ;;
9637 ;; [vstrwq_scatter_base_f]
9638 ;;
9639 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9640   [(set (mem:BLK (scratch))
9641         (unspec:BLK
9642                 [(match_operand:V4SI 0 "s_register_operand" "w")
9643                  (match_operand:SI 1 "immediate_operand" "i")
9644                  (match_operand:V4SF 2 "s_register_operand" "w")]
9645          VSTRWQSB_F))
9646   ]
9647   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9648 {
9649    rtx ops[3];
9650    ops[0] = operands[0];
9651    ops[1] = operands[1];
9652    ops[2] = operands[2];
9653    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9654    return "";
9655 }
9656   [(set_attr "length" "4")])
9657
9658 ;;
9659 ;; [vstrwq_scatter_base_p_f]
9660 ;;
9661 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9662   [(set (mem:BLK (scratch))
9663         (unspec:BLK
9664                 [(match_operand:V4SI 0 "s_register_operand" "w")
9665                  (match_operand:SI 1 "immediate_operand" "i")
9666                  (match_operand:V4SF 2 "s_register_operand" "w")
9667                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9668          VSTRWQSB_F))
9669   ]
9670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9671 {
9672    rtx ops[3];
9673    ops[0] = operands[0];
9674    ops[1] = operands[1];
9675    ops[2] = operands[2];
9676    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9677    return "";
9678 }
9679   [(set_attr "length" "8")])
9680
9681 ;;
9682 ;; [vstrwq_scatter_offset_f]
9683 ;;
9684 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9685   [(match_operand:V4SI 0 "mve_scatter_memory")
9686    (match_operand:V4SI 1 "s_register_operand")
9687    (match_operand:V4SF 2 "s_register_operand")
9688    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9689   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9690 {
9691   rtx ind = XEXP (operands[0], 0);
9692   gcc_assert (REG_P (ind));
9693   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9694                                                        operands[2]));
9695   DONE;
9696 })
9697
9698 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9699   [(set (mem:BLK (scratch))
9700         (unspec:BLK
9701           [(match_operand:SI 0 "register_operand" "r")
9702            (match_operand:V4SI 1 "s_register_operand" "w")
9703            (match_operand:V4SF 2 "s_register_operand" "w")]
9704           VSTRWQSO_F))]
9705   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9706   "vstrw.32\t%q2, [%0, %q1]"
9707   [(set_attr "length" "4")])
9708
9709 ;;
9710 ;; [vstrwq_scatter_offset_p_f]
9711 ;;
9712 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9713   [(match_operand:V4SI 0 "mve_scatter_memory")
9714    (match_operand:V4SI 1 "s_register_operand")
9715    (match_operand:V4SF 2 "s_register_operand")
9716    (match_operand:HI 3 "vpr_register_operand")
9717    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9718   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9719 {
9720   rtx ind = XEXP (operands[0], 0);
9721   gcc_assert (REG_P (ind));
9722   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9723                                                          operands[2],
9724                                                          operands[3]));
9725   DONE;
9726 })
9727
9728 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9729   [(set (mem:BLK (scratch))
9730         (unspec:BLK
9731           [(match_operand:SI 0 "register_operand" "r")
9732            (match_operand:V4SI 1 "s_register_operand" "w")
9733            (match_operand:V4SF 2 "s_register_operand" "w")
9734            (match_operand:HI 3 "vpr_register_operand" "Up")]
9735           VSTRWQSO_F))]
9736   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9737   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9738   [(set_attr "length" "8")])
9739
9740 ;;
9741 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9742 ;;
9743 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9744   [(match_operand:V4SI 0 "mve_scatter_memory")
9745    (match_operand:V4SI 1 "s_register_operand")
9746    (match_operand:V4SI 2 "s_register_operand")
9747    (match_operand:HI 3 "vpr_register_operand")
9748    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9749   "TARGET_HAVE_MVE"
9750 {
9751   rtx ind = XEXP (operands[0], 0);
9752   gcc_assert (REG_P (ind));
9753   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9754                                                               operands[2],
9755                                                               operands[3]));
9756   DONE;
9757 })
9758
9759 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9760   [(set (mem:BLK (scratch))
9761         (unspec:BLK
9762           [(match_operand:SI 0 "register_operand" "r")
9763            (match_operand:V4SI 1 "s_register_operand" "w")
9764            (match_operand:V4SI 2 "s_register_operand" "w")
9765            (match_operand:HI 3 "vpr_register_operand" "Up")]
9766           VSTRWSOQ))]
9767   "TARGET_HAVE_MVE"
9768   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9769   [(set_attr "length" "8")])
9770
9771 ;;
9772 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9773 ;;
9774 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9775   [(match_operand:V4SI 0 "mve_scatter_memory")
9776    (match_operand:V4SI 1 "s_register_operand")
9777    (match_operand:V4SI 2 "s_register_operand")
9778    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9779   "TARGET_HAVE_MVE"
9780 {
9781   rtx ind = XEXP (operands[0], 0);
9782   gcc_assert (REG_P (ind));
9783   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9784                                                             operands[2]));
9785   DONE;
9786 })
9787
9788 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9789   [(set (mem:BLK (scratch))
9790         (unspec:BLK
9791           [(match_operand:SI 0 "register_operand" "r")
9792            (match_operand:V4SI 1 "s_register_operand" "w")
9793            (match_operand:V4SI 2 "s_register_operand" "w")]
9794           VSTRWSOQ))]
9795   "TARGET_HAVE_MVE"
9796   "vstrw.32\t%q2, [%0, %q1]"
9797   [(set_attr "length" "4")])
9798
9799 ;;
9800 ;; [vstrwq_scatter_shifted_offset_f]
9801 ;;
9802 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9803   [(match_operand:V4SI 0 "mve_scatter_memory")
9804    (match_operand:V4SI 1 "s_register_operand")
9805    (match_operand:V4SF 2 "s_register_operand")
9806    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9807   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9808 {
9809   rtx ind = XEXP (operands[0], 0);
9810   gcc_assert (REG_P (ind));
9811   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9812                                                                operands[2]));
9813   DONE;
9814 })
9815
9816 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9817   [(set (mem:BLK (scratch))
9818         (unspec:BLK
9819           [(match_operand:SI 0 "register_operand" "r")
9820            (match_operand:V4SI 1 "s_register_operand" "w")
9821            (match_operand:V4SF 2 "s_register_operand" "w")]
9822          VSTRWQSSO_F))]
9823   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9824   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9825   [(set_attr "length" "8")])
9826
9827 ;;
9828 ;; [vstrwq_scatter_shifted_offset_p_f]
9829 ;;
9830 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9831   [(match_operand:V4SI 0 "mve_scatter_memory")
9832    (match_operand:V4SI 1 "s_register_operand")
9833    (match_operand:V4SF 2 "s_register_operand")
9834    (match_operand:HI 3 "vpr_register_operand")
9835    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9836   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9837 {
9838   rtx ind = XEXP (operands[0], 0);
9839   gcc_assert (REG_P (ind));
9840   emit_insn (
9841     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9842                                                         operands[2],
9843                                                         operands[3]));
9844   DONE;
9845 })
9846
9847 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9848   [(set (mem:BLK (scratch))
9849         (unspec:BLK
9850           [(match_operand:SI 0 "register_operand" "r")
9851            (match_operand:V4SI 1 "s_register_operand" "w")
9852            (match_operand:V4SF 2 "s_register_operand" "w")
9853            (match_operand:HI 3 "vpr_register_operand" "Up")]
9854           VSTRWQSSO_F))]
9855   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9856   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9857   [(set_attr "length" "8")])
9858
9859 ;;
9860 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9861 ;;
9862 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9863   [(match_operand:V4SI 0 "mve_scatter_memory")
9864    (match_operand:V4SI 1 "s_register_operand")
9865    (match_operand:V4SI 2 "s_register_operand")
9866    (match_operand:HI 3 "vpr_register_operand")
9867    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9868   "TARGET_HAVE_MVE"
9869 {
9870   rtx ind = XEXP (operands[0], 0);
9871   gcc_assert (REG_P (ind));
9872   emit_insn (
9873     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9874                                                              operands[2],
9875                                                              operands[3]));
9876   DONE;
9877 })
9878
9879 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9880   [(set (mem:BLK (scratch))
9881         (unspec:BLK
9882           [(match_operand:SI 0 "register_operand" "r")
9883            (match_operand:V4SI 1 "s_register_operand" "w")
9884            (match_operand:V4SI 2 "s_register_operand" "w")
9885            (match_operand:HI 3 "vpr_register_operand" "Up")]
9886           VSTRWSSOQ))]
9887   "TARGET_HAVE_MVE"
9888   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9889   [(set_attr "length" "8")])
9890
9891 ;;
9892 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9893 ;;
9894 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9895   [(match_operand:V4SI 0 "mve_scatter_memory")
9896    (match_operand:V4SI 1 "s_register_operand")
9897    (match_operand:V4SI 2 "s_register_operand")
9898    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9899   "TARGET_HAVE_MVE"
9900 {
9901   rtx ind = XEXP (operands[0], 0);
9902   gcc_assert (REG_P (ind));
9903   emit_insn (
9904     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9905                                                            operands[2]));
9906   DONE;
9907 })
9908
9909 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9910   [(set (mem:BLK (scratch))
9911         (unspec:BLK
9912           [(match_operand:SI 0 "register_operand" "r")
9913            (match_operand:V4SI 1 "s_register_operand" "w")
9914            (match_operand:V4SI 2 "s_register_operand" "w")]
9915           VSTRWSSOQ))]
9916   "TARGET_HAVE_MVE"
9917   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9918   [(set_attr "length" "4")])
9919
9920 ;;
9921 ;; [vaddq_s, vaddq_u])
9922 ;;
9923 (define_insn "mve_vaddq<mode>"
9924   [
9925    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9926         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9927                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9928   ]
9929   "TARGET_HAVE_MVE"
9930   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9931   [(set_attr "type" "mve_move")
9932 ])
9933
9934 ;;
9935 ;; [vaddq_f])
9936 ;;
9937 (define_insn "mve_vaddq_f<mode>"
9938   [
9939    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9940         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9941                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9942   ]
9943   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9944   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9945   [(set_attr "type" "mve_move")
9946 ])
9947
9948 ;;
9949 ;; [vidupq_n_u])
9950 ;;
9951 (define_expand "mve_vidupq_n_u<mode>"
9952  [(match_operand:MVE_2 0 "s_register_operand")
9953   (match_operand:SI 1 "s_register_operand")
9954   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9955  "TARGET_HAVE_MVE"
9956 {
9957   rtx temp = gen_reg_rtx (SImode);
9958   emit_move_insn (temp, operands[1]);
9959   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9960   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9961                                           operands[2], inc));
9962   DONE;
9963 })
9964
9965 ;;
9966 ;; [vidupq_u_insn])
9967 ;;
9968 (define_insn "mve_vidupq_u<mode>_insn"
9969  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9970        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9971                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9972          VIDUPQ))
9973   (set (match_operand:SI 1 "s_register_operand" "=Te")
9974        (plus:SI (match_dup 2)
9975                 (match_operand:SI 4 "immediate_operand" "i")))]
9976  "TARGET_HAVE_MVE"
9977  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9978
9979 ;;
9980 ;; [vidupq_m_n_u])
9981 ;;
9982 (define_expand "mve_vidupq_m_n_u<mode>"
9983   [(match_operand:MVE_2 0 "s_register_operand")
9984    (match_operand:MVE_2 1 "s_register_operand")
9985    (match_operand:SI 2 "s_register_operand")
9986    (match_operand:SI 3 "mve_imm_selective_upto_8")
9987    (match_operand:HI 4 "vpr_register_operand")]
9988   "TARGET_HAVE_MVE"
9989 {
9990   rtx temp = gen_reg_rtx (SImode);
9991   emit_move_insn (temp, operands[2]);
9992   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9993   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9994                                              operands[2], operands[3],
9995                                              operands[4], inc));
9996   DONE;
9997 })
9998
9999 ;;
10000 ;; [vidupq_m_wb_u_insn])
10001 ;;
10002 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
10003  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10004        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
10005                       (match_operand:SI 3 "s_register_operand" "2")
10006                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
10007                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10008         VIDUPQ_M))
10009   (set (match_operand:SI 2 "s_register_operand" "=Te")
10010        (plus:SI (match_dup 3)
10011                 (match_operand:SI 6 "immediate_operand" "i")))]
10012  "TARGET_HAVE_MVE"
10013  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
10014  [(set_attr "length""8")])
10015
10016 ;;
10017 ;; [vddupq_n_u])
10018 ;;
10019 (define_expand "mve_vddupq_n_u<mode>"
10020  [(match_operand:MVE_2 0 "s_register_operand")
10021   (match_operand:SI 1 "s_register_operand")
10022   (match_operand:SI 2 "mve_imm_selective_upto_8")]
10023  "TARGET_HAVE_MVE"
10024 {
10025   rtx temp = gen_reg_rtx (SImode);
10026   emit_move_insn (temp, operands[1]);
10027   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
10028   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
10029                                           operands[2], inc));
10030   DONE;
10031 })
10032
10033 ;;
10034 ;; [vddupq_u_insn])
10035 ;;
10036 (define_insn "mve_vddupq_u<mode>_insn"
10037  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10038        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10039                       (match_operand:SI 3 "immediate_operand" "i")]
10040         VDDUPQ))
10041   (set (match_operand:SI 1 "s_register_operand" "=Te")
10042        (minus:SI (match_dup 2)
10043                  (match_operand:SI 4 "immediate_operand" "i")))]
10044  "TARGET_HAVE_MVE"
10045  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
10046
10047 ;;
10048 ;; [vddupq_m_n_u])
10049 ;;
10050 (define_expand "mve_vddupq_m_n_u<mode>"
10051   [(match_operand:MVE_2 0 "s_register_operand")
10052    (match_operand:MVE_2 1 "s_register_operand")
10053    (match_operand:SI 2 "s_register_operand")
10054    (match_operand:SI 3 "mve_imm_selective_upto_8")
10055    (match_operand:HI 4 "vpr_register_operand")]
10056   "TARGET_HAVE_MVE"
10057 {
10058   rtx temp = gen_reg_rtx (SImode);
10059   emit_move_insn (temp, operands[2]);
10060   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
10061   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
10062                                              operands[2], operands[3],
10063                                              operands[4], inc));
10064   DONE;
10065 })
10066
10067 ;;
10068 ;; [vddupq_m_wb_u_insn])
10069 ;;
10070 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
10071  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10072        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
10073                       (match_operand:SI 3 "s_register_operand" "2")
10074                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
10075                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10076         VDDUPQ_M))
10077   (set (match_operand:SI 2 "s_register_operand" "=Te")
10078        (minus:SI (match_dup 3)
10079                  (match_operand:SI 6 "immediate_operand" "i")))]
10080  "TARGET_HAVE_MVE"
10081  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
10082  [(set_attr "length""8")])
10083
10084 ;;
10085 ;; [vdwdupq_n_u])
10086 ;;
10087 (define_expand "mve_vdwdupq_n_u<mode>"
10088  [(match_operand:MVE_2 0 "s_register_operand")
10089   (match_operand:SI 1 "s_register_operand")
10090   (match_operand:DI 2 "s_register_operand")
10091   (match_operand:SI 3 "mve_imm_selective_upto_8")]
10092  "TARGET_HAVE_MVE"
10093 {
10094   rtx ignore_wb = gen_reg_rtx (SImode);
10095   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
10096                                               operands[1], operands[2],
10097                                               operands[3]));
10098   DONE;
10099 })
10100
10101 ;;
10102 ;; [vdwdupq_wb_u])
10103 ;;
10104 (define_expand "mve_vdwdupq_wb_u<mode>"
10105  [(match_operand:SI 0 "s_register_operand")
10106   (match_operand:SI 1 "s_register_operand")
10107   (match_operand:DI 2 "s_register_operand")
10108   (match_operand:SI 3 "mve_imm_selective_upto_8")
10109   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10110  "TARGET_HAVE_MVE"
10111 {
10112   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10113   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
10114                                               operands[1], operands[2],
10115                                               operands[3]));
10116   DONE;
10117 })
10118
10119 ;;
10120 ;; [vdwdupq_wb_u_insn])
10121 ;;
10122 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
10123   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10124         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10125                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
10126                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
10127          VDWDUPQ))
10128    (set (match_operand:SI 1 "s_register_operand" "=Te")
10129         (unspec:SI [(match_dup 2)
10130                     (subreg:SI (match_dup 3) 4)
10131                     (match_dup 4)]
10132          VDWDUPQ))]
10133   "TARGET_HAVE_MVE"
10134   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
10135 )
10136
10137 ;;
10138 ;; [vdwdupq_m_n_u])
10139 ;;
10140 (define_expand "mve_vdwdupq_m_n_u<mode>"
10141  [(match_operand:MVE_2 0 "s_register_operand")
10142   (match_operand:MVE_2 1 "s_register_operand")
10143   (match_operand:SI 2 "s_register_operand")
10144   (match_operand:DI 3 "s_register_operand")
10145   (match_operand:SI 4 "mve_imm_selective_upto_8")
10146   (match_operand:HI 5 "vpr_register_operand")]
10147  "TARGET_HAVE_MVE"
10148 {
10149   rtx ignore_wb = gen_reg_rtx (SImode);
10150   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
10151                                                 operands[1], operands[2],
10152                                                 operands[3], operands[4],
10153                                                 operands[5]));
10154   DONE;
10155 })
10156
10157 ;;
10158 ;; [vdwdupq_m_wb_u])
10159 ;;
10160 (define_expand "mve_vdwdupq_m_wb_u<mode>"
10161  [(match_operand:SI 0 "s_register_operand")
10162   (match_operand:MVE_2 1 "s_register_operand")
10163   (match_operand:SI 2 "s_register_operand")
10164   (match_operand:DI 3 "s_register_operand")
10165   (match_operand:SI 4 "mve_imm_selective_upto_8")
10166   (match_operand:HI 5 "vpr_register_operand")]
10167  "TARGET_HAVE_MVE"
10168 {
10169   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10170   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
10171                                                 operands[1], operands[2],
10172                                                 operands[3], operands[4],
10173                                                 operands[5]));
10174   DONE;
10175 })
10176
10177 ;;
10178 ;; [vdwdupq_m_wb_u_insn])
10179 ;;
10180 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
10181   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10182         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10183                        (match_operand:SI 3 "s_register_operand" "1")
10184                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
10185                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
10186                        (match_operand:HI 6 "vpr_register_operand" "Up")]
10187          VDWDUPQ_M))
10188    (set (match_operand:SI 1 "s_register_operand" "=Te")
10189         (unspec:SI [(match_dup 2)
10190                     (match_dup 3)
10191                     (subreg:SI (match_dup 4) 4)
10192                     (match_dup 5)
10193                     (match_dup 6)]
10194          VDWDUPQ_M))
10195   ]
10196   "TARGET_HAVE_MVE"
10197   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
10198   [(set_attr "type" "mve_move")
10199    (set_attr "length""8")])
10200
10201 ;;
10202 ;; [viwdupq_n_u])
10203 ;;
10204 (define_expand "mve_viwdupq_n_u<mode>"
10205  [(match_operand:MVE_2 0 "s_register_operand")
10206   (match_operand:SI 1 "s_register_operand")
10207   (match_operand:DI 2 "s_register_operand")
10208   (match_operand:SI 3 "mve_imm_selective_upto_8")]
10209  "TARGET_HAVE_MVE"
10210 {
10211   rtx ignore_wb = gen_reg_rtx (SImode);
10212   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
10213                                               operands[1], operands[2],
10214                                               operands[3]));
10215   DONE;
10216 })
10217
10218 ;;
10219 ;; [viwdupq_wb_u])
10220 ;;
10221 (define_expand "mve_viwdupq_wb_u<mode>"
10222  [(match_operand:SI 0 "s_register_operand")
10223   (match_operand:SI 1 "s_register_operand")
10224   (match_operand:DI 2 "s_register_operand")
10225   (match_operand:SI 3 "mve_imm_selective_upto_8")
10226   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10227  "TARGET_HAVE_MVE"
10228 {
10229   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10230   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
10231                                               operands[1], operands[2],
10232                                               operands[3]));
10233   DONE;
10234 })
10235
10236 ;;
10237 ;; [viwdupq_wb_u_insn])
10238 ;;
10239 (define_insn "mve_viwdupq_wb_u<mode>_insn"
10240   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10241         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10242                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
10243                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
10244          VIWDUPQ))
10245    (set (match_operand:SI 1 "s_register_operand" "=Te")
10246         (unspec:SI [(match_dup 2)
10247                     (subreg:SI (match_dup 3) 4)
10248                     (match_dup 4)]
10249          VIWDUPQ))]
10250   "TARGET_HAVE_MVE"
10251   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
10252 )
10253
10254 ;;
10255 ;; [viwdupq_m_n_u])
10256 ;;
10257 (define_expand "mve_viwdupq_m_n_u<mode>"
10258  [(match_operand:MVE_2 0 "s_register_operand")
10259   (match_operand:MVE_2 1 "s_register_operand")
10260   (match_operand:SI 2 "s_register_operand")
10261   (match_operand:DI 3 "s_register_operand")
10262   (match_operand:SI 4 "mve_imm_selective_upto_8")
10263   (match_operand:HI 5 "vpr_register_operand")]
10264  "TARGET_HAVE_MVE"
10265 {
10266   rtx ignore_wb = gen_reg_rtx (SImode);
10267   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
10268                                                 operands[1], operands[2],
10269                                                 operands[3], operands[4],
10270                                                 operands[5]));
10271   DONE;
10272 })
10273
10274 ;;
10275 ;; [viwdupq_m_wb_u])
10276 ;;
10277 (define_expand "mve_viwdupq_m_wb_u<mode>"
10278  [(match_operand:SI 0 "s_register_operand")
10279   (match_operand:MVE_2 1 "s_register_operand")
10280   (match_operand:SI 2 "s_register_operand")
10281   (match_operand:DI 3 "s_register_operand")
10282   (match_operand:SI 4 "mve_imm_selective_upto_8")
10283   (match_operand:HI 5 "vpr_register_operand")]
10284  "TARGET_HAVE_MVE"
10285 {
10286   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10287   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
10288                                                 operands[1], operands[2],
10289                                                 operands[3], operands[4],
10290                                                 operands[5]));
10291   DONE;
10292 })
10293
10294 ;;
10295 ;; [viwdupq_m_wb_u_insn])
10296 ;;
10297 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
10298   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10299         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10300                        (match_operand:SI 3 "s_register_operand" "1")
10301                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
10302                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
10303                        (match_operand:HI 6 "vpr_register_operand" "Up")]
10304          VIWDUPQ_M))
10305    (set (match_operand:SI 1 "s_register_operand" "=Te")
10306         (unspec:SI [(match_dup 2)
10307                     (match_dup 3)
10308                     (subreg:SI (match_dup 4) 4)
10309                     (match_dup 5)
10310                     (match_dup 6)]
10311          VIWDUPQ_M))
10312   ]
10313   "TARGET_HAVE_MVE"
10314   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
10315   [(set_attr "type" "mve_move")
10316    (set_attr "length""8")])
10317
10318 (define_expand "mve_vstrwq_scatter_base_wb_<supf>v4si"
10319   [(match_operand:V4SI 0 "s_register_operand" "=w")
10320    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10321    (match_operand:V4SI 2 "s_register_operand" "w")
10322    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10323   "TARGET_HAVE_MVE"
10324 {
10325   rtx ignore_wb = gen_reg_rtx (V4SImode);
10326   emit_insn (
10327   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (ignore_wb, operands[0],
10328                                                   operands[1], operands[2]));
10329   DONE;
10330 })
10331
10332 (define_expand "mve_vstrwq_scatter_base_wb_add_<supf>v4si"
10333   [(match_operand:V4SI 0 "s_register_operand" "=w")
10334    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10335    (match_operand:V4SI 2 "s_register_operand" "0")
10336    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10337   "TARGET_HAVE_MVE"
10338 {
10339   rtx ignore_vec = gen_reg_rtx (V4SImode);
10340   emit_insn (
10341   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (operands[0], operands[2],
10342                                                   operands[1], ignore_vec));
10343   DONE;
10344 })
10345
10346 ;;
10347 ;; [vstrwq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10348 ;;
10349 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si_insn"
10350   [(set (mem:BLK (scratch))
10351         (unspec:BLK
10352                 [(match_operand:V4SI 1 "s_register_operand" "0")
10353                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10354                  (match_operand:V4SI 3 "s_register_operand" "w")]
10355          VSTRWSBWBQ))
10356    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10357         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10358          VSTRWSBWBQ))
10359   ]
10360   "TARGET_HAVE_MVE"
10361 {
10362    rtx ops[3];
10363    ops[0] = operands[1];
10364    ops[1] = operands[2];
10365    ops[2] = operands[3];
10366    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10367    return "";
10368 }
10369   [(set_attr "length" "4")])
10370
10371 (define_expand "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
10372   [(match_operand:V4SI 0 "s_register_operand" "=w")
10373    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10374    (match_operand:V4SI 2 "s_register_operand" "w")
10375    (match_operand:HI 3 "vpr_register_operand")
10376    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10377   "TARGET_HAVE_MVE"
10378 {
10379   rtx ignore_wb = gen_reg_rtx (V4SImode);
10380   emit_insn (
10381   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (ignore_wb, operands[0],
10382                                                     operands[1], operands[2],
10383                                                     operands[3]));
10384   DONE;
10385 })
10386
10387 (define_expand "mve_vstrwq_scatter_base_wb_p_add_<supf>v4si"
10388   [(match_operand:V4SI 0 "s_register_operand" "=w")
10389    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10390    (match_operand:V4SI 2 "s_register_operand" "0")
10391    (match_operand:HI 3 "vpr_register_operand")
10392    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10393   "TARGET_HAVE_MVE"
10394 {
10395   rtx ignore_vec = gen_reg_rtx (V4SImode);
10396   emit_insn (
10397   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (operands[0], operands[2],
10398                                                     operands[1], ignore_vec,
10399                                                     operands[3]));
10400   DONE;
10401 })
10402
10403 ;;
10404 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
10405 ;;
10406 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn"
10407  [(set (mem:BLK (scratch))
10408        (unspec:BLK
10409                 [(match_operand:V4SI 1 "s_register_operand" "0")
10410                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10411                  (match_operand:V4SI 3 "s_register_operand" "w")
10412                  (match_operand:HI 4 "vpr_register_operand")]
10413         VSTRWSBWBQ))
10414    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10415         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10416          VSTRWSBWBQ))
10417   ]
10418   "TARGET_HAVE_MVE"
10419 {
10420    rtx ops[3];
10421    ops[0] = operands[1];
10422    ops[1] = operands[2];
10423    ops[2] = operands[3];
10424    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10425    return "";
10426 }
10427   [(set_attr "length" "8")])
10428
10429 (define_expand "mve_vstrwq_scatter_base_wb_fv4sf"
10430   [(match_operand:V4SI 0 "s_register_operand" "=w")
10431    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10432    (match_operand:V4SF 2 "s_register_operand" "w")
10433    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10434   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10435 {
10436   rtx ignore_wb = gen_reg_rtx (V4SImode);
10437   emit_insn (
10438   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (ignore_wb,operands[0],
10439                                              operands[1], operands[2]));
10440   DONE;
10441 })
10442
10443 (define_expand "mve_vstrwq_scatter_base_wb_add_fv4sf"
10444   [(match_operand:V4SI 0 "s_register_operand" "=w")
10445    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10446    (match_operand:V4SI 2 "s_register_operand" "0")
10447    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10448   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10449 {
10450   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10451   emit_insn (
10452   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (operands[0], operands[2],
10453                                              operands[1], ignore_vec));
10454   DONE;
10455 })
10456
10457 ;;
10458 ;; [vstrwq_scatter_base_wb_f]
10459 ;;
10460 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf_insn"
10461  [(set (mem:BLK (scratch))
10462        (unspec:BLK
10463                 [(match_operand:V4SI 1 "s_register_operand" "0")
10464                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10465                  (match_operand:V4SF 3 "s_register_operand" "w")]
10466          VSTRWQSBWB_F))
10467    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10468         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10469          VSTRWQSBWB_F))
10470   ]
10471   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10472 {
10473    rtx ops[3];
10474    ops[0] = operands[1];
10475    ops[1] = operands[2];
10476    ops[2] = operands[3];
10477    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10478    return "";
10479 }
10480   [(set_attr "length" "4")])
10481
10482 (define_expand "mve_vstrwq_scatter_base_wb_p_fv4sf"
10483   [(match_operand:V4SI 0 "s_register_operand" "=w")
10484    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10485    (match_operand:V4SF 2 "s_register_operand" "w")
10486    (match_operand:HI 3 "vpr_register_operand")
10487    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10488   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10489 {
10490   rtx ignore_wb = gen_reg_rtx (V4SImode);
10491   emit_insn (
10492   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (ignore_wb, operands[0],
10493                                                operands[1], operands[2],
10494                                                operands[3]));
10495   DONE;
10496 })
10497
10498 (define_expand "mve_vstrwq_scatter_base_wb_p_add_fv4sf"
10499   [(match_operand:V4SI 0 "s_register_operand" "=w")
10500    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10501    (match_operand:V4SI 2 "s_register_operand" "0")
10502    (match_operand:HI 3 "vpr_register_operand")
10503    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10504   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10505 {
10506   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10507   emit_insn (
10508   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (operands[0], operands[2],
10509                                                operands[1], ignore_vec,
10510                                                operands[3]));
10511   DONE;
10512 })
10513
10514 ;;
10515 ;; [vstrwq_scatter_base_wb_p_f]
10516 ;;
10517 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf_insn"
10518  [(set (mem:BLK (scratch))
10519        (unspec:BLK
10520                 [(match_operand:V4SI 1 "s_register_operand" "0")
10521                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10522                  (match_operand:V4SF 3 "s_register_operand" "w")
10523                  (match_operand:HI 4 "vpr_register_operand")]
10524         VSTRWQSBWB_F))
10525    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10526         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10527          VSTRWQSBWB_F))
10528   ]
10529   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10530 {
10531    rtx ops[3];
10532    ops[0] = operands[1];
10533    ops[1] = operands[2];
10534    ops[2] = operands[3];
10535    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10536    return "";
10537 }
10538   [(set_attr "length" "8")])
10539
10540 (define_expand "mve_vstrdq_scatter_base_wb_<supf>v2di"
10541   [(match_operand:V2DI 0 "s_register_operand" "=w")
10542    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10543    (match_operand:V2DI 2 "s_register_operand" "w")
10544    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10545   "TARGET_HAVE_MVE"
10546 {
10547   rtx ignore_wb = gen_reg_rtx (V2DImode);
10548   emit_insn (
10549   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (ignore_wb, operands[0],
10550                                                   operands[1], operands[2]));
10551   DONE;
10552 })
10553
10554 (define_expand "mve_vstrdq_scatter_base_wb_add_<supf>v2di"
10555   [(match_operand:V2DI 0 "s_register_operand" "=w")
10556    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10557    (match_operand:V2DI 2 "s_register_operand" "0")
10558    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10559   "TARGET_HAVE_MVE"
10560 {
10561   rtx ignore_vec = gen_reg_rtx (V2DImode);
10562   emit_insn (
10563   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (operands[0], operands[2],
10564                                                   operands[1], ignore_vec));
10565   DONE;
10566 })
10567
10568 ;;
10569 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10570 ;;
10571 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di_insn"
10572   [(set (mem:BLK (scratch))
10573         (unspec:BLK
10574                 [(match_operand:V2DI 1 "s_register_operand" "0")
10575                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10576                  (match_operand:V2DI 3 "s_register_operand" "w")]
10577          VSTRDSBWBQ))
10578    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
10579         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10580          VSTRDSBWBQ))
10581   ]
10582   "TARGET_HAVE_MVE"
10583 {
10584    rtx ops[3];
10585    ops[0] = operands[1];
10586    ops[1] = operands[2];
10587    ops[2] = operands[3];
10588    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
10589    return "";
10590 }
10591   [(set_attr "length" "4")])
10592
10593 (define_expand "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
10594   [(match_operand:V2DI 0 "s_register_operand" "=w")
10595    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10596    (match_operand:V2DI 2 "s_register_operand" "w")
10597    (match_operand:HI 3 "vpr_register_operand")
10598    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10599   "TARGET_HAVE_MVE"
10600 {
10601   rtx ignore_wb = gen_reg_rtx (V2DImode);
10602   emit_insn (
10603   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (ignore_wb, operands[0],
10604                                                     operands[1], operands[2],
10605                                                     operands[3]));
10606   DONE;
10607 })
10608
10609 (define_expand "mve_vstrdq_scatter_base_wb_p_add_<supf>v2di"
10610   [(match_operand:V2DI 0 "s_register_operand" "=w")
10611    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10612    (match_operand:V2DI 2 "s_register_operand" "0")
10613    (match_operand:HI 3 "vpr_register_operand")
10614    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10615   "TARGET_HAVE_MVE"
10616 {
10617   rtx ignore_vec = gen_reg_rtx (V2DImode);
10618   emit_insn (
10619   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (operands[0], operands[2],
10620                                                     operands[1], ignore_vec,
10621                                                     operands[3]));
10622   DONE;
10623 })
10624
10625 ;;
10626 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
10627 ;;
10628 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn"
10629   [(set (mem:BLK (scratch))
10630         (unspec:BLK
10631                 [(match_operand:V2DI 1 "s_register_operand" "0")
10632                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10633                  (match_operand:V2DI 3 "s_register_operand" "w")
10634                  (match_operand:HI 4 "vpr_register_operand")]
10635          VSTRDSBWBQ))
10636    (set (match_operand:V2DI 0 "s_register_operand" "=w")
10637         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10638          VSTRDSBWBQ))
10639   ]
10640   "TARGET_HAVE_MVE"
10641 {
10642    rtx ops[3];
10643    ops[0] = operands[1];
10644    ops[1] = operands[2];
10645    ops[2] = operands[3];
10646    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]!",ops);
10647    return "";
10648 }
10649   [(set_attr "length" "8")])
10650
10651 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
10652   [(match_operand:V4SI 0 "s_register_operand")
10653    (match_operand:V4SI 1 "s_register_operand")
10654    (match_operand:SI 2 "mve_vldrd_immediate")
10655    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10656   "TARGET_HAVE_MVE"
10657 {
10658   rtx ignore_result = gen_reg_rtx (V4SImode);
10659   emit_insn (
10660   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
10661                                                  operands[1], operands[2]));
10662   DONE;
10663 })
10664
10665 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
10666   [(match_operand:V4SI 0 "s_register_operand")
10667    (match_operand:V4SI 1 "s_register_operand")
10668    (match_operand:SI 2 "mve_vldrd_immediate")
10669    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10670   "TARGET_HAVE_MVE"
10671 {
10672   rtx ignore_wb = gen_reg_rtx (V4SImode);
10673   emit_insn (
10674   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
10675                                                  operands[1], operands[2]));
10676   DONE;
10677 })
10678
10679 ;;
10680 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
10681 ;;
10682 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
10683   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10684         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10685                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10686                       (mem:BLK (scratch))]
10687          VLDRWGBWBQ))
10688    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10689         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10690          VLDRWGBWBQ))
10691   ]
10692   "TARGET_HAVE_MVE"
10693 {
10694    rtx ops[3];
10695    ops[0] = operands[0];
10696    ops[1] = operands[2];
10697    ops[2] = operands[3];
10698    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10699    return "";
10700 }
10701   [(set_attr "length" "4")])
10702
10703 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
10704   [(match_operand:V4SI 0 "s_register_operand")
10705    (match_operand:V4SI 1 "s_register_operand")
10706    (match_operand:SI 2 "mve_vldrd_immediate")
10707    (match_operand:HI 3 "vpr_register_operand")
10708    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10709   "TARGET_HAVE_MVE"
10710 {
10711   rtx ignore_result = gen_reg_rtx (V4SImode);
10712   emit_insn (
10713   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
10714                                                    operands[1], operands[2],
10715                                                    operands[3]));
10716   DONE;
10717 })
10718 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
10719   [(match_operand:V4SI 0 "s_register_operand")
10720    (match_operand:V4SI 1 "s_register_operand")
10721    (match_operand:SI 2 "mve_vldrd_immediate")
10722    (match_operand:HI 3 "vpr_register_operand")
10723    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10724   "TARGET_HAVE_MVE"
10725 {
10726   rtx ignore_wb = gen_reg_rtx (V4SImode);
10727   emit_insn (
10728   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10729                                                    operands[1], operands[2],
10730                                                    operands[3]));
10731   DONE;
10732 })
10733
10734 ;;
10735 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10736 ;;
10737 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10738   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10739         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10740                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10741                       (match_operand:HI 4 "vpr_register_operand" "Up")
10742                       (mem:BLK (scratch))]
10743          VLDRWGBWBQ))
10744    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10745         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10746          VLDRWGBWBQ))
10747   ]
10748   "TARGET_HAVE_MVE"
10749 {
10750    rtx ops[3];
10751    ops[0] = operands[0];
10752    ops[1] = operands[2];
10753    ops[2] = operands[3];
10754    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10755    return "";
10756 }
10757   [(set_attr "length" "8")])
10758
10759 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10760   [(match_operand:V4SI 0 "s_register_operand")
10761    (match_operand:V4SI 1 "s_register_operand")
10762    (match_operand:SI 2 "mve_vldrd_immediate")
10763    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10764   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10765 {
10766   rtx ignore_result = gen_reg_rtx (V4SFmode);
10767   emit_insn (
10768   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10769                                             operands[1], operands[2]));
10770   DONE;
10771 })
10772
10773 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10774   [(match_operand:V4SF 0 "s_register_operand")
10775    (match_operand:V4SI 1 "s_register_operand")
10776    (match_operand:SI 2 "mve_vldrd_immediate")
10777    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10778   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10779 {
10780   rtx ignore_wb = gen_reg_rtx (V4SImode);
10781   emit_insn (
10782   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10783                                             operands[1], operands[2]));
10784   DONE;
10785 })
10786
10787 ;;
10788 ;; [vldrwq_gather_base_wb_f]
10789 ;;
10790 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10791   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10792         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10793                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10794                       (mem:BLK (scratch))]
10795          VLDRWQGBWB_F))
10796    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10797         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10798          VLDRWQGBWB_F))
10799   ]
10800   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10801 {
10802    rtx ops[3];
10803    ops[0] = operands[0];
10804    ops[1] = operands[2];
10805    ops[2] = operands[3];
10806    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10807    return "";
10808 }
10809   [(set_attr "length" "4")])
10810
10811 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10812   [(match_operand:V4SI 0 "s_register_operand")
10813    (match_operand:V4SI 1 "s_register_operand")
10814    (match_operand:SI 2 "mve_vldrd_immediate")
10815    (match_operand:HI 3 "vpr_register_operand")
10816    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10817   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10818 {
10819   rtx ignore_result = gen_reg_rtx (V4SFmode);
10820   emit_insn (
10821   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10822                                               operands[1], operands[2],
10823                                               operands[3]));
10824   DONE;
10825 })
10826
10827 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10828   [(match_operand:V4SF 0 "s_register_operand")
10829    (match_operand:V4SI 1 "s_register_operand")
10830    (match_operand:SI 2 "mve_vldrd_immediate")
10831    (match_operand:HI 3 "vpr_register_operand")
10832    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10834 {
10835   rtx ignore_wb = gen_reg_rtx (V4SImode);
10836   emit_insn (
10837   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10838                                               operands[1], operands[2],
10839                                               operands[3]));
10840   DONE;
10841 })
10842
10843 ;;
10844 ;; [vldrwq_gather_base_wb_z_f]
10845 ;;
10846 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10847   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10848         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10849                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10850                       (match_operand:HI 4 "vpr_register_operand" "Up")
10851                       (mem:BLK (scratch))]
10852          VLDRWQGBWB_F))
10853    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10854         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10855          VLDRWQGBWB_F))
10856   ]
10857   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10858 {
10859    rtx ops[3];
10860    ops[0] = operands[0];
10861    ops[1] = operands[2];
10862    ops[2] = operands[3];
10863    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10864    return "";
10865 }
10866   [(set_attr "length" "8")])
10867
10868 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10869   [(match_operand:V2DI 0 "s_register_operand")
10870    (match_operand:V2DI 1 "s_register_operand")
10871    (match_operand:SI 2 "mve_vldrd_immediate")
10872    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10873   "TARGET_HAVE_MVE"
10874 {
10875   rtx ignore_result = gen_reg_rtx (V2DImode);
10876   emit_insn (
10877   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10878                                                  operands[1], operands[2]));
10879   DONE;
10880 })
10881
10882 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10883   [(match_operand:V2DI 0 "s_register_operand")
10884    (match_operand:V2DI 1 "s_register_operand")
10885    (match_operand:SI 2 "mve_vldrd_immediate")
10886    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10887   "TARGET_HAVE_MVE"
10888 {
10889   rtx ignore_wb = gen_reg_rtx (V2DImode);
10890   emit_insn (
10891   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10892                                                  operands[1], operands[2]));
10893   DONE;
10894 })
10895
10896
10897 ;;
10898 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10899 ;;
10900 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10901   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10902         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10903                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10904                       (mem:BLK (scratch))]
10905          VLDRDGBWBQ))
10906    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10907         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10908          VLDRDGBWBQ))
10909   ]
10910   "TARGET_HAVE_MVE"
10911 {
10912    rtx ops[3];
10913    ops[0] = operands[0];
10914    ops[1] = operands[2];
10915    ops[2] = operands[3];
10916    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10917    return "";
10918 }
10919   [(set_attr "length" "4")])
10920
10921 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10922   [(match_operand:V2DI 0 "s_register_operand")
10923    (match_operand:V2DI 1 "s_register_operand")
10924    (match_operand:SI 2 "mve_vldrd_immediate")
10925    (match_operand:HI 3 "vpr_register_operand")
10926    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10927   "TARGET_HAVE_MVE"
10928 {
10929   rtx ignore_result = gen_reg_rtx (V2DImode);
10930   emit_insn (
10931   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10932                                                    operands[1], operands[2],
10933                                                    operands[3]));
10934   DONE;
10935 })
10936
10937 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10938   [(match_operand:V2DI 0 "s_register_operand")
10939    (match_operand:V2DI 1 "s_register_operand")
10940    (match_operand:SI 2 "mve_vldrd_immediate")
10941    (match_operand:HI 3 "vpr_register_operand")
10942    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10943   "TARGET_HAVE_MVE"
10944 {
10945   rtx ignore_wb = gen_reg_rtx (V2DImode);
10946   emit_insn (
10947   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10948                                                    operands[1], operands[2],
10949                                                    operands[3]));
10950   DONE;
10951 })
10952
10953 (define_insn "get_fpscr_nzcvqc"
10954  [(set (match_operand:SI 0 "register_operand" "=r")
10955    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10956  "TARGET_HAVE_MVE"
10957  "vmrs\\t%0, FPSCR_nzcvqc"
10958   [(set_attr "type" "mve_move")])
10959
10960 (define_insn "set_fpscr_nzcvqc"
10961  [(set (reg:SI VFPCC_REGNUM)
10962    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10963     VUNSPEC_SET_FPSCR_NZCVQC))]
10964  "TARGET_HAVE_MVE"
10965  "vmsr\\tFPSCR_nzcvqc, %0"
10966   [(set_attr "type" "mve_move")])
10967
10968 ;;
10969 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10970 ;;
10971 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10972   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10973         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10974                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10975                       (match_operand:HI 4 "vpr_register_operand" "Up")
10976                       (mem:BLK (scratch))]
10977          VLDRDGBWBQ))
10978    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10979         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10980          VLDRDGBWBQ))
10981   ]
10982   "TARGET_HAVE_MVE"
10983 {
10984    rtx ops[3];
10985    ops[0] = operands[0];
10986    ops[1] = operands[2];
10987    ops[2] = operands[3];
10988    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10989    return "";
10990 }
10991   [(set_attr "length" "8")])
10992 ;;
10993 ;; [vadciq_m_s, vadciq_m_u])
10994 ;;
10995 (define_insn "mve_vadciq_m_<supf>v4si"
10996   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10997         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10998                       (match_operand:V4SI 2 "s_register_operand" "w")
10999                       (match_operand:V4SI 3 "s_register_operand" "w")
11000                       (match_operand:HI 4 "vpr_register_operand" "Up")]
11001          VADCIQ_M))
11002    (set (reg:SI VFPCC_REGNUM)
11003         (unspec:SI [(const_int 0)]
11004          VADCIQ_M))
11005   ]
11006   "TARGET_HAVE_MVE"
11007   "vpst\;vadcit.i32\t%q0, %q2, %q3"
11008   [(set_attr "type" "mve_move")
11009    (set_attr "length" "8")])
11010
11011 ;;
11012 ;; [vadciq_u, vadciq_s])
11013 ;;
11014 (define_insn "mve_vadciq_<supf>v4si"
11015   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11016         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11017                       (match_operand:V4SI 2 "s_register_operand" "w")]
11018          VADCIQ))
11019    (set (reg:SI VFPCC_REGNUM)
11020         (unspec:SI [(const_int 0)]
11021          VADCIQ))
11022   ]
11023   "TARGET_HAVE_MVE"
11024   "vadci.i32\t%q0, %q1, %q2"
11025   [(set_attr "type" "mve_move")
11026    (set_attr "length" "4")])
11027
11028 ;;
11029 ;; [vadcq_m_s, vadcq_m_u])
11030 ;;
11031 (define_insn "mve_vadcq_m_<supf>v4si"
11032   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11033         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
11034                       (match_operand:V4SI 2 "s_register_operand" "w")
11035                       (match_operand:V4SI 3 "s_register_operand" "w")
11036                       (match_operand:HI 4 "vpr_register_operand" "Up")]
11037          VADCQ_M))
11038    (set (reg:SI VFPCC_REGNUM)
11039         (unspec:SI [(reg:SI VFPCC_REGNUM)]
11040          VADCQ_M))
11041   ]
11042   "TARGET_HAVE_MVE"
11043   "vpst\;vadct.i32\t%q0, %q2, %q3"
11044   [(set_attr "type" "mve_move")
11045    (set_attr "length" "8")])
11046
11047 ;;
11048 ;; [vadcq_u, vadcq_s])
11049 ;;
11050 (define_insn "mve_vadcq_<supf>v4si"
11051   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11052         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11053                        (match_operand:V4SI 2 "s_register_operand" "w")]
11054          VADCQ))
11055    (set (reg:SI VFPCC_REGNUM)
11056         (unspec:SI [(reg:SI VFPCC_REGNUM)]
11057          VADCQ))
11058   ]
11059   "TARGET_HAVE_MVE"
11060   "vadc.i32\t%q0, %q1, %q2"
11061   [(set_attr "type" "mve_move")
11062    (set_attr "length" "4")
11063    (set_attr "conds" "set")])
11064
11065 ;;
11066 ;; [vsbciq_m_u, vsbciq_m_s])
11067 ;;
11068 (define_insn "mve_vsbciq_m_<supf>v4si"
11069   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11070         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11071                       (match_operand:V4SI 2 "s_register_operand" "w")
11072                       (match_operand:V4SI 3 "s_register_operand" "w")
11073                       (match_operand:HI 4 "vpr_register_operand" "Up")]
11074          VSBCIQ_M))
11075    (set (reg:SI VFPCC_REGNUM)
11076         (unspec:SI [(const_int 0)]
11077          VSBCIQ_M))
11078   ]
11079   "TARGET_HAVE_MVE"
11080   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
11081   [(set_attr "type" "mve_move")
11082    (set_attr "length" "8")])
11083
11084 ;;
11085 ;; [vsbciq_s, vsbciq_u])
11086 ;;
11087 (define_insn "mve_vsbciq_<supf>v4si"
11088   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11089         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11090                       (match_operand:V4SI 2 "s_register_operand" "w")]
11091          VSBCIQ))
11092    (set (reg:SI VFPCC_REGNUM)
11093         (unspec:SI [(const_int 0)]
11094          VSBCIQ))
11095   ]
11096   "TARGET_HAVE_MVE"
11097   "vsbci.i32\t%q0, %q1, %q2"
11098   [(set_attr "type" "mve_move")
11099    (set_attr "length" "4")])
11100
11101 ;;
11102 ;; [vsbcq_m_u, vsbcq_m_s])
11103 ;;
11104 (define_insn "mve_vsbcq_m_<supf>v4si"
11105   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11106         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11107                       (match_operand:V4SI 2 "s_register_operand" "w")
11108                       (match_operand:V4SI 3 "s_register_operand" "w")
11109                       (match_operand:HI 4 "vpr_register_operand" "Up")]
11110          VSBCQ_M))
11111    (set (reg:SI VFPCC_REGNUM)
11112         (unspec:SI [(reg:SI VFPCC_REGNUM)]
11113          VSBCQ_M))
11114   ]
11115   "TARGET_HAVE_MVE"
11116   "vpst\;vsbct.i32\t%q0, %q2, %q3"
11117   [(set_attr "type" "mve_move")
11118    (set_attr "length" "8")])
11119
11120 ;;
11121 ;; [vsbcq_s, vsbcq_u])
11122 ;;
11123 (define_insn "mve_vsbcq_<supf>v4si"
11124   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
11125         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
11126                       (match_operand:V4SI 2 "s_register_operand" "w")]
11127          VSBCQ))
11128    (set (reg:SI VFPCC_REGNUM)
11129         (unspec:SI [(reg:SI VFPCC_REGNUM)]
11130          VSBCQ))
11131   ]
11132   "TARGET_HAVE_MVE"
11133   "vsbc.i32\t%q0, %q1, %q2"
11134   [(set_attr "type" "mve_move")
11135    (set_attr "length" "4")])
11136
11137 ;;
11138 ;; [vst2q])
11139 ;;
11140 (define_insn "mve_vst2q<mode>"
11141   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
11142         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
11143                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
11144          VST2Q))
11145   ]
11146   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11147    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11148 {
11149    rtx ops[4];
11150    int regno = REGNO (operands[1]);
11151    ops[0] = gen_rtx_REG (TImode, regno);
11152    ops[1] = gen_rtx_REG (TImode, regno + 4);
11153    rtx reg  = operands[0];
11154    while (reg && !REG_P (reg))
11155     reg = XEXP (reg, 0);
11156    gcc_assert (REG_P (reg));
11157    ops[2] = reg;
11158    ops[3] = operands[0];
11159    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
11160                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
11161    return "";
11162 }
11163   [(set_attr "length" "8")])
11164
11165 ;;
11166 ;; [vld2q])
11167 ;;
11168 (define_insn "mve_vld2q<mode>"
11169   [(set (match_operand:OI 0 "s_register_operand" "=w")
11170         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
11171                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
11172          VLD2Q))
11173   ]
11174   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11175    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11176 {
11177    rtx ops[4];
11178    int regno = REGNO (operands[0]);
11179    ops[0] = gen_rtx_REG (TImode, regno);
11180    ops[1] = gen_rtx_REG (TImode, regno + 4);
11181    rtx reg  = operands[1];
11182    while (reg && !REG_P (reg))
11183     reg = XEXP (reg, 0);
11184    gcc_assert (REG_P (reg));
11185    ops[2] = reg;
11186    ops[3] = operands[1];
11187    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
11188                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
11189    return "";
11190 }
11191   [(set_attr "length" "8")])
11192
11193 ;;
11194 ;; [vld4q])
11195 ;;
11196 (define_insn "mve_vld4q<mode>"
11197   [(set (match_operand:XI 0 "s_register_operand" "=w")
11198         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
11199                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
11200          VLD4Q))
11201   ]
11202   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11203    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11204 {
11205    rtx ops[6];
11206    int regno = REGNO (operands[0]);
11207    ops[0] = gen_rtx_REG (TImode, regno);
11208    ops[1] = gen_rtx_REG (TImode, regno+4);
11209    ops[2] = gen_rtx_REG (TImode, regno+8);
11210    ops[3] = gen_rtx_REG (TImode, regno + 12);
11211    rtx reg  = operands[1];
11212    while (reg && !REG_P (reg))
11213     reg = XEXP (reg, 0);
11214    gcc_assert (REG_P (reg));
11215    ops[4] = reg;
11216    ops[5] = operands[1];
11217    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
11218                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
11219                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
11220                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
11221    return "";
11222 }
11223   [(set_attr "length" "16")])
11224 ;;
11225 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
11226 ;;
11227 (define_insn "mve_vec_extract<mode><V_elem_l>"
11228  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
11229    (vec_select:<V_elem>
11230     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
11231     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
11232   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11233    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11234 {
11235   if (BYTES_BIG_ENDIAN)
11236     {
11237       int elt = INTVAL (operands[2]);
11238       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11239       operands[2] = GEN_INT (elt);
11240     }
11241   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
11242 }
11243  [(set_attr "type" "mve_move")])
11244
11245 (define_insn "mve_vec_extractv2didi"
11246  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
11247    (vec_select:DI
11248     (match_operand:V2DI 1 "s_register_operand" "w")
11249     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
11250   "TARGET_HAVE_MVE"
11251 {
11252   int elt = INTVAL (operands[2]);
11253   if (BYTES_BIG_ENDIAN)
11254     elt = 1 - elt;
11255
11256   if (elt == 0)
11257    return "vmov\t%Q0, %R0, %e1";
11258   else
11259    return "vmov\t%Q0, %R0, %f1";
11260 }
11261  [(set_attr "type" "mve_move")])
11262
11263 (define_insn "*mve_vec_extract_sext_internal<mode>"
11264  [(set (match_operand:SI 0 "s_register_operand" "=r")
11265    (sign_extend:SI
11266     (vec_select:<V_elem>
11267      (match_operand:MVE_2 1 "s_register_operand" "w")
11268      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11269   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11270    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11271 {
11272   if (BYTES_BIG_ENDIAN)
11273     {
11274       int elt = INTVAL (operands[2]);
11275       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11276       operands[2] = GEN_INT (elt);
11277     }
11278   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
11279 }
11280  [(set_attr "type" "mve_move")])
11281
11282 (define_insn "*mve_vec_extract_zext_internal<mode>"
11283  [(set (match_operand:SI 0 "s_register_operand" "=r")
11284    (zero_extend:SI
11285     (vec_select:<V_elem>
11286      (match_operand:MVE_2 1 "s_register_operand" "w")
11287      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11288   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11289    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11290 {
11291   if (BYTES_BIG_ENDIAN)
11292     {
11293       int elt = INTVAL (operands[2]);
11294       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11295       operands[2] = GEN_INT (elt);
11296     }
11297   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
11298 }
11299  [(set_attr "type" "mve_move")])
11300
11301 ;;
11302 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
11303 ;;
11304 (define_insn "mve_vec_set<mode>_internal"
11305  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
11306        (vec_merge:VQ2
11307         (vec_duplicate:VQ2
11308           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
11309         (match_operand:VQ2 3 "s_register_operand" "0")
11310         (match_operand:SI 2 "immediate_operand" "i")))]
11311   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11312    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11313 {
11314   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11315   if (BYTES_BIG_ENDIAN)
11316     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11317   operands[2] = GEN_INT (elt);
11318
11319   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
11320 }
11321  [(set_attr "type" "mve_move")])
11322
11323 (define_insn "mve_vec_setv2di_internal"
11324  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
11325        (vec_merge:V2DI
11326         (vec_duplicate:V2DI
11327           (match_operand:DI 1 "nonimmediate_operand" "r"))
11328         (match_operand:V2DI 3 "s_register_operand" "0")
11329         (match_operand:SI 2 "immediate_operand" "i")))]
11330  "TARGET_HAVE_MVE"
11331 {
11332   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11333   if (BYTES_BIG_ENDIAN)
11334     elt = 1 - elt;
11335
11336   if (elt == 0)
11337    return "vmov\t%e0, %Q1, %R1";
11338   else
11339    return "vmov\t%f0, %J1, %K1";
11340 }
11341  [(set_attr "type" "mve_move")])
11342
11343 ;;
11344 ;; [uqrshll_di]
11345 ;;
11346 (define_insn "mve_uqrshll_sat<supf>_di"
11347   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11348         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11349                     (match_operand:SI 2 "s_register_operand" "r")]
11350          UQRSHLLQ))]
11351   "TARGET_HAVE_MVE"
11352   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
11353   [(set_attr "predicable" "yes")])
11354
11355 ;;
11356 ;; [sqrshrl_di]
11357 ;;
11358 (define_insn "mve_sqrshrl_sat<supf>_di"
11359   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11360         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11361                     (match_operand:SI 2 "s_register_operand" "r")]
11362          SQRSHRLQ))]
11363   "TARGET_HAVE_MVE"
11364   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
11365   [(set_attr "predicable" "yes")])
11366
11367 ;;
11368 ;; [uqrshl_si]
11369 ;;
11370 (define_insn "mve_uqrshl_si"
11371   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11372         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11373                     (match_operand:SI 2 "s_register_operand" "r")]
11374          UQRSHL))]
11375   "TARGET_HAVE_MVE"
11376   "uqrshl%?\\t%1, %2"
11377   [(set_attr "predicable" "yes")])
11378
11379 ;;
11380 ;; [sqrshr_si]
11381 ;;
11382 (define_insn "mve_sqrshr_si"
11383   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11384         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11385                     (match_operand:SI 2 "s_register_operand" "r")]
11386          SQRSHR))]
11387   "TARGET_HAVE_MVE"
11388   "sqrshr%?\\t%1, %2"
11389   [(set_attr "predicable" "yes")])
11390
11391 ;;
11392 ;; [uqshll_di]
11393 ;;
11394 (define_insn "mve_uqshll_di"
11395   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11396         (us_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11397                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11398   "TARGET_HAVE_MVE"
11399   "uqshll%?\\t%Q1, %R1, %2"
11400   [(set_attr "predicable" "yes")])
11401
11402 ;;
11403 ;; [urshrl_di]
11404 ;;
11405 (define_insn "mve_urshrl_di"
11406   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11407         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11408                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11409          URSHRL))]
11410   "TARGET_HAVE_MVE"
11411   "urshrl%?\\t%Q1, %R1, %2"
11412   [(set_attr "predicable" "yes")])
11413
11414 ;;
11415 ;; [uqshl_si]
11416 ;;
11417 (define_insn "mve_uqshl_si"
11418   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11419         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "r")
11420                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11421   "TARGET_HAVE_MVE"
11422   "uqshl%?\\t%1, %2"
11423   [(set_attr "predicable" "yes")])
11424
11425 ;;
11426 ;; [urshr_si]
11427 ;;
11428 (define_insn "mve_urshr_si"
11429   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11430         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11431                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11432          URSHR))]
11433   "TARGET_HAVE_MVE"
11434   "urshr%?\\t%1, %2"
11435   [(set_attr "predicable" "yes")])
11436
11437 ;;
11438 ;; [sqshl_si]
11439 ;;
11440 (define_insn "mve_sqshl_si"
11441   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11442         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "r")
11443                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11444   "TARGET_HAVE_MVE"
11445   "sqshl%?\\t%1, %2"
11446   [(set_attr "predicable" "yes")])
11447
11448 ;;
11449 ;; [srshr_si]
11450 ;;
11451 (define_insn "mve_srshr_si"
11452   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11453         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "r")
11454                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11455          SRSHR))]
11456   "TARGET_HAVE_MVE"
11457   "srshr%?\\t%1, %2"
11458   [(set_attr "predicable" "yes")])
11459
11460 ;;
11461 ;; [srshrl_di]
11462 ;;
11463 (define_insn "mve_srshrl_di"
11464   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11465         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11466                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11467          SRSHRL))]
11468   "TARGET_HAVE_MVE"
11469   "srshrl%?\\t%Q1, %R1, %2"
11470   [(set_attr "predicable" "yes")])
11471
11472 ;;
11473 ;; [sqshll_di]
11474 ;;
11475 (define_insn "mve_sqshll_di"
11476   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11477         (ss_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11478                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11479   "TARGET_HAVE_MVE"
11480   "sqshll%?\\t%Q1, %R1, %2"
11481   [(set_attr "predicable" "yes")])
11482
11483 ;;
11484 ;; [vshlcq_m_u vshlcq_m_s]
11485 ;;
11486 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
11487  [(match_operand:MVE_2 0 "s_register_operand")
11488   (match_operand:MVE_2 1 "s_register_operand")
11489   (match_operand:SI 2 "s_register_operand")
11490   (match_operand:SI 3 "mve_imm_32")
11491   (match_operand:HI 4 "vpr_register_operand")
11492   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11493  "TARGET_HAVE_MVE"
11494 {
11495   rtx ignore_wb = gen_reg_rtx (SImode);
11496   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
11497                                             operands[2], operands[3],
11498                                             operands[4]));
11499   DONE;
11500 })
11501
11502 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
11503  [(match_operand:SI 0 "s_register_operand")
11504   (match_operand:MVE_2 1 "s_register_operand")
11505   (match_operand:SI 2 "s_register_operand")
11506   (match_operand:SI 3 "mve_imm_32")
11507   (match_operand:HI 4 "vpr_register_operand")
11508   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11509  "TARGET_HAVE_MVE"
11510 {
11511   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
11512   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
11513                                             operands[1], operands[2],
11514                                             operands[3], operands[4]));
11515   DONE;
11516 })
11517
11518 (define_insn "mve_vshlcq_m_<supf><mode>"
11519  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
11520        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
11521                       (match_operand:SI 3 "s_register_operand" "1")
11522                       (match_operand:SI 4 "mve_imm_32" "Rf")
11523                       (match_operand:HI 5 "vpr_register_operand" "Up")]
11524         VSHLCQ_M))
11525   (set (match_operand:SI  1 "s_register_operand" "=r")
11526        (unspec:SI [(match_dup 2)
11527                    (match_dup 3)
11528                    (match_dup 4)
11529                    (match_dup 5)]
11530         VSHLCQ_M))
11531  ]
11532  "TARGET_HAVE_MVE"
11533  "vpst\;vshlct\t%q0, %1, %4"
11534  [(set_attr "type" "mve_move")
11535   (set_attr "length" "8")])
11536
11537 (define_insn "*mve_vec_duplicate<mode>"
11538  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
11539        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
11540  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
11541  "vdup.<V_sz_elem>\t%q0, %1"
11542  [(set_attr "type" "mve_move")])
11543
11544 ;; CDE instructions on MVE registers.
11545
11546 (define_insn "arm_vcx1qv16qi"
11547   [(set (match_operand:V16QI 0 "register_operand" "=t")
11548         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11549                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
11550          UNSPEC_VCDE))]
11551   "TARGET_CDE && TARGET_HAVE_MVE"
11552   "vcx1\\tp%c1, %q0, #%c2"
11553   [(set_attr "type" "coproc")]
11554 )
11555
11556 (define_insn "arm_vcx1qav16qi"
11557   [(set (match_operand:V16QI 0 "register_operand" "=t")
11558         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11559                             (match_operand:V16QI 2 "register_operand" "0")
11560                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
11561          UNSPEC_VCDEA))]
11562   "TARGET_CDE && TARGET_HAVE_MVE"
11563   "vcx1a\\tp%c1, %q0, #%c3"
11564   [(set_attr "type" "coproc")]
11565 )
11566
11567 (define_insn "arm_vcx2qv16qi"
11568   [(set (match_operand:V16QI 0 "register_operand" "=t")
11569         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11570                           (match_operand:V16QI 2 "register_operand" "t")
11571                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
11572          UNSPEC_VCDE))]
11573   "TARGET_CDE && TARGET_HAVE_MVE"
11574   "vcx2\\tp%c1, %q0, %q2, #%c3"
11575   [(set_attr "type" "coproc")]
11576 )
11577
11578 (define_insn "arm_vcx2qav16qi"
11579   [(set (match_operand:V16QI 0 "register_operand" "=t")
11580         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11581                           (match_operand:V16QI 2 "register_operand" "0")
11582                           (match_operand:V16QI 3 "register_operand" "t")
11583                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
11584          UNSPEC_VCDEA))]
11585   "TARGET_CDE && TARGET_HAVE_MVE"
11586   "vcx2a\\tp%c1, %q0, %q3, #%c4"
11587   [(set_attr "type" "coproc")]
11588 )
11589
11590 (define_insn "arm_vcx3qv16qi"
11591   [(set (match_operand:V16QI 0 "register_operand" "=t")
11592         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11593                           (match_operand:V16QI 2 "register_operand" "t")
11594                           (match_operand:V16QI 3 "register_operand" "t")
11595                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
11596          UNSPEC_VCDE))]
11597   "TARGET_CDE && TARGET_HAVE_MVE"
11598   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
11599   [(set_attr "type" "coproc")]
11600 )
11601
11602 (define_insn "arm_vcx3qav16qi"
11603   [(set (match_operand:V16QI 0 "register_operand" "=t")
11604         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11605                           (match_operand:V16QI 2 "register_operand" "0")
11606                           (match_operand:V16QI 3 "register_operand" "t")
11607                           (match_operand:V16QI 4 "register_operand" "t")
11608                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
11609          UNSPEC_VCDEA))]
11610   "TARGET_CDE && TARGET_HAVE_MVE"
11611   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
11612   [(set_attr "type" "coproc")]
11613 )
11614
11615 (define_insn "arm_vcx1q<a>_p_v16qi"
11616   [(set (match_operand:V16QI 0 "register_operand" "=t")
11617         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11618                            (match_operand:V16QI 2 "register_operand" "0")
11619                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
11620                            (match_operand:HI 4 "vpr_register_operand" "Up")]
11621          CDE_VCX))]
11622   "TARGET_CDE && TARGET_HAVE_MVE"
11623   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
11624   [(set_attr "type" "coproc")
11625    (set_attr "length" "8")]
11626 )
11627
11628 (define_insn "arm_vcx2q<a>_p_v16qi"
11629   [(set (match_operand:V16QI 0 "register_operand" "=t")
11630         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11631                           (match_operand:V16QI 2 "register_operand" "0")
11632                           (match_operand:V16QI 3 "register_operand" "t")
11633                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
11634                           (match_operand:HI 5 "vpr_register_operand" "Up")]
11635          CDE_VCX))]
11636   "TARGET_CDE && TARGET_HAVE_MVE"
11637   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
11638   [(set_attr "type" "coproc")
11639    (set_attr "length" "8")]
11640 )
11641
11642 (define_insn "arm_vcx3q<a>_p_v16qi"
11643   [(set (match_operand:V16QI 0 "register_operand" "=t")
11644         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11645                           (match_operand:V16QI 2 "register_operand" "0")
11646                           (match_operand:V16QI 3 "register_operand" "t")
11647                           (match_operand:V16QI 4 "register_operand" "t")
11648                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
11649                           (match_operand:HI 6 "vpr_register_operand" "Up")]
11650          CDE_VCX))]
11651   "TARGET_CDE && TARGET_HAVE_MVE"
11652   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
11653   [(set_attr "type" "coproc")
11654    (set_attr "length" "8")]
11655 )