gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_mode_iterator MVE_types [V16QI V8HI V4SI V2DI TI V8HF V4SF V2DF])
  21 (define_mode_iterator MVE_VLD_ST [V16QI V8HI V4SI V8HF V4SF])
  22 (define_mode_iterator MVE_0 [V8HF V4SF])
  23 (define_mode_iterator MVE_1 [V16QI V8HI V4SI V2DI])
  24 (define_mode_iterator MVE_3 [V16QI V8HI])
  25 (define_mode_iterator MVE_2 [V16QI V8HI V4SI])
  26 (define_mode_iterator MVE_5 [V8HI V4SI])
  27 (define_mode_iterator MVE_6 [V8HI V4SI])
  28
  29 (define_c_enum "unspec" [VST4Q VRNDXQ_F VRNDQ_F VRNDPQ_F VRNDNQ_F VRNDMQ_F
  30                          VRNDAQ_F VREV64Q_F VNEGQ_F VDUPQ_N_F VABSQ_F VREV32Q_F
  31                          VCVTTQ_F32_F16 VCVTBQ_F32_F16 VCVTQ_TO_F_S VQNEGQ_S
  32                          VCVTQ_TO_F_U VREV16Q_S VREV16Q_U VADDLVQ_S VMVNQ_N_S
  33                          VMVNQ_N_U VCVTAQ_S VCVTAQ_U VREV64Q_S VREV64Q_U
  34                          VQABSQ_S VNEGQ_S VMVNQ_S VMVNQ_U VDUPQ_N_U VDUPQ_N_S
  35                          VCLZQ_U VCLZQ_S VCLSQ_S VADDVQ_S VADDVQ_U VABSQ_S
  36                          VREV32Q_U VREV32Q_S VMOVLTQ_U VMOVLTQ_S VMOVLBQ_S
  37                          VMOVLBQ_U VCVTQ_FROM_F_S VCVTQ_FROM_F_U VCVTPQ_S
  38                          VCVTPQ_U VCVTNQ_S VCVTNQ_U VCVTMQ_S VCVTMQ_U
  39                          VADDLVQ_U VCTP8Q VCTP16Q VCTP32Q VCTP64Q VPNOT
  40                          VCREATEQ_F VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U VBRSRQ_N_F
  41                          VSUBQ_N_F VCREATEQ_U VCREATEQ_S VSHRQ_N_S VSHRQ_N_U
  42                          VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U VADDLVQ_P_S
  43                          VADDLVQ_P_U VCMPNEQ_U VCMPNEQ_S VSHLQ_S VSHLQ_U VABDQ_S
  44                          VADDQ_N_S VADDVAQ_S VADDVQ_P_S VANDQ_S VBICQ_S
  45                          VBRSRQ_N_S VCADDQ_ROT270_S VCADDQ_ROT90_S VCMPEQQ_S
  46                          VCMPEQQ_N_S VCMPNEQ_N_S VEORQ_S VHADDQ_S VHADDQ_N_S
  47                          VHSUBQ_S VHSUBQ_N_S VMAXQ_S VMAXVQ_S VMINQ_S VMINVQ_S
  48                          VMLADAVQ_S VMULHQ_S VMULLBQ_INT_S VMULLTQ_INT_S VMULQ_S
  49                          VMULQ_N_S VORNQ_S VORRQ_S VQADDQ_S VQADDQ_N_S VQRSHLQ_S
  50                          VQRSHLQ_N_S VQSHLQ_S VQSHLQ_N_S VQSHLQ_R_S VQSUBQ_S
  51                          VQSUBQ_N_S VRHADDQ_S VRMULHQ_S VRSHLQ_S VRSHLQ_N_S
  52                          VRSHRQ_N_S VSHLQ_N_S VSHLQ_R_S VSUBQ_S VSUBQ_N_S
  53                          VABDQ_U VADDQ_N_U VADDVAQ_U VADDVQ_P_U VANDQ_U VBICQ_U
  54                          VBRSRQ_N_U VCADDQ_ROT270_U VCADDQ_ROT90_U VCMPEQQ_U
  55                          VCMPEQQ_N_U VCMPNEQ_N_U VEORQ_U VHADDQ_U VHADDQ_N_U
  56                          VHSUBQ_U VHSUBQ_N_U VMAXQ_U VMAXVQ_U VMINQ_U VMINVQ_U
  57                          VMLADAVQ_U VMULHQ_U VMULLBQ_INT_U VMULLTQ_INT_U VMULQ_U
  58                          VMULQ_N_U VORNQ_U VORRQ_U VQADDQ_U VQADDQ_N_U VQRSHLQ_U
  59                          VQRSHLQ_N_U VQSHLQ_U VQSHLQ_N_U VQSHLQ_R_U VQSUBQ_U
  60                          VQSUBQ_N_U VRHADDQ_U VRMULHQ_U VRSHLQ_U VRSHLQ_N_U
  61                          VRSHRQ_N_U VSHLQ_N_U VSHLQ_R_U VSUBQ_U VSUBQ_N_U
  62                          VCMPGEQ_N_S VCMPGEQ_S VCMPGTQ_N_S VCMPGTQ_S VCMPLEQ_N_S
  63                          VCMPLEQ_S VCMPLTQ_N_S VCMPLTQ_S VHCADDQ_ROT270_S
  64                          VHCADDQ_ROT90_S VMAXAQ_S VMAXAVQ_S VMINAQ_S VMINAVQ_S
  65                          VMLADAVXQ_S VMLSDAVQ_S VMLSDAVXQ_S VQDMULHQ_N_S
  66                          VQDMULHQ_S VQRDMULHQ_N_S VQRDMULHQ_S VQSHLUQ_N_S
  67                          VCMPCSQ_N_U VCMPCSQ_U VCMPHIQ_N_U VCMPHIQ_U VABDQ_M_S
  68                          VABDQ_M_U VABDQ_F VADDQ_N_F VANDQ_F VBICQ_F
  69                          VCADDQ_ROT270_F VCADDQ_ROT90_F VCMPEQQ_F VCMPEQQ_N_F
  70                          VCMPGEQ_F VCMPGEQ_N_F VCMPGTQ_F VCMPGTQ_N_F VCMPLEQ_F
  71                          VCMPLEQ_N_F VCMPLTQ_F VCMPLTQ_N_F VCMPNEQ_F VCMPNEQ_N_F
  72                          VCMULQ_F VCMULQ_ROT180_F VCMULQ_ROT270_F VCMULQ_ROT90_F
  73                          VEORQ_F VMAXNMAQ_F VMAXNMAVQ_F VMAXNMQ_F VMAXNMVQ_F
  74                          VMINNMAQ_F VMINNMAVQ_F VMINNMQ_F VMINNMVQ_F VMULQ_F
  75                          VMULQ_N_F VORNQ_F VORRQ_F VSUBQ_F VADDLVAQ_U
  76                          VADDLVAQ_S VBICQ_N_U VBICQ_N_S VCTP8Q_M VCTP16Q_M
  77                          VCTP32Q_M VCTP64Q_M VCVTBQ_F16_F32 VCVTTQ_F16_F32
  78                          VMLALDAVQ_U VMLALDAVXQ_U VMLALDAVXQ_S VMLALDAVQ_S
  79                          VMLSLDAVQ_S VMLSLDAVXQ_S VMOVNBQ_U VMOVNBQ_S
  80                          VMOVNTQ_U VMOVNTQ_S VORRQ_N_S VORRQ_N_U VQDMULLBQ_N_S
  81                          VQDMULLBQ_S VQDMULLTQ_N_S VQDMULLTQ_S VQMOVNBQ_U
  82                          VQMOVNBQ_S VQMOVUNBQ_S VQMOVUNTQ_S VRMLALDAVHXQ_S
  83                          VRMLSLDAVHQ_S VRMLSLDAVHXQ_S VSHLLBQ_S
  84                          VSHLLBQ_U VSHLLTQ_U VSHLLTQ_S VQMOVNTQ_U VQMOVNTQ_S
  85                          VSHLLBQ_N_S VSHLLBQ_N_U VSHLLTQ_N_U VSHLLTQ_N_S
  86                          VRMLALDAVHQ_U VRMLALDAVHQ_S VMULLTQ_POLY_P
  87                          VMULLBQ_POLY_P VBICQ_M_N_S VBICQ_M_N_U VCMPEQQ_M_F
  88                          VCVTAQ_M_S VCVTAQ_M_U VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U
  89                          VQRSHRNBQ_N_U VQRSHRNBQ_N_S VQRSHRUNBQ_N_S
  90                          VRMLALDAVHAQ_S VABAVQ_S VABAVQ_U VSHLCQ_S VSHLCQ_U
  91                          VRMLALDAVHAQ_U VABSQ_M_S VADDVAQ_P_S VADDVAQ_P_U
  92                          VCLSQ_M_S VCLZQ_M_S VCLZQ_M_U VCMPCSQ_M_N_U
  93                          VCMPCSQ_M_U VCMPEQQ_M_N_S VCMPEQQ_M_N_U VCMPEQQ_M_S
  94                          VCMPEQQ_M_U VCMPGEQ_M_N_S VCMPGEQ_M_S VCMPGTQ_M_N_S
  95                          VCMPGTQ_M_S VCMPHIQ_M_N_U VCMPHIQ_M_U VCMPLEQ_M_N_S
  96                          VCMPLEQ_M_S VCMPLTQ_M_N_S VCMPLTQ_M_S VCMPNEQ_M_N_S
  97                          VCMPNEQ_M_N_U VCMPNEQ_M_S VCMPNEQ_M_U VDUPQ_M_N_S
  98                          VDUPQ_M_N_U VDWDUPQ_N_U VDWDUPQ_WB_U VIWDUPQ_N_U
  99                          VIWDUPQ_WB_U VMAXAQ_M_S VMAXAVQ_P_S VMAXVQ_P_S
 100                          VMAXVQ_P_U VMINAQ_M_S VMINAVQ_P_S VMINVQ_P_S VMINVQ_P_U
 101                          VMLADAVAQ_S VMLADAVAQ_U VMLADAVQ_P_S VMLADAVQ_P_U
 102                          VMLADAVXQ_P_S VMLAQ_N_S VMLAQ_N_U VMLASQ_N_S VMLASQ_N_U
 103                          VMLSDAVQ_P_S VMLSDAVXQ_P_S VMVNQ_M_S VMVNQ_M_U
 104                          VNEGQ_M_S VPSELQ_S VPSELQ_U VQABSQ_M_S VQDMLAHQ_N_S
 105                          VQDMLAHQ_N_U VQNEGQ_M_S VQRDMLADHQ_S VQRDMLADHXQ_S
 106                          VQRDMLAHQ_N_S VQRDMLAHQ_N_U VQRDMLASHQ_N_S
 107                          VQRDMLASHQ_N_U VQRDMLSDHQ_S VQRDMLSDHXQ_S VQRSHLQ_M_N_S
 108                          VQRSHLQ_M_N_U VQSHLQ_M_R_S VQSHLQ_M_R_U VREV64Q_M_S
 109                          VREV64Q_M_U VRSHLQ_M_N_S VRSHLQ_M_N_U VSHLQ_M_R_S
 110                          VSHLQ_M_R_U VSLIQ_N_S VSLIQ_N_U VSRIQ_N_S VSRIQ_N_U
 111                          VQDMLSDHXQ_S VQDMLSDHQ_S VQDMLADHXQ_S VQDMLADHQ_S
 112                          VMLSDAVAXQ_S VMLSDAVAQ_S VMLADAVAXQ_S
 113                          VCMPGEQ_M_F VCMPGTQ_M_N_F VMLSLDAVQ_P_S VRMLALDAVHAXQ_S
 114                          VMLSLDAVXQ_P_S VFMAQ_F VMLSLDAVAQ_S VQSHRUNBQ_N_S
 115                          VQRSHRUNTQ_N_S VCMLAQ_F VMINNMAQ_M_F VFMASQ_N_F
 116                          VDUPQ_M_N_F VCMPGTQ_M_F VCMPLTQ_M_F VRMLSLDAVHQ_P_S
 117                          VQSHRUNTQ_N_S VABSQ_M_F VMAXNMAVQ_P_F VFMAQ_N_F
 118                          VRMLSLDAVHXQ_P_S VREV32Q_M_F VRMLSLDAVHAQ_S
 119                          VRMLSLDAVHAXQ_S VCMPLTQ_M_N_F VCMPNEQ_M_F VRNDAQ_M_F
 120                          VRNDPQ_M_F VADDLVAQ_P_S VQMOVUNBQ_M_S VCMPLEQ_M_F
 121                          VCMLAQ_ROT180_F VMLSLDAVAXQ_S VRNDXQ_M_F VFMSQ_F
 122                          VMINNMVQ_P_F VMAXNMVQ_P_F VPSELQ_F VCMLAQ_ROT90_F
 123                          VQMOVUNTQ_M_S VREV64Q_M_F VNEGQ_M_F VRNDMQ_M_F
 124                          VCMPLEQ_M_N_F VCMPGEQ_M_N_F VRNDNQ_M_F VMINNMAVQ_P_F
 125                          VCMPNEQ_M_N_F VRMLALDAVHQ_P_S VRMLALDAVHXQ_P_S
 126                          VCMPEQQ_M_N_F VCMLAQ_ROT270_F VMAXNMAQ_M_F VRNDQ_M_F
 127                          VMLALDAVQ_P_U VMLALDAVQ_P_S VQMOVNBQ_M_S VQMOVNBQ_M_U
 128                          VMOVLTQ_M_U VMOVLTQ_M_S VMOVNBQ_M_U VMOVNBQ_M_S
 129                          VRSHRNTQ_N_U VRSHRNTQ_N_S VORRQ_M_N_S VORRQ_M_N_U
 130                          VREV32Q_M_S VREV32Q_M_U VQRSHRNTQ_N_U VQRSHRNTQ_N_S
 131                          VMOVNTQ_M_U VMOVNTQ_M_S VMOVLBQ_M_U VMOVLBQ_M_S
 132                          VMLALDAVAQ_S VMLALDAVAQ_U VQSHRNBQ_N_U VQSHRNBQ_N_S
 133                          VSHRNBQ_N_U VSHRNBQ_N_S VRSHRNBQ_N_S VRSHRNBQ_N_U
 134                          VMLALDAVXQ_P_U VMLALDAVXQ_P_S VQMOVNTQ_M_U VQMOVNTQ_M_S
 135                          VMVNQ_M_N_U VMVNQ_M_N_S VQSHRNTQ_N_U VQSHRNTQ_N_S
 136                          VMLALDAVAXQ_S VMLALDAVAXQ_U VSHRNTQ_N_S VSHRNTQ_N_U
 137                          VCVTBQ_M_F16_F32 VCVTBQ_M_F32_F16 VCVTTQ_M_F16_F32
 138                          VCVTTQ_M_F32_F16 VCVTMQ_M_S VCVTMQ_M_U VCVTNQ_M_S
 139                          VCVTPQ_M_S VCVTPQ_M_U VCVTQ_M_N_FROM_F_S VCVTNQ_M_U
 140                          VREV16Q_M_S VREV16Q_M_U VREV32Q_M VCVTQ_M_FROM_F_U
 141                          VCVTQ_M_FROM_F_S VRMLALDAVHQ_P_U VADDLVAQ_P_U
 142                          VCVTQ_M_N_FROM_F_U VQSHLUQ_M_N_S VABAVQ_P_S
 143                          VABAVQ_P_U VSHLQ_M_S VSHLQ_M_U VSRIQ_M_N_S
 144                          VSRIQ_M_N_U VSUBQ_M_U VSUBQ_M_S VCVTQ_M_N_TO_F_U
 145                          VCVTQ_M_N_TO_F_S VQADDQ_M_U VQADDQ_M_S
 146                          VRSHRQ_M_N_S VSUBQ_M_N_S VSUBQ_M_N_U VBRSRQ_M_N_S
 147                          VSUBQ_M_N_F VBICQ_M_F VHADDQ_M_U VBICQ_M_U VBICQ_M_S
 148                          VMULQ_M_N_U VHADDQ_M_S VORNQ_M_F VMLAQ_M_N_S VQSUBQ_M_U
 149                          VQSUBQ_M_S VMLAQ_M_N_U VQSUBQ_M_N_U VQSUBQ_M_N_S
 150                          VMULLTQ_INT_M_S VMULLTQ_INT_M_U VMULQ_M_N_S VMULQ_M_N_F
 151                          VMLASQ_M_N_U VMLASQ_M_N_S VMAXQ_M_U VQRDMLAHQ_M_N_U
 152                          VCADDQ_ROT270_M_F VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S
 153                          VQRSHLQ_M_S VMULQ_M_F VRHADDQ_M_U VSHRQ_M_N_U
 154                          VRHADDQ_M_S VMULQ_M_S VMULQ_M_U VQRDMLASHQ_M_N_S
 155                          VRSHLQ_M_S VRSHLQ_M_U VRSHRQ_M_N_U VADDQ_M_N_F
 156                          VADDQ_M_N_S VADDQ_M_N_U VQRDMLASHQ_M_N_U VMAXQ_M_S
 157                          VQRDMLAHQ_M_N_S VORRQ_M_S VORRQ_M_U VORRQ_M_F
 158                          VQRSHLQ_M_U VRMULHQ_M_U VRMULHQ_M_S VMINQ_M_S VMINQ_M_U
 159                          VANDQ_M_F VANDQ_M_U VANDQ_M_S VHSUBQ_M_N_S VHSUBQ_M_N_U
 160                          VMULHQ_M_S VMULHQ_M_U VMULLBQ_INT_M_U
 161                          VMULLBQ_INT_M_S VCADDQ_ROT90_M_F
 162                          VSHRQ_M_N_S VADDQ_M_U VSLIQ_M_N_U
 163                          VQADDQ_M_N_S VBRSRQ_M_N_F VABDQ_M_F VBRSRQ_M_N_U
 164                          VEORQ_M_F VSHLQ_M_N_S VQDMLAHQ_M_N_U VQDMLAHQ_M_N_S
 165                          VSHLQ_M_N_U VMLADAVAQ_P_U VMLADAVAQ_P_S VSLIQ_M_N_S
 166                          VQSHLQ_M_U VQSHLQ_M_S VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S
 167                          VORNQ_M_U VORNQ_M_S VQSHLQ_M_N_S VQSHLQ_M_N_U VADDQ_M_S
 168                          VHADDQ_M_N_S VADDQ_M_F VQADDQ_M_N_U VEORQ_M_S VEORQ_M_U
 169                          VHSUBQ_M_S VHSUBQ_M_U VHADDQ_M_N_U VHCADDQ_ROT90_M_S
 170                          VQRDMLSDHQ_M_S VQRDMLSDHXQ_M_S VQRDMLADHXQ_M_S
 171                          VQDMULHQ_M_S VMLADAVAXQ_P_S VQDMLADHXQ_M_S
 172                          VQRDMULHQ_M_S VMLSDAVAXQ_P_S VQDMULHQ_M_N_S
 173                          VHCADDQ_ROT270_M_S VQDMLSDHQ_M_S VQDMLSDHXQ_M_S
 174                          VMLSDAVAQ_P_S VQRDMLADHQ_M_S VQDMLADHQ_M_S
 175                          VMLALDAVAQ_P_U VMLALDAVAQ_P_S VMLALDAVAXQ_P_U
 176                          VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S VQRSHRNTQ_M_N_S
 177                          VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S VQSHRNTQ_M_N_S
 178                          VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S VRSHRNTQ_M_N_U
 179                          VSHLLBQ_M_N_U VSHLLBQ_M_N_S VSHLLTQ_M_N_U VSHLLTQ_M_N_S
 180                          VSHRNBQ_M_N_S VSHRNBQ_M_N_U VSHRNTQ_M_N_S VSHRNTQ_M_N_U
 181                          VMLALDAVAXQ_P_S VQRSHRNTQ_M_N_U VQSHRNTQ_M_N_U
 182                          VRSHRNTQ_M_N_S VQRDMULHQ_M_N_S VRMLALDAVHAQ_P_S
 183                          VMLSLDAVAQ_P_S VMLSLDAVAXQ_P_S VMULLBQ_POLY_M_P
 184                          VMULLTQ_POLY_M_P VQDMULLBQ_M_N_S VQDMULLBQ_M_S
 185                          VQDMULLTQ_M_N_S VQDMULLTQ_M_S VQRSHRUNBQ_M_N_S
 186                          VQRSHRUNTQ_M_N_SVQSHRUNBQ_M_N_S VQSHRUNTQ_M_N_S
 187                          VRMLALDAVHAQ_P_U VRMLALDAVHAXQ_P_S VRMLSLDAVHAQ_P_S
 188                          VRMLSLDAVHAXQ_P_S VQRSHRUNTQ_M_N_S VQSHRUNBQ_M_N_S
 189                          VCMLAQ_M_F VCMLAQ_ROT180_M_F VCMLAQ_ROT270_M_F
 190                          VCMLAQ_ROT90_M_F VCMULQ_M_F VCMULQ_ROT180_M_F
 191                          VCMULQ_ROT270_M_F VCMULQ_ROT90_M_F VFMAQ_M_F
 192                          VFMAQ_M_N_F VFMASQ_M_N_F VFMSQ_M_F VMAXNMQ_M_F
 193                          VMINNMQ_M_F VSUBQ_M_F VSTRWQSB_S VSTRWQSB_U
 194                          VSTRBQSO_S VSTRBQSO_U VSTRBQ_S VSTRBQ_U VLDRBQGO_S
 195                          VLDRBQGO_U VLDRBQ_S VLDRBQ_U VLDRWQGB_S VLDRWQGB_U
 196                          VLD1Q_F VLD1Q_S VLD1Q_U VLDRHQ_F VLDRHQGO_S
 197                          VLDRHQGO_U VLDRHQGSO_S VLDRHQGSO_U VLDRHQ_S VLDRHQ_U
 198                          VLDRWQ_F VLDRWQ_S VLDRWQ_U VLDRDQGB_S VLDRDQGB_U
 199                          VLDRDQGO_S VLDRDQGO_U VLDRDQGSO_S VLDRDQGSO_U
 200                          VLDRHQGO_F VLDRHQGSO_F VLDRWQGB_F VLDRWQGO_F
 201                          VLDRWQGO_S VLDRWQGO_U VLDRWQGSO_F VLDRWQGSO_S
 202                          VLDRWQGSO_U VSTRHQ_F VST1Q_S VST1Q_U VSTRHQSO_S
 203                          VSTRHQSO_U VSTRHQSSO_S VSTRHQSSO_U VSTRHQ_S
 204                          VSTRHQ_U VSTRWQ_S VSTRWQ_U VSTRWQ_F VST1Q_F VSTRDQSB_S
 205                          VSTRDQSB_U VSTRDQSO_S VSTRDQSO_U VSTRDQSSO_S
 206                          VSTRDQSSO_U VSTRWQSO_S VSTRWQSO_U VSTRWQSSO_S
 207                          VSTRWQSSO_U VSTRHQSO_F VSTRHQSSO_F VSTRWQSB_F
 208                          VSTRWQSO_F VSTRWQSSO_F VDDUPQ VDDUPQ_M VDWDUPQ
 209                          VDWDUPQ_M VIDUPQ VIDUPQ_M VIWDUPQ VIWDUPQ_M
 210                          VSTRWQSBWB_S VSTRWQSBWB_U VLDRWQGBWB_S VLDRWQGBWB_U
 211                          VSTRWQSBWB_F VLDRWQGBWB_F VSTRDQSBWB_S VSTRDQSBWB_U
 212                          VLDRDQGBWB_S VLDRDQGBWB_U VADCQ_U VADCQ_M_U VADCQ_S
 213                          VADCQ_M_S VSBCIQ_U VSBCIQ_S VSBCIQ_M_U VSBCIQ_M_S
 214                          VSBCQ_U VSBCQ_S VSBCQ_M_U VSBCQ_M_S VADCIQ_U VADCIQ_M_U
 215                          VADCIQ_S VADCIQ_M_S VLD2Q VLD4Q VST2Q SRSHRL SRSHR
 216                          URSHR URSHRL SQRSHR UQRSHL UQRSHLL_64 VSHLCQ_M_U
 217                          UQRSHLL_48 SQRSHRL_64 SQRSHRL_48 VSHLCQ_M_S])
 218
 219 (define_mode_attr MVE_CNVT [(V8HI "V8HF") (V4SI "V4SF") (V8HF "V8HI")
 220                             (V4SF "V4SI")])
 221
 222 (define_int_attr supf [(VCVTQ_TO_F_S "s") (VCVTQ_TO_F_U "u") (VREV16Q_S "s")
 223                        (VREV16Q_U "u") (VMVNQ_N_S "s") (VMVNQ_N_U "u")
 224                        (VCVTAQ_U "u") (VCVTAQ_S "s") (VREV64Q_S "s")
 225                        (VREV64Q_U "u") (VMVNQ_S "s") (VMVNQ_U "u")
 226                        (VDUPQ_N_U "u") (VDUPQ_N_S"s") (VADDVQ_S "s")
 227                        (VADDVQ_U "u") (VADDVQ_S "s") (VADDVQ_U "u")
 228                        (VMOVLTQ_U "u") (VMOVLTQ_S "s") (VMOVLBQ_S "s")
 229                        (VMOVLBQ_U "u") (VCVTQ_FROM_F_S "s") (VCVTQ_FROM_F_U "u")
 230                        (VCVTPQ_S "s") (VCVTPQ_U "u") (VCVTNQ_S "s")
 231                        (VCVTNQ_U "u") (VCVTMQ_S "s") (VCVTMQ_U "u")
 232                        (VCLZQ_U "u") (VCLZQ_S "s") (VREV32Q_U "u")
 233                        (VREV32Q_S "s") (VADDLVQ_U "u") (VADDLVQ_S "s")
 234                        (VCVTQ_N_TO_F_S "s") (VCVTQ_N_TO_F_U "u")
 235                        (VCREATEQ_U "u") (VCREATEQ_S "s") (VSHRQ_N_S "s")
 236                        (VSHRQ_N_U "u") (VCVTQ_N_FROM_F_S "s") (VSHLQ_U "u")
 237                        (VCVTQ_N_FROM_F_U "u") (VADDLVQ_P_S "s") (VSHLQ_S "s")
 238                        (VADDLVQ_P_U "u") (VCMPNEQ_U "u") (VCMPNEQ_S "s")
 239                        (VABDQ_M_S "s") (VABDQ_M_U "u") (VABDQ_S "s")
 240                        (VABDQ_U "u") (VADDQ_N_S "s") (VADDQ_N_U "u")
 241                        (VADDVQ_P_S "s") (VADDVQ_P_U "u") (VANDQ_S "s")
 242                        (VANDQ_U "u") (VBICQ_S "s") (VBICQ_U "u")
 243                        (VBRSRQ_N_S "s") (VBRSRQ_N_U "u") (VCADDQ_ROT270_S "s")
 244                        (VCADDQ_ROT270_U "u") (VCADDQ_ROT90_S "s")
 245                        (VCMPEQQ_S "s") (VCMPEQQ_U "u") (VCADDQ_ROT90_U "u")
 246                        (VCMPEQQ_N_S "s") (VCMPEQQ_N_U "u") (VCMPNEQ_N_S "s")
 247                        (VCMPNEQ_N_U "u") (VEORQ_S "s") (VEORQ_U "u")
 248                        (VHADDQ_N_S "s") (VHADDQ_N_U "u") (VHADDQ_S "s")
 249                        (VHADDQ_U "u") (VHSUBQ_N_S "s")  (VHSUBQ_N_U "u")
 250                        (VHSUBQ_S "s") (VMAXQ_S "s") (VMAXQ_U "u") (VHSUBQ_U "u")
 251                        (VMAXVQ_S "s") (VMAXVQ_U "u") (VMINQ_S "s") (VMINQ_U "u")
 252                        (VMINVQ_S "s") (VMINVQ_U "u") (VMLADAVQ_S "s")
 253                        (VMLADAVQ_U "u") (VMULHQ_S "s") (VMULHQ_U "u")
 254                        (VMULLBQ_INT_S "s") (VMULLBQ_INT_U "u") (VQADDQ_S "s")
 255                        (VMULLTQ_INT_S "s") (VMULLTQ_INT_U "u") (VQADDQ_U "u")
 256                        (VMULQ_N_S "s") (VMULQ_N_U "u") (VMULQ_S "s")
 257                        (VMULQ_U "u") (VORNQ_S "s") (VORNQ_U "u") (VORRQ_S "s")
 258                        (VORRQ_U "u") (VQADDQ_N_S "s") (VQADDQ_N_U "u")
 259                        (VQRSHLQ_N_S "s") (VQRSHLQ_N_U "u") (VQRSHLQ_S "s")
 260                        (VQRSHLQ_U "u") (VQSHLQ_N_S "s") (VQSHLQ_N_U "u")
 261                        (VQSHLQ_R_S "s") (VQSHLQ_R_U "u") (VQSHLQ_S "s")
 262                        (VQSHLQ_U "u") (VQSUBQ_N_S "s") (VQSUBQ_N_U "u")
 263                        (VQSUBQ_S "s") (VQSUBQ_U "u") (VRHADDQ_S "s")
 264                        (VRHADDQ_U "u") (VRMULHQ_S "s") (VRMULHQ_U "u")
 265                        (VRSHLQ_N_S "s") (VRSHLQ_N_U "u") (VRSHLQ_S "s")
 266                        (VRSHLQ_U "u") (VRSHRQ_N_S "s") (VRSHRQ_N_U "u")
 267                        (VSHLQ_N_S "s") (VSHLQ_N_U "u") (VSHLQ_R_S "s")
 268                        (VSHLQ_R_U "u") (VSUBQ_N_S "s") (VSUBQ_N_U "u")
 269                        (VSUBQ_S "s") (VSUBQ_U "u") (VADDVAQ_S "s")
 270                        (VADDVAQ_U "u") (VADDLVAQ_S "s") (VADDLVAQ_U "u")
 271                        (VBICQ_N_S "s") (VBICQ_N_U "u") (VMLALDAVQ_U "u")
 272                        (VMLALDAVQ_S "s") (VMLALDAVXQ_U "u") (VMLALDAVXQ_S "s")
 273                        (VMOVNBQ_U "u") (VMOVNBQ_S "s") (VMOVNTQ_U "u")
 274                        (VMOVNTQ_S "s") (VORRQ_N_S "s") (VORRQ_N_U "u")
 275                        (VQMOVNBQ_U "u") (VQMOVNBQ_S "s") (VQMOVNTQ_S "s")
 276                        (VQMOVNTQ_U "u") (VSHLLBQ_N_U "u") (VSHLLBQ_N_S "s")
 277                        (VSHLLTQ_N_U "u") (VSHLLTQ_N_S "s") (VRMLALDAVHQ_U "u")
 278                        (VRMLALDAVHQ_S "s") (VBICQ_M_N_S "s") (VBICQ_M_N_U "u")
 279                        (VCVTAQ_M_S "s") (VCVTAQ_M_U "u") (VCVTQ_M_TO_F_S "s")
 280                        (VCVTQ_M_TO_F_U "u") (VQRSHRNBQ_N_S "s")
 281                        (VQRSHRNBQ_N_U "u") (VABAVQ_S "s") (VABAVQ_U "u")
 282                        (VRMLALDAVHAQ_U "u") (VRMLALDAVHAQ_S "s") (VSHLCQ_S "s")
 283                        (VSHLCQ_U "u") (VADDVAQ_P_S "s") (VADDVAQ_P_U "u")
 284                        (VCLZQ_M_S "s") (VCLZQ_M_U "u") (VCMPEQQ_M_N_S "s")
 285                        (VCMPEQQ_M_N_U "u") (VCMPEQQ_M_S "s") (VCMPEQQ_M_U "u")
 286                        (VCMPNEQ_M_N_S "s") (VCMPNEQ_M_N_U "u") (VCMPNEQ_M_S "s")
 287                        (VCMPNEQ_M_U "u") (VDUPQ_M_N_S "s") (VDUPQ_M_N_U "u")
 288                        (VMAXVQ_P_S "s") (VMAXVQ_P_U "u") (VMINVQ_P_S "s")
 289                        (VMINVQ_P_U "u") (VMLADAVAQ_S "s") (VMLADAVAQ_U "u")
 290                        (VMLADAVQ_P_S "s") (VMLADAVQ_P_U "u") (VMLAQ_N_S "s")
 291                        (VMLAQ_N_U "u") (VMLASQ_N_S "s") (VMLASQ_N_U "u")
 292                        (VMVNQ_M_S "s") (VMVNQ_M_U "u") (VPSELQ_S "s")
 293                        (VPSELQ_U "u") (VQDMLAHQ_N_S "s") (VQDMLAHQ_N_U "u")
 294                        (VQRDMLAHQ_N_S "s") (VQRDMLAHQ_N_U "u")
 295                        (VQRDMLASHQ_N_S "s") (VQRDMLASHQ_N_U "u")
 296                        (VQRSHLQ_M_N_S "s") (VQRSHLQ_M_N_U "u")
 297                        (VQSHLQ_M_R_S "s") (VQSHLQ_M_R_U "u") (VSRIQ_N_S "s")
 298                        (VREV64Q_M_S "s") (VREV64Q_M_U "u") (VSRIQ_N_U "u")
 299                        (VRSHLQ_M_N_S "s") (VRSHLQ_M_N_U "u") (VSHLQ_M_R_S "s")
 300                        (VSHLQ_M_R_U "u") (VSLIQ_N_S "s") (VSLIQ_N_U "u")
 301                        (VMLALDAVQ_P_S "s") (VQMOVNBQ_M_S "s") (VMOVLTQ_M_S "s")
 302                        (VMOVNBQ_M_S "s") (VRSHRNTQ_N_S "s") (VORRQ_M_N_S "s")
 303                        (VREV32Q_M_S "s") (VQRSHRNTQ_N_S "s") (VMOVNTQ_M_S "s")
 304                        (VMOVLBQ_M_S "s") (VMLALDAVAQ_S "s") (VQSHRNBQ_N_S "s")
 305                        (VSHRNBQ_N_S "s") (VRSHRNBQ_N_S "s") (VMLALDAVXQ_P_S "s")
 306                        (VQMOVNTQ_M_S "s") (VMVNQ_M_N_S "s") (VQSHRNTQ_N_S "s")
 307                        (VMLALDAVAXQ_S "s") (VSHRNTQ_N_S "s") (VMLALDAVQ_P_U "u")
 308                        (VQMOVNBQ_M_U "u") (VMOVLTQ_M_U "u") (VMOVNBQ_M_U "u")
 309                        (VRSHRNTQ_N_U "u") (VORRQ_M_N_U "u") (VREV32Q_M_U "u")
 310                        (VREV16Q_M_S "s") (VREV16Q_M_U "u")
 311                        (VQRSHRNTQ_N_U "u") (VMOVNTQ_M_U "u") (VMOVLBQ_M_U "u")
 312                        (VMLALDAVAQ_U "u") (VQSHRNBQ_N_U "u") (VSHRNBQ_N_U "u")
 313                        (VRSHRNBQ_N_U "u") (VMLALDAVXQ_P_U "u")
 314                        (VMVNQ_M_N_U "u") (VQSHRNTQ_N_U "u") (VMLALDAVAXQ_U "u")
 315                        (VQMOVNTQ_M_U "u") (VSHRNTQ_N_U "u") (VCVTMQ_M_S "s")
 316                        (VCVTMQ_M_U "u") (VCVTNQ_M_S "s") (VCVTNQ_M_U "u")
 317                        (VCVTPQ_M_S "s") (VCVTPQ_M_U "u") (VADDLVAQ_P_S "s")
 318                        (VCVTQ_M_N_FROM_F_U "u") (VCVTQ_M_FROM_F_S "s")
 319                        (VCVTQ_M_FROM_F_U "u") (VRMLALDAVHQ_P_U "u")
 320                        (VRMLALDAVHQ_P_S "s") (VADDLVAQ_P_U "u")
 321                        (VCVTQ_M_N_FROM_F_S "s") (VABAVQ_P_U "u")
 322                        (VABAVQ_P_S "s") (VSHLQ_M_S "s") (VSHLQ_M_U "u")
 323                        (VSRIQ_M_N_S "s") (VSRIQ_M_N_U "u") (VSUBQ_M_S "s")
 324                        (VSUBQ_M_U "u") (VCVTQ_M_N_TO_F_S "s")
 325                        (VCVTQ_M_N_TO_F_U "u") (VADDQ_M_N_U "u")
 326                        (VSHLQ_M_N_S "s") (VMAXQ_M_U "u") (VHSUBQ_M_N_U "u")
 327                        (VMULQ_M_N_S "s") (VQSHLQ_M_U "u") (VRHADDQ_M_S "s")
 328                        (VEORQ_M_U "u") (VSHRQ_M_N_U "u") (VCADDQ_ROT90_M_U "u")
 329                        (VMLADAVAQ_P_U "u") (VEORQ_M_S "s") (VBRSRQ_M_N_S "s")
 330                        (VMULQ_M_U "u") (VQRDMLAHQ_M_N_S "s") (VHSUBQ_M_N_S "s")
 331                        (VQRSHLQ_M_S "s") (VMULQ_M_N_U "u")
 332                        (VMULQ_M_S "s") (VQSHLQ_M_N_U "u") (VSLIQ_M_N_U "u")
 333                        (VMLADAVAQ_P_S "s") (VQRSHLQ_M_U "u")
 334                        (VMULLBQ_INT_M_U "u") (VSHLQ_M_N_U "u") (VQSUBQ_M_U "u")
 335                        (VQRDMLASHQ_M_N_U "u") (VRSHRQ_M_N_S "s")
 336                        (VORNQ_M_S "s") (VCADDQ_ROT270_M_S "s") (VRHADDQ_M_U "u")
 337                        (VRSHRQ_M_N_U "u") (VMLASQ_M_N_U "u") (VHSUBQ_M_U "u")
 338                        (VQSUBQ_M_N_S "s") (VMULLTQ_INT_M_S "s")
 339                        (VORRQ_M_S "s") (VQDMLAHQ_M_N_U "u") (VRSHLQ_M_S "s")
 340                        (VHADDQ_M_U "u") (VHADDQ_M_N_S "s") (VMULLTQ_INT_M_U "u")
 341                        (VORRQ_M_U "u") (VHADDQ_M_S "s") (VHADDQ_M_N_U "u")
 342                        (VQDMLAHQ_M_N_S "s") (VMAXQ_M_S "s") (VORNQ_M_U "u")
 343                        (VCADDQ_ROT270_M_U "u") (VQADDQ_M_U "u")
 344                        (VQRDMLASHQ_M_N_S "s") (VBICQ_M_U "u") (VMINQ_M_U "u")
 345                        (VSUBQ_M_N_S "s") (VMULLBQ_INT_M_S "s") (VQSUBQ_M_S "s")
 346                        (VCADDQ_ROT90_M_S "s") (VRMULHQ_M_S "s") (VANDQ_M_U "u")
 347                        (VMULHQ_M_S "s") (VADDQ_M_S "s") (VQRDMLAHQ_M_N_U "u")
 348                        (VMLASQ_M_N_S "s") (VHSUBQ_M_S "s") (VRMULHQ_M_U "u")
 349                        (VQADDQ_M_N_S "s") (VSHRQ_M_N_S "s") (VANDQ_M_S "s")
 350                        (VABDQ_M_U "u") (VQSHLQ_M_S "s") (VABDQ_M_S "s")
 351                        (VSUBQ_M_N_U "u") (VMLAQ_M_N_S "s") (VBRSRQ_M_N_U "u")
 352                        (VADDQ_M_U "u") (VRSHLQ_M_U "u") (VSLIQ_M_N_S "s")
 353                        (VQADDQ_M_N_U "u") (VADDQ_M_N_S "s") (VQSUBQ_M_N_U "u")
 354                        (VMLAQ_M_N_U "u") (VMINQ_M_S "s") (VMULHQ_M_U "u")
 355                        (VQADDQ_M_S "s") (VBICQ_M_S "s") (VQSHLQ_M_N_S "s")
 356                        (VQSHRNTQ_M_N_S "s") (VQSHRNTQ_M_N_U "u")
 357                        (VSHRNTQ_M_N_U "u") (VSHRNTQ_M_N_S "s")
 358                        (VSHRNBQ_M_N_S "s") (VSHRNBQ_M_N_U "u")
 359                        (VSHLLTQ_M_N_S "s") (VSHLLTQ_M_N_U "u")
 360                        (VSHLLBQ_M_N_S "s") (VSHLLBQ_M_N_U "u")
 361                        (VRSHRNTQ_M_N_S "s") (VRSHRNTQ_M_N_U "u")
 362                        (VRSHRNBQ_M_N_U "u") (VRSHRNBQ_M_N_S "s")
 363                        (VQSHRNTQ_M_N_U "u") (VQSHRNTQ_M_N_S "s")
 364                        (VQSHRNBQ_M_N_S "s") (VQSHRNBQ_M_N_U "u")
 365                        (VQRSHRNTQ_M_N_S "s") (VQRSHRNTQ_M_N_U "u")
 366                        (VQRSHRNBQ_M_N_S "s") (VQRSHRNBQ_M_N_U "u")
 367                        (VMLALDAVAXQ_P_S "s") (VMLALDAVAXQ_P_U "u")
 368                        (VMLALDAVAQ_P_S "s") (VMLALDAVAQ_P_U "u")
 369                        (VSTRWQSB_S "s") (VSTRWQSB_U "u") (VSTRBQSO_S "s")
 370                        (VSTRBQSO_U "u") (VSTRBQ_S "s") (VSTRBQ_U "u")
 371                        (VLDRBQGO_S "s") (VLDRBQGO_U "u") (VLDRBQ_S "s")
 372                        (VLDRBQ_U "u") (VLDRWQGB_S "s") (VLDRWQGB_U "u")
 373                        (VLD1Q_S "s") (VLD1Q_U "u") (VLDRHQGO_S "s")
 374                        (VLDRHQGO_U "u") (VLDRHQGSO_S "s") (VLDRHQGSO_U "u")
 375                        (VLDRHQ_S "s") (VLDRHQ_U "u") (VLDRWQ_S "s")
 376                        (VLDRWQ_U "u") (VLDRDQGB_S "s") (VLDRDQGB_U "u")
 377                        (VLDRDQGO_S "s") (VLDRDQGO_U "u") (VLDRDQGSO_S "s")
 378                        (VLDRDQGSO_U "u") (VLDRWQGO_S "s") (VLDRWQGO_U "u")
 379                        (VLDRWQGSO_S "s") (VLDRWQGSO_U "u") (VST1Q_S "s")
 380                        (VST1Q_U "u") (VSTRHQSO_S "s") (VSTRHQSO_U "u")
 381                        (VSTRHQSSO_S "s") (VSTRHQSSO_U "u") (VSTRHQ_S "s")
 382                        (VSTRHQ_U "u") (VSTRWQ_S "s") (VSTRWQ_U "u")
 383                        (VSTRDQSB_S "s") (VSTRDQSB_U "u") (VSTRDQSO_S "s")
 384                        (VSTRDQSO_U "u") (VSTRDQSSO_S "s") (VSTRDQSSO_U "u")
 385                        (VSTRWQSO_U "u") (VSTRWQSO_S "s") (VSTRWQSSO_U "u")
 386                        (VSTRWQSSO_S "s") (VSTRWQSBWB_S "s") (VSTRWQSBWB_U "u")
 387                        (VLDRWQGBWB_S "s") (VLDRWQGBWB_U "u") (VLDRDQGBWB_S "s")
 388                        (VLDRDQGBWB_U "u") (VSTRDQSBWB_S "s") (VADCQ_M_S "s")
 389                        (VSTRDQSBWB_U "u") (VSBCQ_U "u")  (VSBCQ_M_U "u")
 390                        (VSBCQ_S "s")  (VSBCQ_M_S "s") (VSBCIQ_U "u")
 391                        (VSBCIQ_M_U "u") (VSBCIQ_S "s") (VSBCIQ_M_S "s")
 392                        (VADCQ_U "u")  (VADCQ_M_U "u") (VADCQ_S "s")
 393                        (VADCIQ_U "u") (VADCIQ_M_U "u") (VADCIQ_S "s")
 394                        (VADCIQ_M_S "s") (SQRSHRL_64 "64") (SQRSHRL_48 "48")
 395                        (UQRSHLL_64 "64") (UQRSHLL_48 "48") (VSHLCQ_M_S "s")
 396                        (VSHLCQ_M_U "u")])
 397
 398 (define_int_attr mode1 [(VCTP8Q "8") (VCTP16Q "16") (VCTP32Q "32")
 399                         (VCTP64Q "64") (VCTP8Q_M "8") (VCTP16Q_M "16")
 400                         (VCTP32Q_M "32") (VCTP64Q_M "64")])
 401 (define_mode_attr MVE_pred2 [(V16QI "mve_imm_8") (V8HI "mve_imm_16")
 402                              (V4SI "mve_imm_32")
 403                              (V8HF "mve_imm_16") (V4SF "mve_imm_32")])
 404 (define_mode_attr MVE_constraint2 [(V16QI "Rb") (V8HI "Rd") (V4SI "Rf")
 405                                     (V8HF "Rd") (V4SF "Rf")])
 406 (define_mode_attr MVE_LANES [(V16QI "16") (V8HI "8") (V4SI "4")])
 407 (define_mode_attr MVE_constraint [ (V16QI "Ra") (V8HI "Rc") (V4SI "Re")])
 408 (define_mode_attr MVE_pred [ (V16QI "mve_imm_7") (V8HI "mve_imm_15")
 409                                    (V4SI "mve_imm_31")])
 410 (define_mode_attr MVE_constraint3 [ (V8HI "Rb") (V4SI "Rd")])
 411 (define_mode_attr MVE_pred3 [ (V8HI "mve_imm_8") (V4SI "mve_imm_16")])
 412 (define_mode_attr MVE_constraint1 [ (V8HI "Ra") (V4SI "Rc")])
 413 (define_mode_attr MVE_pred1 [ (V8HI "mve_imm_7") (V4SI "mve_imm_15")])
 414 (define_mode_attr MVE_B_ELEM [ (V16QI "V16QI") (V8HI "V8QI") (V4SI "V4QI")])
 415 (define_mode_attr MVE_H_ELEM [ (V8HI "V8HI") (V4SI "V4HI")])
 416 (define_mode_attr V_sz_elem1 [(V16QI "b") (V8HI  "h") (V4SI "w") (V8HF "h")
 417                               (V4SF "w")])
 418 (define_mode_attr V_extr_elem [(V16QI "u8") (V8HI "u16") (V4SI "32")
 419                               (V8HF "u16") (V4SF "32")])
 420
 421 (define_mode_attr earlyclobber_32 [(V16QI "=w") (V8HI "=w") (V4SI "=&w")
 422                                                 (V8HF "=w") (V4SF "=&w")])
 423
 424 (define_int_iterator VCVTQ_TO_F [VCVTQ_TO_F_S VCVTQ_TO_F_U])
 425 (define_int_iterator VMVNQ_N [VMVNQ_N_U VMVNQ_N_S])
 426 (define_int_iterator VREV64Q [VREV64Q_S VREV64Q_U])
 427 (define_int_iterator VCVTQ_FROM_F [VCVTQ_FROM_F_S VCVTQ_FROM_F_U])
 428 (define_int_iterator VREV16Q [VREV16Q_U VREV16Q_S])
 429 (define_int_iterator VCVTAQ [VCVTAQ_U VCVTAQ_S])
 430 (define_int_iterator VMVNQ [VMVNQ_U VMVNQ_S])
 431 (define_int_iterator VDUPQ_N [VDUPQ_N_U VDUPQ_N_S])
 432 (define_int_iterator VCLZQ [VCLZQ_U VCLZQ_S])
 433 (define_int_iterator VADDVQ [VADDVQ_U VADDVQ_S])
 434 (define_int_iterator VREV32Q [VREV32Q_U VREV32Q_S])
 435 (define_int_iterator VMOVLBQ [VMOVLBQ_S VMOVLBQ_U])
 436 (define_int_iterator VMOVLTQ [VMOVLTQ_U VMOVLTQ_S])
 437 (define_int_iterator VCVTPQ [VCVTPQ_S VCVTPQ_U])
 438 (define_int_iterator VCVTNQ [VCVTNQ_S VCVTNQ_U])
 439 (define_int_iterator VCVTMQ [VCVTMQ_S VCVTMQ_U])
 440 (define_int_iterator VADDLVQ [VADDLVQ_U VADDLVQ_S])
 441 (define_int_iterator VCTPQ [VCTP8Q VCTP16Q VCTP32Q VCTP64Q])
 442 (define_int_iterator VCTPQ_M [VCTP8Q_M VCTP16Q_M VCTP32Q_M VCTP64Q_M])
 443 (define_int_iterator VCVTQ_N_TO_F [VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U])
 444 (define_int_iterator VCREATEQ [VCREATEQ_U VCREATEQ_S])
 445 (define_int_iterator VSHRQ_N [VSHRQ_N_S VSHRQ_N_U])
 446 (define_int_iterator VCVTQ_N_FROM_F [VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U])
 447 (define_int_iterator VADDLVQ_P [VADDLVQ_P_S VADDLVQ_P_U])
 448 (define_int_iterator VCMPNEQ [VCMPNEQ_U VCMPNEQ_S])
 449 (define_int_iterator VSHLQ [VSHLQ_S VSHLQ_U])
 450 (define_int_iterator VABDQ [VABDQ_S VABDQ_U])
 451 (define_int_iterator VADDQ_N [VADDQ_N_S VADDQ_N_U])
 452 (define_int_iterator VADDVAQ [VADDVAQ_S VADDVAQ_U])
 453 (define_int_iterator VADDVQ_P [VADDVQ_P_U VADDVQ_P_S])
 454 (define_int_iterator VANDQ [VANDQ_U VANDQ_S])
 455 (define_int_iterator VBICQ [VBICQ_S VBICQ_U])
 456 (define_int_iterator VBRSRQ_N [VBRSRQ_N_U VBRSRQ_N_S])
 457 (define_int_iterator VCADDQ_ROT270 [VCADDQ_ROT270_S VCADDQ_ROT270_U])
 458 (define_int_iterator VCADDQ_ROT90 [VCADDQ_ROT90_U VCADDQ_ROT90_S])
 459 (define_int_iterator VCMPEQQ [VCMPEQQ_U VCMPEQQ_S])
 460 (define_int_iterator VCMPEQQ_N [VCMPEQQ_N_S VCMPEQQ_N_U])
 461 (define_int_iterator VCMPNEQ_N [VCMPNEQ_N_U VCMPNEQ_N_S])
 462 (define_int_iterator VEORQ [VEORQ_U VEORQ_S])
 463 (define_int_iterator VHADDQ [VHADDQ_S VHADDQ_U])
 464 (define_int_iterator VHADDQ_N [VHADDQ_N_U VHADDQ_N_S])
 465 (define_int_iterator VHSUBQ [VHSUBQ_S VHSUBQ_U])
 466 (define_int_iterator VHSUBQ_N [VHSUBQ_N_U VHSUBQ_N_S])
 467 (define_int_iterator VMAXQ [VMAXQ_U VMAXQ_S])
 468 (define_int_iterator VMAXVQ [VMAXVQ_U VMAXVQ_S])
 469 (define_int_iterator VMINQ [VMINQ_S VMINQ_U])
 470 (define_int_iterator VMINVQ [VMINVQ_U VMINVQ_S])
 471 (define_int_iterator VMLADAVQ [VMLADAVQ_U VMLADAVQ_S])
 472 (define_int_iterator VMULHQ [VMULHQ_S VMULHQ_U])
 473 (define_int_iterator VMULLBQ_INT [VMULLBQ_INT_U VMULLBQ_INT_S])
 474 (define_int_iterator VMULLTQ_INT [VMULLTQ_INT_U VMULLTQ_INT_S])
 475 (define_int_iterator VMULQ [VMULQ_U VMULQ_S])
 476 (define_int_iterator VMULQ_N [VMULQ_N_U VMULQ_N_S])
 477 (define_int_iterator VORNQ [VORNQ_U VORNQ_S])
 478 (define_int_iterator VORRQ [VORRQ_S VORRQ_U])
 479 (define_int_iterator VQADDQ [VQADDQ_U VQADDQ_S])
 480 (define_int_iterator VQADDQ_N [VQADDQ_N_S VQADDQ_N_U])
 481 (define_int_iterator VQRSHLQ [VQRSHLQ_S VQRSHLQ_U])
 482 (define_int_iterator VQRSHLQ_N [VQRSHLQ_N_S VQRSHLQ_N_U])
 483 (define_int_iterator VQSHLQ [VQSHLQ_S VQSHLQ_U])
 484 (define_int_iterator VQSHLQ_N [VQSHLQ_N_S VQSHLQ_N_U])
 485 (define_int_iterator VQSHLQ_R [VQSHLQ_R_U VQSHLQ_R_S])
 486 (define_int_iterator VQSUBQ [VQSUBQ_U VQSUBQ_S])
 487 (define_int_iterator VQSUBQ_N [VQSUBQ_N_S VQSUBQ_N_U])
 488 (define_int_iterator VRHADDQ [VRHADDQ_S VRHADDQ_U])
 489 (define_int_iterator VRMULHQ [VRMULHQ_S VRMULHQ_U])
 490 (define_int_iterator VRSHLQ [VRSHLQ_S VRSHLQ_U])
 491 (define_int_iterator VRSHLQ_N [VRSHLQ_N_U VRSHLQ_N_S])
 492 (define_int_iterator VRSHRQ_N [VRSHRQ_N_S VRSHRQ_N_U])
 493 (define_int_iterator VSHLQ_N [VSHLQ_N_U VSHLQ_N_S])
 494 (define_int_iterator VSHLQ_R [VSHLQ_R_S VSHLQ_R_U])
 495 (define_int_iterator VSUBQ [VSUBQ_S VSUBQ_U])
 496 (define_int_iterator VSUBQ_N [VSUBQ_N_S VSUBQ_N_U])
 497 (define_int_iterator VADDLVAQ [VADDLVAQ_S VADDLVAQ_U])
 498 (define_int_iterator VBICQ_N [VBICQ_N_S VBICQ_N_U])
 499 (define_int_iterator VMLALDAVQ [VMLALDAVQ_U VMLALDAVQ_S])
 500 (define_int_iterator VMLALDAVXQ [VMLALDAVXQ_U VMLALDAVXQ_S])
 501 (define_int_iterator VMOVNBQ [VMOVNBQ_U VMOVNBQ_S])
 502 (define_int_iterator VMOVNTQ [VMOVNTQ_S VMOVNTQ_U])
 503 (define_int_iterator VORRQ_N [VORRQ_N_U VORRQ_N_S])
 504 (define_int_iterator VQMOVNBQ [VQMOVNBQ_U VQMOVNBQ_S])
 505 (define_int_iterator VQMOVNTQ [VQMOVNTQ_U VQMOVNTQ_S])
 506 (define_int_iterator VSHLLBQ_N [VSHLLBQ_N_S VSHLLBQ_N_U])
 507 (define_int_iterator VSHLLTQ_N [VSHLLTQ_N_U VSHLLTQ_N_S])
 508 (define_int_iterator VRMLALDAVHQ [VRMLALDAVHQ_U VRMLALDAVHQ_S])
 509 (define_int_iterator VBICQ_M_N [VBICQ_M_N_S VBICQ_M_N_U])
 510 (define_int_iterator VCVTAQ_M [VCVTAQ_M_S VCVTAQ_M_U])
 511 (define_int_iterator VCVTQ_M_TO_F [VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U])
 512 (define_int_iterator VQRSHRNBQ_N [VQRSHRNBQ_N_U VQRSHRNBQ_N_S])
 513 (define_int_iterator VABAVQ [VABAVQ_S VABAVQ_U])
 514 (define_int_iterator VSHLCQ [VSHLCQ_S VSHLCQ_U])
 515 (define_int_iterator VRMLALDAVHAQ [VRMLALDAVHAQ_S VRMLALDAVHAQ_U])
 516 (define_int_iterator VADDVAQ_P [VADDVAQ_P_S VADDVAQ_P_U])
 517 (define_int_iterator VCLZQ_M [VCLZQ_M_S VCLZQ_M_U])
 518 (define_int_iterator VCMPEQQ_M_N [VCMPEQQ_M_N_S VCMPEQQ_M_N_U])
 519 (define_int_iterator VCMPEQQ_M [VCMPEQQ_M_S VCMPEQQ_M_U])
 520 (define_int_iterator VCMPNEQ_M_N [VCMPNEQ_M_N_S VCMPNEQ_M_N_U])
 521 (define_int_iterator VCMPNEQ_M [VCMPNEQ_M_S VCMPNEQ_M_U])
 522 (define_int_iterator VDUPQ_M_N [VDUPQ_M_N_S VDUPQ_M_N_U])
 523 (define_int_iterator VMAXVQ_P [VMAXVQ_P_S VMAXVQ_P_U])
 524 (define_int_iterator VMINVQ_P [VMINVQ_P_S VMINVQ_P_U])
 525 (define_int_iterator VMLADAVAQ [VMLADAVAQ_S VMLADAVAQ_U])
 526 (define_int_iterator VMLADAVQ_P [VMLADAVQ_P_S VMLADAVQ_P_U])
 527 (define_int_iterator VMLAQ_N [VMLAQ_N_S VMLAQ_N_U])
 528 (define_int_iterator VMLASQ_N [VMLASQ_N_S VMLASQ_N_U])
 529 (define_int_iterator VMVNQ_M [VMVNQ_M_S VMVNQ_M_U])
 530 (define_int_iterator VPSELQ [VPSELQ_S VPSELQ_U])
 531 (define_int_iterator VQDMLAHQ_N [VQDMLAHQ_N_S VQDMLAHQ_N_U])
 532 (define_int_iterator VQRDMLAHQ_N [VQRDMLAHQ_N_S VQRDMLAHQ_N_U])
 533 (define_int_iterator VQRDMLASHQ_N [VQRDMLASHQ_N_S VQRDMLASHQ_N_U])
 534 (define_int_iterator VQRSHLQ_M_N [VQRSHLQ_M_N_S VQRSHLQ_M_N_U])
 535 (define_int_iterator VQSHLQ_M_R [VQSHLQ_M_R_S VQSHLQ_M_R_U])
 536 (define_int_iterator VREV64Q_M [VREV64Q_M_S VREV64Q_M_U])
 537 (define_int_iterator VRSHLQ_M_N [VRSHLQ_M_N_S VRSHLQ_M_N_U])
 538 (define_int_iterator VSHLQ_M_R [VSHLQ_M_R_S VSHLQ_M_R_U])
 539 (define_int_iterator VSLIQ_N [VSLIQ_N_S VSLIQ_N_U])
 540 (define_int_iterator VSRIQ_N [VSRIQ_N_S VSRIQ_N_U])
 541 (define_int_iterator VMLALDAVQ_P [VMLALDAVQ_P_U VMLALDAVQ_P_S])
 542 (define_int_iterator VQMOVNBQ_M [VQMOVNBQ_M_S VQMOVNBQ_M_U])
 543 (define_int_iterator VMOVLTQ_M [VMOVLTQ_M_U VMOVLTQ_M_S])
 544 (define_int_iterator VMOVNBQ_M [VMOVNBQ_M_U VMOVNBQ_M_S])
 545 (define_int_iterator VRSHRNTQ_N [VRSHRNTQ_N_U VRSHRNTQ_N_S])
 546 (define_int_iterator VORRQ_M_N [VORRQ_M_N_S VORRQ_M_N_U])
 547 (define_int_iterator VREV32Q_M [VREV32Q_M_S VREV32Q_M_U])
 548 (define_int_iterator VREV16Q_M [VREV16Q_M_S VREV16Q_M_U])
 549 (define_int_iterator VQRSHRNTQ_N [VQRSHRNTQ_N_U VQRSHRNTQ_N_S])
 550 (define_int_iterator VMOVNTQ_M [VMOVNTQ_M_U VMOVNTQ_M_S])
 551 (define_int_iterator VMOVLBQ_M [VMOVLBQ_M_U VMOVLBQ_M_S])
 552 (define_int_iterator VMLALDAVAQ [VMLALDAVAQ_S VMLALDAVAQ_U])
 553 (define_int_iterator VQSHRNBQ_N [VQSHRNBQ_N_U VQSHRNBQ_N_S])
 554 (define_int_iterator VSHRNBQ_N [VSHRNBQ_N_U VSHRNBQ_N_S])
 555 (define_int_iterator VRSHRNBQ_N [VRSHRNBQ_N_S VRSHRNBQ_N_U])
 556 (define_int_iterator VMLALDAVXQ_P [VMLALDAVXQ_P_U VMLALDAVXQ_P_S])
 557 (define_int_iterator VQMOVNTQ_M [VQMOVNTQ_M_U VQMOVNTQ_M_S])
 558 (define_int_iterator VMVNQ_M_N [VMVNQ_M_N_U VMVNQ_M_N_S])
 559 (define_int_iterator VQSHRNTQ_N [VQSHRNTQ_N_U VQSHRNTQ_N_S])
 560 (define_int_iterator VMLALDAVAXQ [VMLALDAVAXQ_S VMLALDAVAXQ_U])
 561 (define_int_iterator VSHRNTQ_N [VSHRNTQ_N_S VSHRNTQ_N_U])
 562 (define_int_iterator VCVTMQ_M [VCVTMQ_M_S VCVTMQ_M_U])
 563 (define_int_iterator VCVTNQ_M [VCVTNQ_M_S VCVTNQ_M_U])
 564 (define_int_iterator VCVTPQ_M [VCVTPQ_M_S VCVTPQ_M_U])
 565 (define_int_iterator VCVTQ_M_N_FROM_F [VCVTQ_M_N_FROM_F_S VCVTQ_M_N_FROM_F_U])
 566 (define_int_iterator VCVTQ_M_FROM_F [VCVTQ_M_FROM_F_U VCVTQ_M_FROM_F_S])
 567 (define_int_iterator VRMLALDAVHQ_P [VRMLALDAVHQ_P_S VRMLALDAVHQ_P_U])
 568 (define_int_iterator VADDLVAQ_P [VADDLVAQ_P_U VADDLVAQ_P_S])
 569 (define_int_iterator VABAVQ_P [VABAVQ_P_S VABAVQ_P_U])
 570 (define_int_iterator VSHLQ_M [VSHLQ_M_S VSHLQ_M_U])
 571 (define_int_iterator VSRIQ_M_N [VSRIQ_M_N_S VSRIQ_M_N_U])
 572 (define_int_iterator VSUBQ_M [VSUBQ_M_U VSUBQ_M_S])
 573 (define_int_iterator VCVTQ_M_N_TO_F [VCVTQ_M_N_TO_F_U VCVTQ_M_N_TO_F_S])
 574 (define_int_iterator VHSUBQ_M [VHSUBQ_M_S VHSUBQ_M_U])
 575 (define_int_iterator VSLIQ_M_N [VSLIQ_M_N_U VSLIQ_M_N_S])
 576 (define_int_iterator VRSHLQ_M [VRSHLQ_M_S VRSHLQ_M_U])
 577 (define_int_iterator VMINQ_M [VMINQ_M_S VMINQ_M_U])
 578 (define_int_iterator VMULLBQ_INT_M [VMULLBQ_INT_M_U VMULLBQ_INT_M_S])
 579 (define_int_iterator VMULHQ_M [VMULHQ_M_S VMULHQ_M_U])
 580 (define_int_iterator VMULQ_M [VMULQ_M_S VMULQ_M_U])
 581 (define_int_iterator VHSUBQ_M_N [VHSUBQ_M_N_S VHSUBQ_M_N_U])
 582 (define_int_iterator VHADDQ_M_N [VHADDQ_M_N_S VHADDQ_M_N_U])
 583 (define_int_iterator VORRQ_M [VORRQ_M_S VORRQ_M_U])
 584 (define_int_iterator VRMULHQ_M [VRMULHQ_M_U VRMULHQ_M_S])
 585 (define_int_iterator VQADDQ_M [VQADDQ_M_U VQADDQ_M_S])
 586 (define_int_iterator VRSHRQ_M_N [VRSHRQ_M_N_S VRSHRQ_M_N_U])
 587 (define_int_iterator VQSUBQ_M_N [VQSUBQ_M_N_U VQSUBQ_M_N_S])
 588 (define_int_iterator VADDQ_M [VADDQ_M_U VADDQ_M_S])
 589 (define_int_iterator VORNQ_M [VORNQ_M_U VORNQ_M_S])
 590 (define_int_iterator VRHADDQ_M [VRHADDQ_M_U VRHADDQ_M_S])
 591 (define_int_iterator VQSHLQ_M [VQSHLQ_M_U VQSHLQ_M_S])
 592 (define_int_iterator VANDQ_M [VANDQ_M_U VANDQ_M_S])
 593 (define_int_iterator VBICQ_M [VBICQ_M_U VBICQ_M_S])
 594 (define_int_iterator VSHLQ_M_N [VSHLQ_M_N_S VSHLQ_M_N_U])
 595 (define_int_iterator VCADDQ_ROT270_M [VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S])
 596 (define_int_iterator VQRSHLQ_M [VQRSHLQ_M_U VQRSHLQ_M_S])
 597 (define_int_iterator VQADDQ_M_N [VQADDQ_M_N_U VQADDQ_M_N_S])
 598 (define_int_iterator VADDQ_M_N [VADDQ_M_N_S VADDQ_M_N_U])
 599 (define_int_iterator VMAXQ_M [VMAXQ_M_S VMAXQ_M_U])
 600 (define_int_iterator VQSUBQ_M [VQSUBQ_M_U VQSUBQ_M_S])
 601 (define_int_iterator VMLASQ_M_N [VMLASQ_M_N_U VMLASQ_M_N_S])
 602 (define_int_iterator VMLADAVAQ_P [VMLADAVAQ_P_U VMLADAVAQ_P_S])
 603 (define_int_iterator VBRSRQ_M_N [VBRSRQ_M_N_U VBRSRQ_M_N_S])
 604 (define_int_iterator VMULQ_M_N [VMULQ_M_N_U VMULQ_M_N_S])
 605 (define_int_iterator VCADDQ_ROT90_M [VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S])
 606 (define_int_iterator VMULLTQ_INT_M [VMULLTQ_INT_M_S VMULLTQ_INT_M_U])
 607 (define_int_iterator VEORQ_M [VEORQ_M_S VEORQ_M_U])
 608 (define_int_iterator VSHRQ_M_N [VSHRQ_M_N_S VSHRQ_M_N_U])
 609 (define_int_iterator VSUBQ_M_N [VSUBQ_M_N_S VSUBQ_M_N_U])
 610 (define_int_iterator VHADDQ_M [VHADDQ_M_S VHADDQ_M_U])
 611 (define_int_iterator VABDQ_M [VABDQ_M_S VABDQ_M_U])
 612 (define_int_iterator VMLAQ_M_N [VMLAQ_M_N_S VMLAQ_M_N_U])
 613 (define_int_iterator VQSHLQ_M_N [VQSHLQ_M_N_S VQSHLQ_M_N_U])
 614 (define_int_iterator VMLALDAVAQ_P [VMLALDAVAQ_P_U VMLALDAVAQ_P_S])
 615 (define_int_iterator VMLALDAVAXQ_P [VMLALDAVAXQ_P_U VMLALDAVAXQ_P_S])
 616 (define_int_iterator VQRSHRNBQ_M_N [VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S])
 617 (define_int_iterator VQRSHRNTQ_M_N [VQRSHRNTQ_M_N_S VQRSHRNTQ_M_N_U])
 618 (define_int_iterator VQSHRNBQ_M_N [VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S])
 619 (define_int_iterator VQSHRNTQ_M_N [VQSHRNTQ_M_N_S VQSHRNTQ_M_N_U])
 620 (define_int_iterator VRSHRNBQ_M_N [VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S])
 621 (define_int_iterator VRSHRNTQ_M_N [VRSHRNTQ_M_N_U VRSHRNTQ_M_N_S])
 622 (define_int_iterator VSHLLBQ_M_N [VSHLLBQ_M_N_U VSHLLBQ_M_N_S])
 623 (define_int_iterator VSHLLTQ_M_N [VSHLLTQ_M_N_U VSHLLTQ_M_N_S])
 624 (define_int_iterator VSHRNBQ_M_N [VSHRNBQ_M_N_S VSHRNBQ_M_N_U])
 625 (define_int_iterator VSHRNTQ_M_N [VSHRNTQ_M_N_S VSHRNTQ_M_N_U])
 626 (define_int_iterator VSTRWSBQ [VSTRWQSB_S VSTRWQSB_U])
 627 (define_int_iterator VSTRBSOQ [VSTRBQSO_S VSTRBQSO_U])
 628 (define_int_iterator VSTRBQ [VSTRBQ_S VSTRBQ_U])
 629 (define_int_iterator VLDRBGOQ [VLDRBQGO_S VLDRBQGO_U])
 630 (define_int_iterator VLDRBQ [VLDRBQ_S VLDRBQ_U])
 631 (define_int_iterator VLDRWGBQ [VLDRWQGB_S VLDRWQGB_U])
 632 (define_int_iterator VLD1Q [VLD1Q_S VLD1Q_U])
 633 (define_int_iterator VLDRHGOQ [VLDRHQGO_S VLDRHQGO_U])
 634 (define_int_iterator VLDRHGSOQ [VLDRHQGSO_S VLDRHQGSO_U])
 635 (define_int_iterator VLDRHQ [VLDRHQ_S VLDRHQ_U])
 636 (define_int_iterator VLDRWQ [VLDRWQ_S VLDRWQ_U])
 637 (define_int_iterator VLDRDGBQ [VLDRDQGB_S VLDRDQGB_U])
 638 (define_int_iterator VLDRDGOQ [VLDRDQGO_S VLDRDQGO_U])
 639 (define_int_iterator VLDRDGSOQ [VLDRDQGSO_S VLDRDQGSO_U])
 640 (define_int_iterator VLDRWGOQ [VLDRWQGO_S VLDRWQGO_U])
 641 (define_int_iterator VLDRWGSOQ [VLDRWQGSO_S VLDRWQGSO_U])
 642 (define_int_iterator VST1Q [VST1Q_S VST1Q_U])
 643 (define_int_iterator VSTRHSOQ [VSTRHQSO_S VSTRHQSO_U])
 644 (define_int_iterator VSTRHSSOQ [VSTRHQSSO_S VSTRHQSSO_U])
 645 (define_int_iterator VSTRHQ [VSTRHQ_S VSTRHQ_U])
 646 (define_int_iterator VSTRWQ [VSTRWQ_S VSTRWQ_U])
 647 (define_int_iterator VSTRDSBQ [VSTRDQSB_S VSTRDQSB_U])
 648 (define_int_iterator VSTRDSOQ [VSTRDQSO_S VSTRDQSO_U])
 649 (define_int_iterator VSTRDSSOQ [VSTRDQSSO_S VSTRDQSSO_U])
 650 (define_int_iterator VSTRWSOQ [VSTRWQSO_S VSTRWQSO_U])
 651 (define_int_iterator VSTRWSSOQ [VSTRWQSSO_S VSTRWQSSO_U])
 652 (define_int_iterator VSTRWSBWBQ [VSTRWQSBWB_S VSTRWQSBWB_U])
 653 (define_int_iterator VLDRWGBWBQ [VLDRWQGBWB_S VLDRWQGBWB_U])
 654 (define_int_iterator VSTRDSBWBQ [VSTRDQSBWB_S VSTRDQSBWB_U])
 655 (define_int_iterator VLDRDGBWBQ [VLDRDQGBWB_S VLDRDQGBWB_U])
 656 (define_int_iterator VADCIQ [VADCIQ_U VADCIQ_S])
 657 (define_int_iterator VADCIQ_M [VADCIQ_M_U VADCIQ_M_S])
 658 (define_int_iterator VSBCQ [VSBCQ_U VSBCQ_S])
 659 (define_int_iterator VSBCQ_M [VSBCQ_M_U VSBCQ_M_S])
 660 (define_int_iterator VSBCIQ [VSBCIQ_U VSBCIQ_S])
 661 (define_int_iterator VSBCIQ_M [VSBCIQ_M_U VSBCIQ_M_S])
 662 (define_int_iterator VADCQ [VADCQ_U VADCQ_S])
 663 (define_int_iterator VADCQ_M [VADCQ_M_U VADCQ_M_S])
 664 (define_int_iterator UQRSHLLQ [UQRSHLL_64 UQRSHLL_48])
 665 (define_int_iterator SQRSHRLQ [SQRSHRL_64 SQRSHRL_48])
 666 (define_int_iterator VSHLCQ_M [VSHLCQ_M_S VSHLCQ_M_U])
 667
 668 (define_insn "*mve_mov<mode>"
 669   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Us")
 670         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Usi,r,Dm,w"))]
 671   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 672 {
 673   if (which_alternative == 3 || which_alternative == 6)
 674     {
 675       int width, is_valid;
 676       static char templ[40];
 677
 678       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
 679         &operands[1], &width);
 680
 681       gcc_assert (is_valid != 0);
 682
 683       if (width == 0)
 684         return "vmov.f32\t%q0, %1  @ <mode>";
 685       else
 686         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
 687       return templ;
 688     }
 689   switch (which_alternative)
 690     {
 691     case 0:
 692       return "vmov\t%q0, %q1";
 693     case 1:
 694       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 695     case 2:
 696       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 697     case 4:
 698       if (MEM_P (operands[1])
 699           && (GET_CODE (XEXP (operands[1], 0)) == LABEL_REF
 700               || GET_CODE (XEXP (operands[1], 0)) == CONST))
 701         return output_move_neon (operands);
 702       else
 703         return "vldrb.8 %q0, %E1";
 704     case 5:
 705       return output_move_quad (operands);
 706     case 7:
 707       return "vstrb.8 %q1, %E0";
 708     default:
 709       gcc_unreachable ();
 710       return "";
 711     }
 712 }
 713   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store")
 714    (set_attr "length" "4,8,8,4,8,8,4,4")
 715    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*")
 716    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*")])
 717
 718 (define_insn "*mve_mov<mode>"
 719   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 720         (vec_duplicate:MVE_types
 721           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 722   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 723 {
 724   if (which_alternative == 0)
 725     return "vdup.<V_sz_elem>\t%q0, %1";
 726   return "vmov.<V_sz_elem>\t%q0, %1";
 727 }
 728   [(set_attr "length" "4,4")
 729    (set_attr "type" "mve_move,mve_move")])
 730
 731 ;;
 732 ;; [vst4q])
 733 ;;
 734 (define_insn "mve_vst4q<mode>"
 735   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 736         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 737                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 738          VST4Q))
 739   ]
 740   "TARGET_HAVE_MVE"
 741 {
 742    rtx ops[6];
 743    int regno = REGNO (operands[1]);
 744    ops[0] = gen_rtx_REG (TImode, regno);
 745    ops[1] = gen_rtx_REG (TImode, regno+4);
 746    ops[2] = gen_rtx_REG (TImode, regno+8);
 747    ops[3] = gen_rtx_REG (TImode, regno+12);
 748    rtx reg  = operands[0];
 749    while (reg && !REG_P (reg))
 750     reg = XEXP (reg, 0);
 751    gcc_assert (REG_P (reg));
 752    ops[4] = reg;
 753    ops[5] = operands[0];
 754    /* Here in first three instructions data is stored to ops[4]'s location but
 755       in the fourth instruction data is stored to operands[0], this is to
 756       support the writeback.  */
 757    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 758                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 759                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 760                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 761    return "";
 762 }
 763   [(set_attr "length" "16")])
 764
 765 ;;
 766 ;; [vrndq_m_f])
 767 ;;
 768 (define_insn "mve_vrndq_m_f<mode>"
 769   [
 770    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 771         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 772                        (match_operand:MVE_0 2 "s_register_operand" "w")
 773                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 774          VRNDQ_M_F))
 775   ]
 776   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 777   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 778   [(set_attr "type" "mve_move")
 779    (set_attr "length""8")])
 780
 781 ;;
 782 ;; [vrndxq_f])
 783 ;;
 784 (define_insn "mve_vrndxq_f<mode>"
 785   [
 786    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 787         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 788          VRNDXQ_F))
 789   ]
 790   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 791   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 792   [(set_attr "type" "mve_move")
 793 ])
 794
 795 ;;
 796 ;; [vrndq_f])
 797 ;;
 798 (define_insn "mve_vrndq_f<mode>"
 799   [
 800    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 801         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 802          VRNDQ_F))
 803   ]
 804   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 805   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 806   [(set_attr "type" "mve_move")
 807 ])
 808
 809 ;;
 810 ;; [vrndpq_f])
 811 ;;
 812 (define_insn "mve_vrndpq_f<mode>"
 813   [
 814    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 815         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 816          VRNDPQ_F))
 817   ]
 818   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 819   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 820   [(set_attr "type" "mve_move")
 821 ])
 822
 823 ;;
 824 ;; [vrndnq_f])
 825 ;;
 826 (define_insn "mve_vrndnq_f<mode>"
 827   [
 828    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 829         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 830          VRNDNQ_F))
 831   ]
 832   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 833   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 834   [(set_attr "type" "mve_move")
 835 ])
 836
 837 ;;
 838 ;; [vrndmq_f])
 839 ;;
 840 (define_insn "mve_vrndmq_f<mode>"
 841   [
 842    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 843         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 844          VRNDMQ_F))
 845   ]
 846   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 847   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 848   [(set_attr "type" "mve_move")
 849 ])
 850
 851 ;;
 852 ;; [vrndaq_f])
 853 ;;
 854 (define_insn "mve_vrndaq_f<mode>"
 855   [
 856    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 857         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 858          VRNDAQ_F))
 859   ]
 860   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 861   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 862   [(set_attr "type" "mve_move")
 863 ])
 864
 865 ;;
 866 ;; [vrev64q_f])
 867 ;;
 868 (define_insn "mve_vrev64q_f<mode>"
 869   [
 870    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 871         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 872          VREV64Q_F))
 873   ]
 874   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 875   "vrev64.%#<V_sz_elem> %q0, %q1"
 876   [(set_attr "type" "mve_move")
 877 ])
 878
 879 ;;
 880 ;; [vnegq_f])
 881 ;;
 882 (define_insn "mve_vnegq_f<mode>"
 883   [
 884    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 885         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 886          VNEGQ_F))
 887   ]
 888   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 889   "vneg.f%#<V_sz_elem>  %q0, %q1"
 890   [(set_attr "type" "mve_move")
 891 ])
 892
 893 ;;
 894 ;; [vdupq_n_f])
 895 ;;
 896 (define_insn "mve_vdupq_n_f<mode>"
 897   [
 898    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 899         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 900          VDUPQ_N_F))
 901   ]
 902   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 903   "vdup.%#<V_sz_elem>   %q0, %1"
 904   [(set_attr "type" "mve_move")
 905 ])
 906
 907 ;;
 908 ;; [vabsq_f])
 909 ;;
 910 (define_insn "mve_vabsq_f<mode>"
 911   [
 912    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 913         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 914          VABSQ_F))
 915   ]
 916   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 917   "vabs.f%#<V_sz_elem>  %q0, %q1"
 918   [(set_attr "type" "mve_move")
 919 ])
 920
 921 ;;
 922 ;; [vrev32q_f])
 923 ;;
 924 (define_insn "mve_vrev32q_fv8hf"
 925   [
 926    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 927         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 928          VREV32Q_F))
 929   ]
 930   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 931   "vrev32.16 %q0, %q1"
 932   [(set_attr "type" "mve_move")
 933 ])
 934 ;;
 935 ;; [vcvttq_f32_f16])
 936 ;;
 937 (define_insn "mve_vcvttq_f32_f16v4sf"
 938   [
 939    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 940         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 941          VCVTTQ_F32_F16))
 942   ]
 943   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 944   "vcvtt.f32.f16 %q0, %q1"
 945   [(set_attr "type" "mve_move")
 946 ])
 947
 948 ;;
 949 ;; [vcvtbq_f32_f16])
 950 ;;
 951 (define_insn "mve_vcvtbq_f32_f16v4sf"
 952   [
 953    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 954         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 955          VCVTBQ_F32_F16))
 956   ]
 957   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 958   "vcvtb.f32.f16 %q0, %q1"
 959   [(set_attr "type" "mve_move")
 960 ])
 961
 962 ;;
 963 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 964 ;;
 965 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 966   [
 967    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 968         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 969          VCVTQ_TO_F))
 970   ]
 971   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 972   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 973   [(set_attr "type" "mve_move")
 974 ])
 975
 976 ;;
 977 ;; [vrev64q_u, vrev64q_s])
 978 ;;
 979 (define_insn "mve_vrev64q_<supf><mode>"
 980   [
 981    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 982         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 983          VREV64Q))
 984   ]
 985   "TARGET_HAVE_MVE"
 986   "vrev64.%#<V_sz_elem> %q0, %q1"
 987   [(set_attr "type" "mve_move")
 988 ])
 989
 990 ;;
 991 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 992 ;;
 993 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 994   [
 995    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 996         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 997          VCVTQ_FROM_F))
 998   ]
 999   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1000   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
1001   [(set_attr "type" "mve_move")
1002 ])
1003 ;; [vqnegq_s])
1004 ;;
1005 (define_insn "mve_vqnegq_s<mode>"
1006   [
1007    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1008         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1009          VQNEGQ_S))
1010   ]
1011   "TARGET_HAVE_MVE"
1012   "vqneg.s%#<V_sz_elem> %q0, %q1"
1013   [(set_attr "type" "mve_move")
1014 ])
1015
1016 ;;
1017 ;; [vqabsq_s])
1018 ;;
1019 (define_insn "mve_vqabsq_s<mode>"
1020   [
1021    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1022         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1023          VQABSQ_S))
1024   ]
1025   "TARGET_HAVE_MVE"
1026   "vqabs.s%#<V_sz_elem> %q0, %q1"
1027   [(set_attr "type" "mve_move")
1028 ])
1029
1030 ;;
1031 ;; [vnegq_s])
1032 ;;
1033 (define_insn "mve_vnegq_s<mode>"
1034   [
1035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1037          VNEGQ_S))
1038   ]
1039   "TARGET_HAVE_MVE"
1040   "vneg.s%#<V_sz_elem>  %q0, %q1"
1041   [(set_attr "type" "mve_move")
1042 ])
1043
1044 ;;
1045 ;; [vmvnq_u, vmvnq_s])
1046 ;;
1047 (define_insn "mve_vmvnq_<supf><mode>"
1048   [
1049    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1050         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1051          VMVNQ))
1052   ]
1053   "TARGET_HAVE_MVE"
1054   "vmvn %q0, %q1"
1055   [(set_attr "type" "mve_move")
1056 ])
1057
1058 ;;
1059 ;; [vdupq_n_u, vdupq_n_s])
1060 ;;
1061 (define_insn "mve_vdupq_n_<supf><mode>"
1062   [
1063    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1064         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
1065          VDUPQ_N))
1066   ]
1067   "TARGET_HAVE_MVE"
1068   "vdup.%#<V_sz_elem>   %q0, %1"
1069   [(set_attr "type" "mve_move")
1070 ])
1071
1072 ;;
1073 ;; [vclzq_u, vclzq_s])
1074 ;;
1075 (define_insn "mve_vclzq_<supf><mode>"
1076   [
1077    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1078         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1079          VCLZQ))
1080   ]
1081   "TARGET_HAVE_MVE"
1082   "vclz.i%#<V_sz_elem>  %q0, %q1"
1083   [(set_attr "type" "mve_move")
1084 ])
1085
1086 ;;
1087 ;; [vclsq_s])
1088 ;;
1089 (define_insn "mve_vclsq_s<mode>"
1090   [
1091    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1092         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1093          VCLSQ_S))
1094   ]
1095   "TARGET_HAVE_MVE"
1096   "vcls.s%#<V_sz_elem>  %q0, %q1"
1097   [(set_attr "type" "mve_move")
1098 ])
1099
1100 ;;
1101 ;; [vaddvq_s, vaddvq_u])
1102 ;;
1103 (define_insn "mve_vaddvq_<supf><mode>"
1104   [
1105    (set (match_operand:SI 0 "s_register_operand" "=e")
1106         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
1107          VADDVQ))
1108   ]
1109   "TARGET_HAVE_MVE"
1110   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
1111   [(set_attr "type" "mve_move")
1112 ])
1113
1114 ;;
1115 ;; [vabsq_s])
1116 ;;
1117 (define_insn "mve_vabsq_s<mode>"
1118   [
1119    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1120         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1121          VABSQ_S))
1122   ]
1123   "TARGET_HAVE_MVE"
1124   "vabs.s%#<V_sz_elem>\t%q0, %q1"
1125   [(set_attr "type" "mve_move")
1126 ])
1127
1128 ;;
1129 ;; [vrev32q_u, vrev32q_s])
1130 ;;
1131 (define_insn "mve_vrev32q_<supf><mode>"
1132   [
1133    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
1134         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
1135          VREV32Q))
1136   ]
1137   "TARGET_HAVE_MVE"
1138   "vrev32.%#<V_sz_elem>\t%q0, %q1"
1139   [(set_attr "type" "mve_move")
1140 ])
1141
1142 ;;
1143 ;; [vmovltq_u, vmovltq_s])
1144 ;;
1145 (define_insn "mve_vmovltq_<supf><mode>"
1146   [
1147    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1148         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1149          VMOVLTQ))
1150   ]
1151   "TARGET_HAVE_MVE"
1152   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
1153   [(set_attr "type" "mve_move")
1154 ])
1155
1156 ;;
1157 ;; [vmovlbq_s, vmovlbq_u])
1158 ;;
1159 (define_insn "mve_vmovlbq_<supf><mode>"
1160   [
1161    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1162         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1163          VMOVLBQ))
1164   ]
1165   "TARGET_HAVE_MVE"
1166   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
1167   [(set_attr "type" "mve_move")
1168 ])
1169
1170 ;;
1171 ;; [vcvtpq_s, vcvtpq_u])
1172 ;;
1173 (define_insn "mve_vcvtpq_<supf><mode>"
1174   [
1175    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1176         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1177          VCVTPQ))
1178   ]
1179   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1180   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1181   [(set_attr "type" "mve_move")
1182 ])
1183
1184 ;;
1185 ;; [vcvtnq_s, vcvtnq_u])
1186 ;;
1187 (define_insn "mve_vcvtnq_<supf><mode>"
1188   [
1189    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1190         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1191          VCVTNQ))
1192   ]
1193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1194   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1195   [(set_attr "type" "mve_move")
1196 ])
1197
1198 ;;
1199 ;; [vcvtmq_s, vcvtmq_u])
1200 ;;
1201 (define_insn "mve_vcvtmq_<supf><mode>"
1202   [
1203    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1204         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1205          VCVTMQ))
1206   ]
1207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1208   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1209   [(set_attr "type" "mve_move")
1210 ])
1211
1212 ;;
1213 ;; [vcvtaq_u, vcvtaq_s])
1214 ;;
1215 (define_insn "mve_vcvtaq_<supf><mode>"
1216   [
1217    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1218         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1219          VCVTAQ))
1220   ]
1221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1222   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1223   [(set_attr "type" "mve_move")
1224 ])
1225
1226 ;;
1227 ;; [vmvnq_n_u, vmvnq_n_s])
1228 ;;
1229 (define_insn "mve_vmvnq_n_<supf><mode>"
1230   [
1231    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1232         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
1233          VMVNQ_N))
1234   ]
1235   "TARGET_HAVE_MVE"
1236   "vmvn.i%#<V_sz_elem>  %q0, %1"
1237   [(set_attr "type" "mve_move")
1238 ])
1239
1240 ;;
1241 ;; [vrev16q_u, vrev16q_s])
1242 ;;
1243 (define_insn "mve_vrev16q_<supf>v16qi"
1244   [
1245    (set (match_operand:V16QI 0 "s_register_operand" "=w")
1246         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
1247          VREV16Q))
1248   ]
1249   "TARGET_HAVE_MVE"
1250   "vrev16.8 %q0, %q1"
1251   [(set_attr "type" "mve_move")
1252 ])
1253
1254 ;;
1255 ;; [vaddlvq_s vaddlvq_u])
1256 ;;
1257 (define_insn "mve_vaddlvq_<supf>v4si"
1258   [
1259    (set (match_operand:DI 0 "s_register_operand" "=r")
1260         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
1261          VADDLVQ))
1262   ]
1263   "TARGET_HAVE_MVE"
1264   "vaddlv.<supf>32 %Q0, %R0, %q1"
1265   [(set_attr "type" "mve_move")
1266 ])
1267
1268 ;;
1269 ;; [vctp8q vctp16q vctp32q vctp64q])
1270 ;;
1271 (define_insn "mve_vctp<mode1>qhi"
1272   [
1273    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1274         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
1275         VCTPQ))
1276   ]
1277   "TARGET_HAVE_MVE"
1278   "vctp.<mode1> %1"
1279   [(set_attr "type" "mve_move")
1280 ])
1281
1282 ;;
1283 ;; [vpnot])
1284 ;;
1285 (define_insn "mve_vpnothi"
1286   [
1287    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1288         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
1289          VPNOT))
1290   ]
1291   "TARGET_HAVE_MVE"
1292   "vpnot"
1293   [(set_attr "type" "mve_move")
1294 ])
1295
1296 ;;
1297 ;; [vsubq_n_f])
1298 ;;
1299 (define_insn "mve_vsubq_n_f<mode>"
1300   [
1301    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1302         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1303                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1304          VSUBQ_N_F))
1305   ]
1306   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1307   "vsub.f<V_sz_elem>  %q0, %q1, %2"
1308   [(set_attr "type" "mve_move")
1309 ])
1310
1311 ;;
1312 ;; [vbrsrq_n_f])
1313 ;;
1314 (define_insn "mve_vbrsrq_n_f<mode>"
1315   [
1316    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1317         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1318                        (match_operand:SI 2 "s_register_operand" "r")]
1319          VBRSRQ_N_F))
1320   ]
1321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1322   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
1323   [(set_attr "type" "mve_move")
1324 ])
1325
1326 ;;
1327 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
1328 ;;
1329 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
1330   [
1331    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1332         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1333                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1334          VCVTQ_N_TO_F))
1335   ]
1336   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1337   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
1338   [(set_attr "type" "mve_move")
1339 ])
1340
1341 ;; [vcreateq_f])
1342 ;;
1343 (define_insn "mve_vcreateq_f<mode>"
1344   [
1345    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1346         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
1347                        (match_operand:DI 2 "s_register_operand" "r")]
1348          VCREATEQ_F))
1349   ]
1350   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1351   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1352   [(set_attr "type" "mve_move")
1353    (set_attr "length""8")])
1354
1355 ;;
1356 ;; [vcreateq_u, vcreateq_s])
1357 ;;
1358 (define_insn "mve_vcreateq_<supf><mode>"
1359   [
1360    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
1361         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
1362                        (match_operand:DI 2 "s_register_operand" "r")]
1363          VCREATEQ))
1364   ]
1365   "TARGET_HAVE_MVE"
1366   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1367   [(set_attr "type" "mve_move")
1368    (set_attr "length""8")])
1369
1370 ;;
1371 ;; [vshrq_n_s, vshrq_n_u])
1372 ;;
1373 (define_insn "mve_vshrq_n_<supf><mode>"
1374   [
1375    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1376         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1377                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1378          VSHRQ_N))
1379   ]
1380   "TARGET_HAVE_MVE"
1381   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
1382   [(set_attr "type" "mve_move")
1383 ])
1384
1385 ;;
1386 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
1387 ;;
1388 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
1389   [
1390    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1391         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1392                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1393          VCVTQ_N_FROM_F))
1394   ]
1395   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1396   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
1397   [(set_attr "type" "mve_move")
1398 ])
1399
1400 ;;
1401 ;; [vaddlvq_p_s])
1402 ;;
1403 (define_insn "mve_vaddlvq_p_<supf>v4si"
1404   [
1405    (set (match_operand:DI 0 "s_register_operand" "=r")
1406         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
1407                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1408          VADDLVQ_P))
1409   ]
1410   "TARGET_HAVE_MVE"
1411   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
1412   [(set_attr "type" "mve_move")
1413    (set_attr "length""8")])
1414
1415 ;;
1416 ;; [vcmpneq_u, vcmpneq_s])
1417 ;;
1418 (define_insn "mve_vcmpneq_<supf><mode>"
1419   [
1420    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1421         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1422                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1423          VCMPNEQ))
1424   ]
1425   "TARGET_HAVE_MVE"
1426   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
1427   [(set_attr "type" "mve_move")
1428 ])
1429
1430 ;;
1431 ;; [vshlq_s, vshlq_u])
1432 ;;
1433 (define_insn "mve_vshlq_<supf><mode>"
1434   [
1435    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1436         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1437                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1438          VSHLQ))
1439   ]
1440   "TARGET_HAVE_MVE"
1441   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1442   [(set_attr "type" "mve_move")
1443 ])
1444
1445 ;;
1446 ;; [vabdq_s, vabdq_u])
1447 ;;
1448 (define_insn "mve_vabdq_<supf><mode>"
1449   [
1450    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1451         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1452                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1453          VABDQ))
1454   ]
1455   "TARGET_HAVE_MVE"
1456   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
1457   [(set_attr "type" "mve_move")
1458 ])
1459
1460 ;;
1461 ;; [vaddq_n_s, vaddq_n_u])
1462 ;;
1463 (define_insn "mve_vaddq_n_<supf><mode>"
1464   [
1465    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1466         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1467                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1468          VADDQ_N))
1469   ]
1470   "TARGET_HAVE_MVE"
1471   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
1472   [(set_attr "type" "mve_move")
1473 ])
1474
1475 ;;
1476 ;; [vaddvaq_s, vaddvaq_u])
1477 ;;
1478 (define_insn "mve_vaddvaq_<supf><mode>"
1479   [
1480    (set (match_operand:SI 0 "s_register_operand" "=e")
1481         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1482                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1483          VADDVAQ))
1484   ]
1485   "TARGET_HAVE_MVE"
1486   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
1487   [(set_attr "type" "mve_move")
1488 ])
1489
1490 ;;
1491 ;; [vaddvq_p_u, vaddvq_p_s])
1492 ;;
1493 (define_insn "mve_vaddvq_p_<supf><mode>"
1494   [
1495    (set (match_operand:SI 0 "s_register_operand" "=e")
1496         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1497                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1498          VADDVQ_P))
1499   ]
1500   "TARGET_HAVE_MVE"
1501   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
1502   [(set_attr "type" "mve_move")
1503    (set_attr "length""8")])
1504
1505 ;;
1506 ;; [vandq_u, vandq_s])
1507 ;;
1508 (define_insn "mve_vandq_<supf><mode>"
1509   [
1510    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1511         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1512                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1513          VANDQ))
1514   ]
1515   "TARGET_HAVE_MVE"
1516   "vand %q0, %q1, %q2"
1517   [(set_attr "type" "mve_move")
1518 ])
1519
1520 ;;
1521 ;; [vbicq_s, vbicq_u])
1522 ;;
1523 (define_insn "mve_vbicq_<supf><mode>"
1524   [
1525    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1526         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1527                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1528          VBICQ))
1529   ]
1530   "TARGET_HAVE_MVE"
1531   "vbic %q0, %q1, %q2"
1532   [(set_attr "type" "mve_move")
1533 ])
1534
1535 ;;
1536 ;; [vbrsrq_n_u, vbrsrq_n_s])
1537 ;;
1538 (define_insn "mve_vbrsrq_n_<supf><mode>"
1539   [
1540    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1541         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1542                        (match_operand:SI 2 "s_register_operand" "r")]
1543          VBRSRQ_N))
1544   ]
1545   "TARGET_HAVE_MVE"
1546   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
1547   [(set_attr "type" "mve_move")
1548 ])
1549
1550 ;;
1551 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
1552 ;;
1553 (define_insn "mve_vcaddq_rot270_<supf><mode>"
1554   [
1555    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1556         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1557                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1558          VCADDQ_ROT270))
1559   ]
1560   "TARGET_HAVE_MVE"
1561   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
1562   [(set_attr "type" "mve_move")
1563 ])
1564
1565 ;;
1566 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
1567 ;;
1568 (define_insn "mve_vcaddq_rot90_<supf><mode>"
1569   [
1570    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1571         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1572                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1573          VCADDQ_ROT90))
1574   ]
1575   "TARGET_HAVE_MVE"
1576   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
1577   [(set_attr "type" "mve_move")
1578 ])
1579
1580 ;;
1581 ;; [vcmpcsq_n_u])
1582 ;;
1583 (define_insn "mve_vcmpcsq_n_u<mode>"
1584   [
1585    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1586         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1587                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1588          VCMPCSQ_N_U))
1589   ]
1590   "TARGET_HAVE_MVE"
1591   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1592   [(set_attr "type" "mve_move")
1593 ])
1594
1595 ;;
1596 ;; [vcmpcsq_u])
1597 ;;
1598 (define_insn "mve_vcmpcsq_u<mode>"
1599   [
1600    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1601         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1602                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1603          VCMPCSQ_U))
1604   ]
1605   "TARGET_HAVE_MVE"
1606   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1607   [(set_attr "type" "mve_move")
1608 ])
1609
1610 ;;
1611 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1612 ;;
1613 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1614   [
1615    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1616         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1617                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1618          VCMPEQQ_N))
1619   ]
1620   "TARGET_HAVE_MVE"
1621   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1622   [(set_attr "type" "mve_move")
1623 ])
1624
1625 ;;
1626 ;; [vcmpeqq_u, vcmpeqq_s])
1627 ;;
1628 (define_insn "mve_vcmpeqq_<supf><mode>"
1629   [
1630    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1631         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1632                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1633          VCMPEQQ))
1634   ]
1635   "TARGET_HAVE_MVE"
1636   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1637   [(set_attr "type" "mve_move")
1638 ])
1639
1640 ;;
1641 ;; [vcmpgeq_n_s])
1642 ;;
1643 (define_insn "mve_vcmpgeq_n_s<mode>"
1644   [
1645    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1646         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1647                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1648          VCMPGEQ_N_S))
1649   ]
1650   "TARGET_HAVE_MVE"
1651   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1652   [(set_attr "type" "mve_move")
1653 ])
1654
1655 ;;
1656 ;; [vcmpgeq_s])
1657 ;;
1658 (define_insn "mve_vcmpgeq_s<mode>"
1659   [
1660    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1661         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1662                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1663          VCMPGEQ_S))
1664   ]
1665   "TARGET_HAVE_MVE"
1666   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1667   [(set_attr "type" "mve_move")
1668 ])
1669
1670 ;;
1671 ;; [vcmpgtq_n_s])
1672 ;;
1673 (define_insn "mve_vcmpgtq_n_s<mode>"
1674   [
1675    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1676         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1677                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1678          VCMPGTQ_N_S))
1679   ]
1680   "TARGET_HAVE_MVE"
1681   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1682   [(set_attr "type" "mve_move")
1683 ])
1684
1685 ;;
1686 ;; [vcmpgtq_s])
1687 ;;
1688 (define_insn "mve_vcmpgtq_s<mode>"
1689   [
1690    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1691         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1692                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1693          VCMPGTQ_S))
1694   ]
1695   "TARGET_HAVE_MVE"
1696   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1697   [(set_attr "type" "mve_move")
1698 ])
1699
1700 ;;
1701 ;; [vcmphiq_n_u])
1702 ;;
1703 (define_insn "mve_vcmphiq_n_u<mode>"
1704   [
1705    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1706         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1707                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1708          VCMPHIQ_N_U))
1709   ]
1710   "TARGET_HAVE_MVE"
1711   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1712   [(set_attr "type" "mve_move")
1713 ])
1714
1715 ;;
1716 ;; [vcmphiq_u])
1717 ;;
1718 (define_insn "mve_vcmphiq_u<mode>"
1719   [
1720    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1721         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1722                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1723          VCMPHIQ_U))
1724   ]
1725   "TARGET_HAVE_MVE"
1726   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1727   [(set_attr "type" "mve_move")
1728 ])
1729
1730 ;;
1731 ;; [vcmpleq_n_s])
1732 ;;
1733 (define_insn "mve_vcmpleq_n_s<mode>"
1734   [
1735    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1736         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1737                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1738          VCMPLEQ_N_S))
1739   ]
1740   "TARGET_HAVE_MVE"
1741   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1742   [(set_attr "type" "mve_move")
1743 ])
1744
1745 ;;
1746 ;; [vcmpleq_s])
1747 ;;
1748 (define_insn "mve_vcmpleq_s<mode>"
1749   [
1750    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1751         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1752                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1753          VCMPLEQ_S))
1754   ]
1755   "TARGET_HAVE_MVE"
1756   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1757   [(set_attr "type" "mve_move")
1758 ])
1759
1760 ;;
1761 ;; [vcmpltq_n_s])
1762 ;;
1763 (define_insn "mve_vcmpltq_n_s<mode>"
1764   [
1765    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1766         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1767                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1768          VCMPLTQ_N_S))
1769   ]
1770   "TARGET_HAVE_MVE"
1771   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1772   [(set_attr "type" "mve_move")
1773 ])
1774
1775 ;;
1776 ;; [vcmpltq_s])
1777 ;;
1778 (define_insn "mve_vcmpltq_s<mode>"
1779   [
1780    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1781         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1782                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1783          VCMPLTQ_S))
1784   ]
1785   "TARGET_HAVE_MVE"
1786   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1787   [(set_attr "type" "mve_move")
1788 ])
1789
1790 ;;
1791 ;; [vcmpneq_n_u, vcmpneq_n_s])
1792 ;;
1793 (define_insn "mve_vcmpneq_n_<supf><mode>"
1794   [
1795    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1796         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1797                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1798          VCMPNEQ_N))
1799   ]
1800   "TARGET_HAVE_MVE"
1801   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1802   [(set_attr "type" "mve_move")
1803 ])
1804
1805 ;;
1806 ;; [veorq_u, veorq_s])
1807 ;;
1808 (define_insn "mve_veorq_<supf><mode>"
1809   [
1810    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1811         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1812                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1813          VEORQ))
1814   ]
1815   "TARGET_HAVE_MVE"
1816   "veor %q0, %q1, %q2"
1817   [(set_attr "type" "mve_move")
1818 ])
1819
1820 ;;
1821 ;; [vhaddq_n_u, vhaddq_n_s])
1822 ;;
1823 (define_insn "mve_vhaddq_n_<supf><mode>"
1824   [
1825    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1826         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1827                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1828          VHADDQ_N))
1829   ]
1830   "TARGET_HAVE_MVE"
1831   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1832   [(set_attr "type" "mve_move")
1833 ])
1834
1835 ;;
1836 ;; [vhaddq_s, vhaddq_u])
1837 ;;
1838 (define_insn "mve_vhaddq_<supf><mode>"
1839   [
1840    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1841         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1842                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1843          VHADDQ))
1844   ]
1845   "TARGET_HAVE_MVE"
1846   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1847   [(set_attr "type" "mve_move")
1848 ])
1849
1850 ;;
1851 ;; [vhcaddq_rot270_s])
1852 ;;
1853 (define_insn "mve_vhcaddq_rot270_s<mode>"
1854   [
1855    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1856         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1857                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1858          VHCADDQ_ROT270_S))
1859   ]
1860   "TARGET_HAVE_MVE"
1861   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1862   [(set_attr "type" "mve_move")
1863 ])
1864
1865 ;;
1866 ;; [vhcaddq_rot90_s])
1867 ;;
1868 (define_insn "mve_vhcaddq_rot90_s<mode>"
1869   [
1870    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1871         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1872                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1873          VHCADDQ_ROT90_S))
1874   ]
1875   "TARGET_HAVE_MVE"
1876   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1877   [(set_attr "type" "mve_move")
1878 ])
1879
1880 ;;
1881 ;; [vhsubq_n_u, vhsubq_n_s])
1882 ;;
1883 (define_insn "mve_vhsubq_n_<supf><mode>"
1884   [
1885    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1886         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1887                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1888          VHSUBQ_N))
1889   ]
1890   "TARGET_HAVE_MVE"
1891   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1892   [(set_attr "type" "mve_move")
1893 ])
1894
1895 ;;
1896 ;; [vhsubq_s, vhsubq_u])
1897 ;;
1898 (define_insn "mve_vhsubq_<supf><mode>"
1899   [
1900    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1901         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1902                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1903          VHSUBQ))
1904   ]
1905   "TARGET_HAVE_MVE"
1906   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1907   [(set_attr "type" "mve_move")
1908 ])
1909
1910 ;;
1911 ;; [vmaxaq_s])
1912 ;;
1913 (define_insn "mve_vmaxaq_s<mode>"
1914   [
1915    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1916         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1917                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1918          VMAXAQ_S))
1919   ]
1920   "TARGET_HAVE_MVE"
1921   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1922   [(set_attr "type" "mve_move")
1923 ])
1924
1925 ;;
1926 ;; [vmaxavq_s])
1927 ;;
1928 (define_insn "mve_vmaxavq_s<mode>"
1929   [
1930    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1931         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1932                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1933          VMAXAVQ_S))
1934   ]
1935   "TARGET_HAVE_MVE"
1936   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1937   [(set_attr "type" "mve_move")
1938 ])
1939
1940 ;;
1941 ;; [vmaxq_u, vmaxq_s])
1942 ;;
1943 (define_insn "mve_vmaxq_<supf><mode>"
1944   [
1945    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1946         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1947                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1948          VMAXQ))
1949   ]
1950   "TARGET_HAVE_MVE"
1951   "vmax.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1952   [(set_attr "type" "mve_move")
1953 ])
1954
1955 ;;
1956 ;; [vmaxvq_u, vmaxvq_s])
1957 ;;
1958 (define_insn "mve_vmaxvq_<supf><mode>"
1959   [
1960    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1961         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1962                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1963          VMAXVQ))
1964   ]
1965   "TARGET_HAVE_MVE"
1966   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1967   [(set_attr "type" "mve_move")
1968 ])
1969
1970 ;;
1971 ;; [vminaq_s])
1972 ;;
1973 (define_insn "mve_vminaq_s<mode>"
1974   [
1975    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1976         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1977                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1978          VMINAQ_S))
1979   ]
1980   "TARGET_HAVE_MVE"
1981   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1982   [(set_attr "type" "mve_move")
1983 ])
1984
1985 ;;
1986 ;; [vminavq_s])
1987 ;;
1988 (define_insn "mve_vminavq_s<mode>"
1989   [
1990    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1991         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1992                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1993          VMINAVQ_S))
1994   ]
1995   "TARGET_HAVE_MVE"
1996   "vminav.s%#<V_sz_elem>\t%0, %q2"
1997   [(set_attr "type" "mve_move")
1998 ])
1999
2000 ;;
2001 ;; [vminq_s, vminq_u])
2002 ;;
2003 (define_insn "mve_vminq_<supf><mode>"
2004   [
2005    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2006         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2007                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2008          VMINQ))
2009   ]
2010   "TARGET_HAVE_MVE"
2011   "vmin.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2012   [(set_attr "type" "mve_move")
2013 ])
2014
2015 ;;
2016 ;; [vminvq_u, vminvq_s])
2017 ;;
2018 (define_insn "mve_vminvq_<supf><mode>"
2019   [
2020    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2021         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2022                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2023          VMINVQ))
2024   ]
2025   "TARGET_HAVE_MVE"
2026   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
2027   [(set_attr "type" "mve_move")
2028 ])
2029
2030 ;;
2031 ;; [vmladavq_u, vmladavq_s])
2032 ;;
2033 (define_insn "mve_vmladavq_<supf><mode>"
2034   [
2035    (set (match_operand:SI 0 "s_register_operand" "=e")
2036         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2037                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2038          VMLADAVQ))
2039   ]
2040   "TARGET_HAVE_MVE"
2041   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
2042   [(set_attr "type" "mve_move")
2043 ])
2044
2045 ;;
2046 ;; [vmladavxq_s])
2047 ;;
2048 (define_insn "mve_vmladavxq_s<mode>"
2049   [
2050    (set (match_operand:SI 0 "s_register_operand" "=e")
2051         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2052                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2053          VMLADAVXQ_S))
2054   ]
2055   "TARGET_HAVE_MVE"
2056   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2057   [(set_attr "type" "mve_move")
2058 ])
2059
2060 ;;
2061 ;; [vmlsdavq_s])
2062 ;;
2063 (define_insn "mve_vmlsdavq_s<mode>"
2064   [
2065    (set (match_operand:SI 0 "s_register_operand" "=e")
2066         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2067                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2068          VMLSDAVQ_S))
2069   ]
2070   "TARGET_HAVE_MVE"
2071   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
2072   [(set_attr "type" "mve_move")
2073 ])
2074
2075 ;;
2076 ;; [vmlsdavxq_s])
2077 ;;
2078 (define_insn "mve_vmlsdavxq_s<mode>"
2079   [
2080    (set (match_operand:SI 0 "s_register_operand" "=e")
2081         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2082                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2083          VMLSDAVXQ_S))
2084   ]
2085   "TARGET_HAVE_MVE"
2086   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2087   [(set_attr "type" "mve_move")
2088 ])
2089
2090 ;;
2091 ;; [vmulhq_s, vmulhq_u])
2092 ;;
2093 (define_insn "mve_vmulhq_<supf><mode>"
2094   [
2095    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2096         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2097                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2098          VMULHQ))
2099   ]
2100   "TARGET_HAVE_MVE"
2101   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2102   [(set_attr "type" "mve_move")
2103 ])
2104
2105 ;;
2106 ;; [vmullbq_int_u, vmullbq_int_s])
2107 ;;
2108 (define_insn "mve_vmullbq_int_<supf><mode>"
2109   [
2110    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2111         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2112                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2113          VMULLBQ_INT))
2114   ]
2115   "TARGET_HAVE_MVE"
2116   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2117   [(set_attr "type" "mve_move")
2118 ])
2119
2120 ;;
2121 ;; [vmulltq_int_u, vmulltq_int_s])
2122 ;;
2123 (define_insn "mve_vmulltq_int_<supf><mode>"
2124   [
2125    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2126         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2127                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2128          VMULLTQ_INT))
2129   ]
2130   "TARGET_HAVE_MVE"
2131   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2132   [(set_attr "type" "mve_move")
2133 ])
2134
2135 ;;
2136 ;; [vmulq_n_u, vmulq_n_s])
2137 ;;
2138 (define_insn "mve_vmulq_n_<supf><mode>"
2139   [
2140    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2141         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2142                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2143          VMULQ_N))
2144   ]
2145   "TARGET_HAVE_MVE"
2146   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
2147   [(set_attr "type" "mve_move")
2148 ])
2149
2150 ;;
2151 ;; [vmulq_u, vmulq_s])
2152 ;;
2153 (define_insn "mve_vmulq_<supf><mode>"
2154   [
2155    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2156         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2157                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2158          VMULQ))
2159   ]
2160   "TARGET_HAVE_MVE"
2161   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
2162   [(set_attr "type" "mve_move")
2163 ])
2164
2165 ;;
2166 ;; [vornq_u, vornq_s])
2167 ;;
2168 (define_insn "mve_vornq_<supf><mode>"
2169   [
2170    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2171         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2172                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2173          VORNQ))
2174   ]
2175   "TARGET_HAVE_MVE"
2176   "vorn %q0, %q1, %q2"
2177   [(set_attr "type" "mve_move")
2178 ])
2179
2180 ;;
2181 ;; [vorrq_s, vorrq_u])
2182 ;;
2183 (define_insn "mve_vorrq_<supf><mode>"
2184   [
2185    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2186         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2187                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2188          VORRQ))
2189   ]
2190   "TARGET_HAVE_MVE"
2191   "vorr %q0, %q1, %q2"
2192   [(set_attr "type" "mve_move")
2193 ])
2194
2195 ;;
2196 ;; [vqaddq_n_s, vqaddq_n_u])
2197 ;;
2198 (define_insn "mve_vqaddq_n_<supf><mode>"
2199   [
2200    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2201         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2202                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2203          VQADDQ_N))
2204   ]
2205   "TARGET_HAVE_MVE"
2206   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2207   [(set_attr "type" "mve_move")
2208 ])
2209
2210 ;;
2211 ;; [vqaddq_u, vqaddq_s])
2212 ;;
2213 (define_insn "mve_vqaddq_<supf><mode>"
2214   [
2215    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2216         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2217                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2218          VQADDQ))
2219   ]
2220   "TARGET_HAVE_MVE"
2221   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2222   [(set_attr "type" "mve_move")
2223 ])
2224
2225 ;;
2226 ;; [vqdmulhq_n_s])
2227 ;;
2228 (define_insn "mve_vqdmulhq_n_s<mode>"
2229   [
2230    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2231         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2232                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2233          VQDMULHQ_N_S))
2234   ]
2235   "TARGET_HAVE_MVE"
2236   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2237   [(set_attr "type" "mve_move")
2238 ])
2239
2240 ;;
2241 ;; [vqdmulhq_s])
2242 ;;
2243 (define_insn "mve_vqdmulhq_s<mode>"
2244   [
2245    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2246         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2247                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2248          VQDMULHQ_S))
2249   ]
2250   "TARGET_HAVE_MVE"
2251   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2252   [(set_attr "type" "mve_move")
2253 ])
2254
2255 ;;
2256 ;; [vqrdmulhq_n_s])
2257 ;;
2258 (define_insn "mve_vqrdmulhq_n_s<mode>"
2259   [
2260    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2261         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2262                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2263          VQRDMULHQ_N_S))
2264   ]
2265   "TARGET_HAVE_MVE"
2266   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2267   [(set_attr "type" "mve_move")
2268 ])
2269
2270 ;;
2271 ;; [vqrdmulhq_s])
2272 ;;
2273 (define_insn "mve_vqrdmulhq_s<mode>"
2274   [
2275    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2276         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2277                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2278          VQRDMULHQ_S))
2279   ]
2280   "TARGET_HAVE_MVE"
2281   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2282   [(set_attr "type" "mve_move")
2283 ])
2284
2285 ;;
2286 ;; [vqrshlq_n_s, vqrshlq_n_u])
2287 ;;
2288 (define_insn "mve_vqrshlq_n_<supf><mode>"
2289   [
2290    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2291         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2292                        (match_operand:SI 2 "s_register_operand" "r")]
2293          VQRSHLQ_N))
2294   ]
2295   "TARGET_HAVE_MVE"
2296   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2297   [(set_attr "type" "mve_move")
2298 ])
2299
2300 ;;
2301 ;; [vqrshlq_s, vqrshlq_u])
2302 ;;
2303 (define_insn "mve_vqrshlq_<supf><mode>"
2304   [
2305    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2306         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2307                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2308          VQRSHLQ))
2309   ]
2310   "TARGET_HAVE_MVE"
2311   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2312   [(set_attr "type" "mve_move")
2313 ])
2314
2315 ;;
2316 ;; [vqshlq_n_s, vqshlq_n_u])
2317 ;;
2318 (define_insn "mve_vqshlq_n_<supf><mode>"
2319   [
2320    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2321         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2322                        (match_operand:SI 2 "immediate_operand" "i")]
2323          VQSHLQ_N))
2324   ]
2325   "TARGET_HAVE_MVE"
2326   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2327   [(set_attr "type" "mve_move")
2328 ])
2329
2330 ;;
2331 ;; [vqshlq_r_u, vqshlq_r_s])
2332 ;;
2333 (define_insn "mve_vqshlq_r_<supf><mode>"
2334   [
2335    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2336         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2337                        (match_operand:SI 2 "s_register_operand" "r")]
2338          VQSHLQ_R))
2339   ]
2340   "TARGET_HAVE_MVE"
2341   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
2342   [(set_attr "type" "mve_move")
2343 ])
2344
2345 ;;
2346 ;; [vqshlq_s, vqshlq_u])
2347 ;;
2348 (define_insn "mve_vqshlq_<supf><mode>"
2349   [
2350    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2351         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2352                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2353          VQSHLQ))
2354   ]
2355   "TARGET_HAVE_MVE"
2356   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2357   [(set_attr "type" "mve_move")
2358 ])
2359
2360 ;;
2361 ;; [vqshluq_n_s])
2362 ;;
2363 (define_insn "mve_vqshluq_n_s<mode>"
2364   [
2365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2367                        (match_operand:SI 2 "mve_imm_7" "Ra")]
2368          VQSHLUQ_N_S))
2369   ]
2370   "TARGET_HAVE_MVE"
2371   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
2372   [(set_attr "type" "mve_move")
2373 ])
2374
2375 ;;
2376 ;; [vqsubq_n_s, vqsubq_n_u])
2377 ;;
2378 (define_insn "mve_vqsubq_n_<supf><mode>"
2379   [
2380    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2381         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2382                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2383          VQSUBQ_N))
2384   ]
2385   "TARGET_HAVE_MVE"
2386   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2387   [(set_attr "type" "mve_move")
2388 ])
2389
2390 ;;
2391 ;; [vqsubq_u, vqsubq_s])
2392 ;;
2393 (define_insn "mve_vqsubq_<supf><mode>"
2394   [
2395    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2396         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2397                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2398          VQSUBQ))
2399   ]
2400   "TARGET_HAVE_MVE"
2401   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2402   [(set_attr "type" "mve_move")
2403 ])
2404
2405 ;;
2406 ;; [vrhaddq_s, vrhaddq_u])
2407 ;;
2408 (define_insn "mve_vrhaddq_<supf><mode>"
2409   [
2410    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2411         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2412                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2413          VRHADDQ))
2414   ]
2415   "TARGET_HAVE_MVE"
2416   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2417   [(set_attr "type" "mve_move")
2418 ])
2419
2420 ;;
2421 ;; [vrmulhq_s, vrmulhq_u])
2422 ;;
2423 (define_insn "mve_vrmulhq_<supf><mode>"
2424   [
2425    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2426         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2427                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2428          VRMULHQ))
2429   ]
2430   "TARGET_HAVE_MVE"
2431   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2432   [(set_attr "type" "mve_move")
2433 ])
2434
2435 ;;
2436 ;; [vrshlq_n_u, vrshlq_n_s])
2437 ;;
2438 (define_insn "mve_vrshlq_n_<supf><mode>"
2439   [
2440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2441         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2442                        (match_operand:SI 2 "s_register_operand" "r")]
2443          VRSHLQ_N))
2444   ]
2445   "TARGET_HAVE_MVE"
2446   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2447   [(set_attr "type" "mve_move")
2448 ])
2449
2450 ;;
2451 ;; [vrshlq_s, vrshlq_u])
2452 ;;
2453 (define_insn "mve_vrshlq_<supf><mode>"
2454   [
2455    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2456         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2457                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2458          VRSHLQ))
2459   ]
2460   "TARGET_HAVE_MVE"
2461   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2462   [(set_attr "type" "mve_move")
2463 ])
2464
2465 ;;
2466 ;; [vrshrq_n_s, vrshrq_n_u])
2467 ;;
2468 (define_insn "mve_vrshrq_n_<supf><mode>"
2469   [
2470    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2471         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2472                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
2473          VRSHRQ_N))
2474   ]
2475   "TARGET_HAVE_MVE"
2476   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2477   [(set_attr "type" "mve_move")
2478 ])
2479
2480 ;;
2481 ;; [vshlq_n_u, vshlq_n_s])
2482 ;;
2483 (define_insn "mve_vshlq_n_<supf><mode>"
2484   [
2485    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2486         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2487                        (match_operand:SI 2 "immediate_operand" "i")]
2488          VSHLQ_N))
2489   ]
2490   "TARGET_HAVE_MVE"
2491   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2492   [(set_attr "type" "mve_move")
2493 ])
2494
2495 ;;
2496 ;; [vshlq_r_s, vshlq_r_u])
2497 ;;
2498 (define_insn "mve_vshlq_r_<supf><mode>"
2499   [
2500    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2501         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2502                        (match_operand:SI 2 "s_register_operand" "r")]
2503          VSHLQ_R))
2504   ]
2505   "TARGET_HAVE_MVE"
2506   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
2507   [(set_attr "type" "mve_move")
2508 ])
2509
2510 ;;
2511 ;; [vsubq_n_s, vsubq_n_u])
2512 ;;
2513 (define_insn "mve_vsubq_n_<supf><mode>"
2514   [
2515    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2516         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2517                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2518          VSUBQ_N))
2519   ]
2520   "TARGET_HAVE_MVE"
2521   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
2522   [(set_attr "type" "mve_move")
2523 ])
2524
2525 ;;
2526 ;; [vsubq_s, vsubq_u])
2527 ;;
2528 (define_insn "mve_vsubq_<supf><mode>"
2529   [
2530    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2531         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2532                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2533          VSUBQ))
2534   ]
2535   "TARGET_HAVE_MVE"
2536   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2537   [(set_attr "type" "mve_move")
2538 ])
2539
2540 ;;
2541 ;; [vabdq_f])
2542 ;;
2543 (define_insn "mve_vabdq_f<mode>"
2544   [
2545    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2546         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2547                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2548          VABDQ_F))
2549   ]
2550   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2551   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2552   [(set_attr "type" "mve_move")
2553 ])
2554
2555 ;;
2556 ;; [vaddlvaq_s vaddlvaq_u])
2557 ;;
2558 (define_insn "mve_vaddlvaq_<supf>v4si"
2559   [
2560    (set (match_operand:DI 0 "s_register_operand" "=r")
2561         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2562                     (match_operand:V4SI 2 "s_register_operand" "w")]
2563          VADDLVAQ))
2564   ]
2565   "TARGET_HAVE_MVE"
2566   "vaddlva.<supf>32 %Q0, %R0, %q2"
2567   [(set_attr "type" "mve_move")
2568 ])
2569
2570 ;;
2571 ;; [vaddq_n_f])
2572 ;;
2573 (define_insn "mve_vaddq_n_f<mode>"
2574   [
2575    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2576         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2577                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2578          VADDQ_N_F))
2579   ]
2580   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2581   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2582   [(set_attr "type" "mve_move")
2583 ])
2584
2585 ;;
2586 ;; [vandq_f])
2587 ;;
2588 (define_insn "mve_vandq_f<mode>"
2589   [
2590    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2591         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2592                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2593          VANDQ_F))
2594   ]
2595   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2596   "vand %q0, %q1, %q2"
2597   [(set_attr "type" "mve_move")
2598 ])
2599
2600 ;;
2601 ;; [vbicq_f])
2602 ;;
2603 (define_insn "mve_vbicq_f<mode>"
2604   [
2605    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2606         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2607                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2608          VBICQ_F))
2609   ]
2610   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2611   "vbic %q0, %q1, %q2"
2612   [(set_attr "type" "mve_move")
2613 ])
2614
2615 ;;
2616 ;; [vbicq_n_s, vbicq_n_u])
2617 ;;
2618 (define_insn "mve_vbicq_n_<supf><mode>"
2619   [
2620    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2621         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2622                        (match_operand:SI 2 "immediate_operand" "i")]
2623          VBICQ_N))
2624   ]
2625   "TARGET_HAVE_MVE"
2626   "vbic.i%#<V_sz_elem>  %q0, %2"
2627   [(set_attr "type" "mve_move")
2628 ])
2629
2630 ;;
2631 ;; [vcaddq_rot270_f])
2632 ;;
2633 (define_insn "mve_vcaddq_rot270_f<mode>"
2634   [
2635    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2636         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2637                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2638          VCADDQ_ROT270_F))
2639   ]
2640   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2641   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2642   [(set_attr "type" "mve_move")
2643 ])
2644
2645 ;;
2646 ;; [vcaddq_rot90_f])
2647 ;;
2648 (define_insn "mve_vcaddq_rot90_f<mode>"
2649   [
2650    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2651         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2652                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2653          VCADDQ_ROT90_F))
2654   ]
2655   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2656   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2657   [(set_attr "type" "mve_move")
2658 ])
2659
2660 ;;
2661 ;; [vcmpeqq_f])
2662 ;;
2663 (define_insn "mve_vcmpeqq_f<mode>"
2664   [
2665    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2666         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2667                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2668          VCMPEQQ_F))
2669   ]
2670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2671   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2672   [(set_attr "type" "mve_move")
2673 ])
2674
2675 ;;
2676 ;; [vcmpeqq_n_f])
2677 ;;
2678 (define_insn "mve_vcmpeqq_n_f<mode>"
2679   [
2680    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2681         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2682                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2683          VCMPEQQ_N_F))
2684   ]
2685   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2686   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2687   [(set_attr "type" "mve_move")
2688 ])
2689
2690 ;;
2691 ;; [vcmpgeq_f])
2692 ;;
2693 (define_insn "mve_vcmpgeq_f<mode>"
2694   [
2695    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2696         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2697                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2698          VCMPGEQ_F))
2699   ]
2700   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2701   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2702   [(set_attr "type" "mve_move")
2703 ])
2704
2705 ;;
2706 ;; [vcmpgeq_n_f])
2707 ;;
2708 (define_insn "mve_vcmpgeq_n_f<mode>"
2709   [
2710    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2711         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2712                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2713          VCMPGEQ_N_F))
2714   ]
2715   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2716   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2717   [(set_attr "type" "mve_move")
2718 ])
2719
2720 ;;
2721 ;; [vcmpgtq_f])
2722 ;;
2723 (define_insn "mve_vcmpgtq_f<mode>"
2724   [
2725    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2726         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2727                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2728          VCMPGTQ_F))
2729   ]
2730   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2731   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2732   [(set_attr "type" "mve_move")
2733 ])
2734
2735 ;;
2736 ;; [vcmpgtq_n_f])
2737 ;;
2738 (define_insn "mve_vcmpgtq_n_f<mode>"
2739   [
2740    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2741         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2742                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2743          VCMPGTQ_N_F))
2744   ]
2745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2746   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2747   [(set_attr "type" "mve_move")
2748 ])
2749
2750 ;;
2751 ;; [vcmpleq_f])
2752 ;;
2753 (define_insn "mve_vcmpleq_f<mode>"
2754   [
2755    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2756         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2757                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2758          VCMPLEQ_F))
2759   ]
2760   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2761   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2762   [(set_attr "type" "mve_move")
2763 ])
2764
2765 ;;
2766 ;; [vcmpleq_n_f])
2767 ;;
2768 (define_insn "mve_vcmpleq_n_f<mode>"
2769   [
2770    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2771         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2772                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2773          VCMPLEQ_N_F))
2774   ]
2775   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2776   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2777   [(set_attr "type" "mve_move")
2778 ])
2779
2780 ;;
2781 ;; [vcmpltq_f])
2782 ;;
2783 (define_insn "mve_vcmpltq_f<mode>"
2784   [
2785    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2786         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2787                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2788          VCMPLTQ_F))
2789   ]
2790   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2791   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2792   [(set_attr "type" "mve_move")
2793 ])
2794
2795 ;;
2796 ;; [vcmpltq_n_f])
2797 ;;
2798 (define_insn "mve_vcmpltq_n_f<mode>"
2799   [
2800    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2801         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2802                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2803          VCMPLTQ_N_F))
2804   ]
2805   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2806   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2807   [(set_attr "type" "mve_move")
2808 ])
2809
2810 ;;
2811 ;; [vcmpneq_f])
2812 ;;
2813 (define_insn "mve_vcmpneq_f<mode>"
2814   [
2815    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2816         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2817                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2818          VCMPNEQ_F))
2819   ]
2820   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2821   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2822   [(set_attr "type" "mve_move")
2823 ])
2824
2825 ;;
2826 ;; [vcmpneq_n_f])
2827 ;;
2828 (define_insn "mve_vcmpneq_n_f<mode>"
2829   [
2830    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2831         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2832                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2833          VCMPNEQ_N_F))
2834   ]
2835   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2836   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2837   [(set_attr "type" "mve_move")
2838 ])
2839
2840 ;;
2841 ;; [vcmulq_f])
2842 ;;
2843 (define_insn "mve_vcmulq_f<mode>"
2844   [
2845    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2846         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2847                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2848          VCMULQ_F))
2849   ]
2850   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2851   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2852   [(set_attr "type" "mve_move")
2853 ])
2854
2855 ;;
2856 ;; [vcmulq_rot180_f])
2857 ;;
2858 (define_insn "mve_vcmulq_rot180_f<mode>"
2859   [
2860    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2861         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2862                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2863          VCMULQ_ROT180_F))
2864   ]
2865   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2866   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2867   [(set_attr "type" "mve_move")
2868 ])
2869
2870 ;;
2871 ;; [vcmulq_rot270_f])
2872 ;;
2873 (define_insn "mve_vcmulq_rot270_f<mode>"
2874   [
2875    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2876         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2877                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2878          VCMULQ_ROT270_F))
2879   ]
2880   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2881   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2882   [(set_attr "type" "mve_move")
2883 ])
2884
2885 ;;
2886 ;; [vcmulq_rot90_f])
2887 ;;
2888 (define_insn "mve_vcmulq_rot90_f<mode>"
2889   [
2890    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2891         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2892                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2893          VCMULQ_ROT90_F))
2894   ]
2895   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2896   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2897   [(set_attr "type" "mve_move")
2898 ])
2899
2900 ;;
2901 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2902 ;;
2903 (define_insn "mve_vctp<mode1>q_mhi"
2904   [
2905    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2906         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2907                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2908          VCTPQ_M))
2909   ]
2910   "TARGET_HAVE_MVE"
2911   "vpst\;vctpt.<mode1> %1"
2912   [(set_attr "type" "mve_move")
2913    (set_attr "length""8")])
2914
2915 ;;
2916 ;; [vcvtbq_f16_f32])
2917 ;;
2918 (define_insn "mve_vcvtbq_f16_f32v8hf"
2919   [
2920    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2921         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2922                       (match_operand:V4SF 2 "s_register_operand" "w")]
2923          VCVTBQ_F16_F32))
2924   ]
2925   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2926   "vcvtb.f16.f32 %q0, %q2"
2927   [(set_attr "type" "mve_move")
2928 ])
2929
2930 ;;
2931 ;; [vcvttq_f16_f32])
2932 ;;
2933 (define_insn "mve_vcvttq_f16_f32v8hf"
2934   [
2935    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2936         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2937                       (match_operand:V4SF 2 "s_register_operand" "w")]
2938          VCVTTQ_F16_F32))
2939   ]
2940   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2941   "vcvtt.f16.f32 %q0, %q2"
2942   [(set_attr "type" "mve_move")
2943 ])
2944
2945 ;;
2946 ;; [veorq_f])
2947 ;;
2948 (define_insn "mve_veorq_f<mode>"
2949   [
2950    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2951         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2952                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2953          VEORQ_F))
2954   ]
2955   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2956   "veor %q0, %q1, %q2"
2957   [(set_attr "type" "mve_move")
2958 ])
2959
2960 ;;
2961 ;; [vmaxnmaq_f])
2962 ;;
2963 (define_insn "mve_vmaxnmaq_f<mode>"
2964   [
2965    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2966         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2967                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2968          VMAXNMAQ_F))
2969   ]
2970   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2971   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2972   [(set_attr "type" "mve_move")
2973 ])
2974
2975 ;;
2976 ;; [vmaxnmavq_f])
2977 ;;
2978 (define_insn "mve_vmaxnmavq_f<mode>"
2979   [
2980    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2981         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2982                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2983          VMAXNMAVQ_F))
2984   ]
2985   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2986   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2987   [(set_attr "type" "mve_move")
2988 ])
2989
2990 ;;
2991 ;; [vmaxnmq_f])
2992 ;;
2993 (define_insn "mve_vmaxnmq_f<mode>"
2994   [
2995    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2996         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2997                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2998          VMAXNMQ_F))
2999   ]
3000   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3001   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3002   [(set_attr "type" "mve_move")
3003 ])
3004
3005 ;;
3006 ;; [vmaxnmvq_f])
3007 ;;
3008 (define_insn "mve_vmaxnmvq_f<mode>"
3009   [
3010    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3011         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3012                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3013          VMAXNMVQ_F))
3014   ]
3015   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3016   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
3017   [(set_attr "type" "mve_move")
3018 ])
3019
3020 ;;
3021 ;; [vminnmaq_f])
3022 ;;
3023 (define_insn "mve_vminnmaq_f<mode>"
3024   [
3025    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3026         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3027                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3028          VMINNMAQ_F))
3029   ]
3030   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3031   "vminnma.f%#<V_sz_elem>       %q0, %q2"
3032   [(set_attr "type" "mve_move")
3033 ])
3034
3035 ;;
3036 ;; [vminnmavq_f])
3037 ;;
3038 (define_insn "mve_vminnmavq_f<mode>"
3039   [
3040    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3041         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3042                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3043          VMINNMAVQ_F))
3044   ]
3045   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3046   "vminnmav.f%#<V_sz_elem>      %0, %q2"
3047   [(set_attr "type" "mve_move")
3048 ])
3049
3050 ;;
3051 ;; [vminnmq_f])
3052 ;;
3053 (define_insn "mve_vminnmq_f<mode>"
3054   [
3055    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3056         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3057                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3058          VMINNMQ_F))
3059   ]
3060   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3061   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3062   [(set_attr "type" "mve_move")
3063 ])
3064
3065 ;;
3066 ;; [vminnmvq_f])
3067 ;;
3068 (define_insn "mve_vminnmvq_f<mode>"
3069   [
3070    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3071         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3072                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3073          VMINNMVQ_F))
3074   ]
3075   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3076   "vminnmv.f%#<V_sz_elem>       %0, %q2"
3077   [(set_attr "type" "mve_move")
3078 ])
3079
3080 ;;
3081 ;; [vmlaldavq_u, vmlaldavq_s])
3082 ;;
3083 (define_insn "mve_vmlaldavq_<supf><mode>"
3084   [
3085    (set (match_operand:DI 0 "s_register_operand" "=r")
3086         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3087                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3088          VMLALDAVQ))
3089   ]
3090   "TARGET_HAVE_MVE"
3091   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3092   [(set_attr "type" "mve_move")
3093 ])
3094
3095 ;;
3096 ;; [vmlaldavxq_s])
3097 ;;
3098 (define_insn "mve_vmlaldavxq_s<mode>"
3099   [
3100    (set (match_operand:DI 0 "s_register_operand" "=r")
3101         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3102                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3103          VMLALDAVXQ_S))
3104   ]
3105   "TARGET_HAVE_MVE"
3106   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3107   [(set_attr "type" "mve_move")
3108 ])
3109
3110 ;;
3111 ;; [vmlsldavq_s])
3112 ;;
3113 (define_insn "mve_vmlsldavq_s<mode>"
3114   [
3115    (set (match_operand:DI 0 "s_register_operand" "=r")
3116         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3117                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3118          VMLSLDAVQ_S))
3119   ]
3120   "TARGET_HAVE_MVE"
3121   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3122   [(set_attr "type" "mve_move")
3123 ])
3124
3125 ;;
3126 ;; [vmlsldavxq_s])
3127 ;;
3128 (define_insn "mve_vmlsldavxq_s<mode>"
3129   [
3130    (set (match_operand:DI 0 "s_register_operand" "=r")
3131         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3132                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3133          VMLSLDAVXQ_S))
3134   ]
3135   "TARGET_HAVE_MVE"
3136   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3137   [(set_attr "type" "mve_move")
3138 ])
3139
3140 ;;
3141 ;; [vmovnbq_u, vmovnbq_s])
3142 ;;
3143 (define_insn "mve_vmovnbq_<supf><mode>"
3144   [
3145    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3146         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3147                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3148          VMOVNBQ))
3149   ]
3150   "TARGET_HAVE_MVE"
3151   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
3152   [(set_attr "type" "mve_move")
3153 ])
3154
3155 ;;
3156 ;; [vmovntq_s, vmovntq_u])
3157 ;;
3158 (define_insn "mve_vmovntq_<supf><mode>"
3159   [
3160    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3161         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3162                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3163          VMOVNTQ))
3164   ]
3165   "TARGET_HAVE_MVE"
3166   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
3167   [(set_attr "type" "mve_move")
3168 ])
3169
3170 ;;
3171 ;; [vmulq_f])
3172 ;;
3173 (define_insn "mve_vmulq_f<mode>"
3174   [
3175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3177                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3178          VMULQ_F))
3179   ]
3180   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3181   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
3182   [(set_attr "type" "mve_move")
3183 ])
3184
3185 ;;
3186 ;; [vmulq_n_f])
3187 ;;
3188 (define_insn "mve_vmulq_n_f<mode>"
3189   [
3190    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3191         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3192                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
3193          VMULQ_N_F))
3194   ]
3195   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3196   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
3197   [(set_attr "type" "mve_move")
3198 ])
3199
3200 ;;
3201 ;; [vornq_f])
3202 ;;
3203 (define_insn "mve_vornq_f<mode>"
3204   [
3205    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3206         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3207                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3208          VORNQ_F))
3209   ]
3210   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3211   "vorn %q0, %q1, %q2"
3212   [(set_attr "type" "mve_move")
3213 ])
3214
3215 ;;
3216 ;; [vorrq_f])
3217 ;;
3218 (define_insn "mve_vorrq_f<mode>"
3219   [
3220    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3221         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3222                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3223          VORRQ_F))
3224   ]
3225   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3226   "vorr %q0, %q1, %q2"
3227   [(set_attr "type" "mve_move")
3228 ])
3229
3230 ;;
3231 ;; [vorrq_n_u, vorrq_n_s])
3232 ;;
3233 (define_insn "mve_vorrq_n_<supf><mode>"
3234   [
3235    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3236         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3237                        (match_operand:SI 2 "immediate_operand" "i")]
3238          VORRQ_N))
3239   ]
3240   "TARGET_HAVE_MVE"
3241   "vorr.i%#<V_sz_elem>  %q0, %2"
3242   [(set_attr "type" "mve_move")
3243 ])
3244
3245 ;;
3246 ;; [vqdmullbq_n_s])
3247 ;;
3248 (define_insn "mve_vqdmullbq_n_s<mode>"
3249   [
3250    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3251         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3252                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3253          VQDMULLBQ_N_S))
3254   ]
3255   "TARGET_HAVE_MVE"
3256   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
3257   [(set_attr "type" "mve_move")
3258 ])
3259
3260 ;;
3261 ;; [vqdmullbq_s])
3262 ;;
3263 (define_insn "mve_vqdmullbq_s<mode>"
3264   [
3265    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3266         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3267                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3268          VQDMULLBQ_S))
3269   ]
3270   "TARGET_HAVE_MVE"
3271   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
3272   [(set_attr "type" "mve_move")
3273 ])
3274
3275 ;;
3276 ;; [vqdmulltq_n_s])
3277 ;;
3278 (define_insn "mve_vqdmulltq_n_s<mode>"
3279   [
3280    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3281         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3282                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3283          VQDMULLTQ_N_S))
3284   ]
3285   "TARGET_HAVE_MVE"
3286   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
3287   [(set_attr "type" "mve_move")
3288 ])
3289
3290 ;;
3291 ;; [vqdmulltq_s])
3292 ;;
3293 (define_insn "mve_vqdmulltq_s<mode>"
3294   [
3295    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3296         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3297                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3298          VQDMULLTQ_S))
3299   ]
3300   "TARGET_HAVE_MVE"
3301   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
3302   [(set_attr "type" "mve_move")
3303 ])
3304
3305 ;;
3306 ;; [vqmovnbq_u, vqmovnbq_s])
3307 ;;
3308 (define_insn "mve_vqmovnbq_<supf><mode>"
3309   [
3310    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3311         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3312                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3313          VQMOVNBQ))
3314   ]
3315   "TARGET_HAVE_MVE"
3316   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
3317   [(set_attr "type" "mve_move")
3318 ])
3319
3320 ;;
3321 ;; [vqmovntq_u, vqmovntq_s])
3322 ;;
3323 (define_insn "mve_vqmovntq_<supf><mode>"
3324   [
3325    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3326         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3327                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3328          VQMOVNTQ))
3329   ]
3330   "TARGET_HAVE_MVE"
3331   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
3332   [(set_attr "type" "mve_move")
3333 ])
3334
3335 ;;
3336 ;; [vqmovunbq_s])
3337 ;;
3338 (define_insn "mve_vqmovunbq_s<mode>"
3339   [
3340    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3341         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3342                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3343          VQMOVUNBQ_S))
3344   ]
3345   "TARGET_HAVE_MVE"
3346   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
3347   [(set_attr "type" "mve_move")
3348 ])
3349
3350 ;;
3351 ;; [vqmovuntq_s])
3352 ;;
3353 (define_insn "mve_vqmovuntq_s<mode>"
3354   [
3355    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3356         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3357                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3358          VQMOVUNTQ_S))
3359   ]
3360   "TARGET_HAVE_MVE"
3361   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
3362   [(set_attr "type" "mve_move")
3363 ])
3364
3365 ;;
3366 ;; [vrmlaldavhxq_s])
3367 ;;
3368 (define_insn "mve_vrmlaldavhxq_sv4si"
3369   [
3370    (set (match_operand:DI 0 "s_register_operand" "=r")
3371         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3372                     (match_operand:V4SI 2 "s_register_operand" "w")]
3373          VRMLALDAVHXQ_S))
3374   ]
3375   "TARGET_HAVE_MVE"
3376   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
3377   [(set_attr "type" "mve_move")
3378 ])
3379
3380 ;;
3381 ;; [vrmlsldavhq_s])
3382 ;;
3383 (define_insn "mve_vrmlsldavhq_sv4si"
3384   [
3385    (set (match_operand:DI 0 "s_register_operand" "=r")
3386         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3387                     (match_operand:V4SI 2 "s_register_operand" "w")]
3388          VRMLSLDAVHQ_S))
3389   ]
3390   "TARGET_HAVE_MVE"
3391   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
3392   [(set_attr "type" "mve_move")
3393 ])
3394
3395 ;;
3396 ;; [vrmlsldavhxq_s])
3397 ;;
3398 (define_insn "mve_vrmlsldavhxq_sv4si"
3399   [
3400    (set (match_operand:DI 0 "s_register_operand" "=r")
3401         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3402                     (match_operand:V4SI 2 "s_register_operand" "w")]
3403          VRMLSLDAVHXQ_S))
3404   ]
3405   "TARGET_HAVE_MVE"
3406   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
3407   [(set_attr "type" "mve_move")
3408 ])
3409
3410 ;;
3411 ;; [vshllbq_n_s, vshllbq_n_u])
3412 ;;
3413 (define_insn "mve_vshllbq_n_<supf><mode>"
3414   [
3415    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3416         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3417                                   (match_operand:SI 2 "immediate_operand" "i")]
3418          VSHLLBQ_N))
3419   ]
3420   "TARGET_HAVE_MVE"
3421   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3422   [(set_attr "type" "mve_move")
3423 ])
3424
3425 ;;
3426 ;; [vshlltq_n_u, vshlltq_n_s])
3427 ;;
3428 (define_insn "mve_vshlltq_n_<supf><mode>"
3429   [
3430    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3431         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3432                                   (match_operand:SI 2 "immediate_operand" "i")]
3433          VSHLLTQ_N))
3434   ]
3435   "TARGET_HAVE_MVE"
3436   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3437   [(set_attr "type" "mve_move")
3438 ])
3439
3440 ;;
3441 ;; [vsubq_f])
3442 ;;
3443 (define_insn "mve_vsubq_f<mode>"
3444   [
3445    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3446         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3447                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3448          VSUBQ_F))
3449   ]
3450   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3451   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
3452   [(set_attr "type" "mve_move")
3453 ])
3454
3455 ;;
3456 ;; [vmulltq_poly_p])
3457 ;;
3458 (define_insn "mve_vmulltq_poly_p<mode>"
3459   [
3460    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3461         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3462                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3463          VMULLTQ_POLY_P))
3464   ]
3465   "TARGET_HAVE_MVE"
3466   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
3467   [(set_attr "type" "mve_move")
3468 ])
3469
3470 ;;
3471 ;; [vmullbq_poly_p])
3472 ;;
3473 (define_insn "mve_vmullbq_poly_p<mode>"
3474   [
3475    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3476         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3477                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3478          VMULLBQ_POLY_P))
3479   ]
3480   "TARGET_HAVE_MVE"
3481   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
3482   [(set_attr "type" "mve_move")
3483 ])
3484
3485 ;;
3486 ;; [vrmlaldavhq_u vrmlaldavhq_s])
3487 ;;
3488 (define_insn "mve_vrmlaldavhq_<supf>v4si"
3489   [
3490    (set (match_operand:DI 0 "s_register_operand" "=r")
3491         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3492                     (match_operand:V4SI 2 "s_register_operand" "w")]
3493          VRMLALDAVHQ))
3494   ]
3495   "TARGET_HAVE_MVE"
3496   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
3497   [(set_attr "type" "mve_move")
3498 ])
3499
3500 ;;
3501 ;; [vbicq_m_n_s, vbicq_m_n_u])
3502 ;;
3503 (define_insn "mve_vbicq_m_n_<supf><mode>"
3504   [
3505    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3506         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3507                        (match_operand:SI 2 "immediate_operand" "i")
3508                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3509          VBICQ_M_N))
3510   ]
3511   "TARGET_HAVE_MVE"
3512   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
3513   [(set_attr "type" "mve_move")
3514    (set_attr "length""8")])
3515 ;;
3516 ;; [vcmpeqq_m_f])
3517 ;;
3518 (define_insn "mve_vcmpeqq_m_f<mode>"
3519   [
3520    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3521         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3522                     (match_operand:MVE_0 2 "s_register_operand" "w")
3523                     (match_operand:HI 3 "vpr_register_operand" "Up")]
3524          VCMPEQQ_M_F))
3525   ]
3526   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3527   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
3528   [(set_attr "type" "mve_move")
3529    (set_attr "length""8")])
3530 ;;
3531 ;; [vcvtaq_m_u, vcvtaq_m_s])
3532 ;;
3533 (define_insn "mve_vcvtaq_m_<supf><mode>"
3534   [
3535    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3536         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3537                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3538                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3539          VCVTAQ_M))
3540   ]
3541   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3542   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3543   [(set_attr "type" "mve_move")
3544    (set_attr "length""8")])
3545 ;;
3546 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3547 ;;
3548 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3549   [
3550    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3551         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3552                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3553                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3554          VCVTQ_M_TO_F))
3555   ]
3556   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3557   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3558   [(set_attr "type" "mve_move")
3559    (set_attr "length""8")])
3560 ;;
3561 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3562 ;;
3563 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3564   [
3565    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3566         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3567                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3568                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3569          VQRSHRNBQ_N))
3570   ]
3571   "TARGET_HAVE_MVE"
3572   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3573   [(set_attr "type" "mve_move")
3574 ])
3575 ;;
3576 ;; [vqrshrunbq_n_s])
3577 ;;
3578 (define_insn "mve_vqrshrunbq_n_s<mode>"
3579   [
3580    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3581         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3582                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3583                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3584          VQRSHRUNBQ_N_S))
3585   ]
3586   "TARGET_HAVE_MVE"
3587   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3588   [(set_attr "type" "mve_move")
3589 ])
3590 ;;
3591 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3592 ;;
3593 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3594   [
3595    (set (match_operand:DI 0 "s_register_operand" "=r")
3596         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3597                     (match_operand:V4SI 2 "s_register_operand" "w")
3598                     (match_operand:V4SI 3 "s_register_operand" "w")]
3599          VRMLALDAVHAQ))
3600   ]
3601   "TARGET_HAVE_MVE"
3602   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3603   [(set_attr "type" "mve_move")
3604 ])
3605
3606 ;;
3607 ;; [vabavq_s, vabavq_u])
3608 ;;
3609 (define_insn "mve_vabavq_<supf><mode>"
3610   [
3611    (set (match_operand:SI 0 "s_register_operand" "=r")
3612         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3613                     (match_operand:MVE_2 2 "s_register_operand" "w")
3614                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3615          VABAVQ))
3616   ]
3617   "TARGET_HAVE_MVE"
3618   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3619   [(set_attr "type" "mve_move")
3620 ])
3621
3622 ;;
3623 ;; [vshlcq_u vshlcq_s]
3624 ;;
3625 (define_expand "mve_vshlcq_vec_<supf><mode>"
3626  [(match_operand:MVE_2 0 "s_register_operand")
3627   (match_operand:MVE_2 1 "s_register_operand")
3628   (match_operand:SI 2 "s_register_operand")
3629   (match_operand:SI 3 "mve_imm_32")
3630   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3631  "TARGET_HAVE_MVE"
3632 {
3633   rtx ignore_wb = gen_reg_rtx (SImode);
3634   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3635                                       operands[2], operands[3]));
3636   DONE;
3637 })
3638
3639 (define_expand "mve_vshlcq_carry_<supf><mode>"
3640  [(match_operand:SI 0 "s_register_operand")
3641   (match_operand:MVE_2 1 "s_register_operand")
3642   (match_operand:SI 2 "s_register_operand")
3643   (match_operand:SI 3 "mve_imm_32")
3644   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3645  "TARGET_HAVE_MVE"
3646 {
3647   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3648   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3649                                       operands[2], operands[3]));
3650   DONE;
3651 })
3652
3653 (define_insn "mve_vshlcq_<supf><mode>"
3654  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3655        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3656                       (match_operand:SI 3 "s_register_operand" "1")
3657                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3658         VSHLCQ))
3659   (set (match_operand:SI  1 "s_register_operand" "=r")
3660        (unspec:SI [(match_dup 2)
3661                    (match_dup 3)
3662                    (match_dup 4)]
3663         VSHLCQ))]
3664  "TARGET_HAVE_MVE"
3665  "vshlc %q0, %1, %4")
3666
3667 ;;
3668 ;; [vabsq_m_s])
3669 ;;
3670 (define_insn "mve_vabsq_m_s<mode>"
3671   [
3672    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3673         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3674                        (match_operand:MVE_2 2 "s_register_operand" "w")
3675                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3676          VABSQ_M_S))
3677   ]
3678   "TARGET_HAVE_MVE"
3679   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3680   [(set_attr "type" "mve_move")
3681    (set_attr "length""8")])
3682
3683 ;;
3684 ;; [vaddvaq_p_u, vaddvaq_p_s])
3685 ;;
3686 (define_insn "mve_vaddvaq_p_<supf><mode>"
3687   [
3688    (set (match_operand:SI 0 "s_register_operand" "=e")
3689         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3690                        (match_operand:MVE_2 2 "s_register_operand" "w")
3691                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3692          VADDVAQ_P))
3693   ]
3694   "TARGET_HAVE_MVE"
3695   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3696   [(set_attr "type" "mve_move")
3697    (set_attr "length""8")])
3698
3699 ;;
3700 ;; [vclsq_m_s])
3701 ;;
3702 (define_insn "mve_vclsq_m_s<mode>"
3703   [
3704    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3705         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3706                        (match_operand:MVE_2 2 "s_register_operand" "w")
3707                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3708          VCLSQ_M_S))
3709   ]
3710   "TARGET_HAVE_MVE"
3711   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3712   [(set_attr "type" "mve_move")
3713    (set_attr "length""8")])
3714
3715 ;;
3716 ;; [vclzq_m_s, vclzq_m_u])
3717 ;;
3718 (define_insn "mve_vclzq_m_<supf><mode>"
3719   [
3720    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3721         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3722                        (match_operand:MVE_2 2 "s_register_operand" "w")
3723                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3724          VCLZQ_M))
3725   ]
3726   "TARGET_HAVE_MVE"
3727   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3728   [(set_attr "type" "mve_move")
3729    (set_attr "length""8")])
3730
3731 ;;
3732 ;; [vcmpcsq_m_n_u])
3733 ;;
3734 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3735   [
3736    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3737         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3738                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3739                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3740          VCMPCSQ_M_N_U))
3741   ]
3742   "TARGET_HAVE_MVE"
3743   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3744   [(set_attr "type" "mve_move")
3745    (set_attr "length""8")])
3746
3747 ;;
3748 ;; [vcmpcsq_m_u])
3749 ;;
3750 (define_insn "mve_vcmpcsq_m_u<mode>"
3751   [
3752    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3753         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3754                        (match_operand:MVE_2 2 "s_register_operand" "w")
3755                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3756          VCMPCSQ_M_U))
3757   ]
3758   "TARGET_HAVE_MVE"
3759   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3760   [(set_attr "type" "mve_move")
3761    (set_attr "length""8")])
3762
3763 ;;
3764 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3765 ;;
3766 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3767   [
3768    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3769         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3770                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3771                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3772          VCMPEQQ_M_N))
3773   ]
3774   "TARGET_HAVE_MVE"
3775   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3776   [(set_attr "type" "mve_move")
3777    (set_attr "length""8")])
3778
3779 ;;
3780 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3781 ;;
3782 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3783   [
3784    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3785         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3786                        (match_operand:MVE_2 2 "s_register_operand" "w")
3787                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3788          VCMPEQQ_M))
3789   ]
3790   "TARGET_HAVE_MVE"
3791   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3792   [(set_attr "type" "mve_move")
3793    (set_attr "length""8")])
3794
3795 ;;
3796 ;; [vcmpgeq_m_n_s])
3797 ;;
3798 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3799   [
3800    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3801         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3802                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3803                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3804          VCMPGEQ_M_N_S))
3805   ]
3806   "TARGET_HAVE_MVE"
3807   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3808   [(set_attr "type" "mve_move")
3809    (set_attr "length""8")])
3810
3811 ;;
3812 ;; [vcmpgeq_m_s])
3813 ;;
3814 (define_insn "mve_vcmpgeq_m_s<mode>"
3815   [
3816    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3817         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3818                        (match_operand:MVE_2 2 "s_register_operand" "w")
3819                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3820          VCMPGEQ_M_S))
3821   ]
3822   "TARGET_HAVE_MVE"
3823   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3824   [(set_attr "type" "mve_move")
3825    (set_attr "length""8")])
3826
3827 ;;
3828 ;; [vcmpgtq_m_n_s])
3829 ;;
3830 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3831   [
3832    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3833         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3834                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3835                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3836          VCMPGTQ_M_N_S))
3837   ]
3838   "TARGET_HAVE_MVE"
3839   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3840   [(set_attr "type" "mve_move")
3841    (set_attr "length""8")])
3842
3843 ;;
3844 ;; [vcmpgtq_m_s])
3845 ;;
3846 (define_insn "mve_vcmpgtq_m_s<mode>"
3847   [
3848    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3849         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3850                        (match_operand:MVE_2 2 "s_register_operand" "w")
3851                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3852          VCMPGTQ_M_S))
3853   ]
3854   "TARGET_HAVE_MVE"
3855   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3856   [(set_attr "type" "mve_move")
3857    (set_attr "length""8")])
3858
3859 ;;
3860 ;; [vcmphiq_m_n_u])
3861 ;;
3862 (define_insn "mve_vcmphiq_m_n_u<mode>"
3863   [
3864    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3865         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3866                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3867                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3868          VCMPHIQ_M_N_U))
3869   ]
3870   "TARGET_HAVE_MVE"
3871   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3872   [(set_attr "type" "mve_move")
3873    (set_attr "length""8")])
3874
3875 ;;
3876 ;; [vcmphiq_m_u])
3877 ;;
3878 (define_insn "mve_vcmphiq_m_u<mode>"
3879   [
3880    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3881         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3882                        (match_operand:MVE_2 2 "s_register_operand" "w")
3883                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3884          VCMPHIQ_M_U))
3885   ]
3886   "TARGET_HAVE_MVE"
3887   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3888   [(set_attr "type" "mve_move")
3889    (set_attr "length""8")])
3890
3891 ;;
3892 ;; [vcmpleq_m_n_s])
3893 ;;
3894 (define_insn "mve_vcmpleq_m_n_s<mode>"
3895   [
3896    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3897         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3898                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3899                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3900          VCMPLEQ_M_N_S))
3901   ]
3902   "TARGET_HAVE_MVE"
3903   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3904   [(set_attr "type" "mve_move")
3905    (set_attr "length""8")])
3906
3907 ;;
3908 ;; [vcmpleq_m_s])
3909 ;;
3910 (define_insn "mve_vcmpleq_m_s<mode>"
3911   [
3912    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3913         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3914                        (match_operand:MVE_2 2 "s_register_operand" "w")
3915                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3916          VCMPLEQ_M_S))
3917   ]
3918   "TARGET_HAVE_MVE"
3919   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3920   [(set_attr "type" "mve_move")
3921    (set_attr "length""8")])
3922
3923 ;;
3924 ;; [vcmpltq_m_n_s])
3925 ;;
3926 (define_insn "mve_vcmpltq_m_n_s<mode>"
3927   [
3928    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3929         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3930                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3931                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3932          VCMPLTQ_M_N_S))
3933   ]
3934   "TARGET_HAVE_MVE"
3935   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3936   [(set_attr "type" "mve_move")
3937    (set_attr "length""8")])
3938
3939 ;;
3940 ;; [vcmpltq_m_s])
3941 ;;
3942 (define_insn "mve_vcmpltq_m_s<mode>"
3943   [
3944    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3945         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3946                        (match_operand:MVE_2 2 "s_register_operand" "w")
3947                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3948          VCMPLTQ_M_S))
3949   ]
3950   "TARGET_HAVE_MVE"
3951   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3952   [(set_attr "type" "mve_move")
3953    (set_attr "length""8")])
3954
3955 ;;
3956 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3957 ;;
3958 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3959   [
3960    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3961         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3962                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3963                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3964          VCMPNEQ_M_N))
3965   ]
3966   "TARGET_HAVE_MVE"
3967   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3968   [(set_attr "type" "mve_move")
3969    (set_attr "length""8")])
3970
3971 ;;
3972 ;; [vcmpneq_m_s, vcmpneq_m_u])
3973 ;;
3974 (define_insn "mve_vcmpneq_m_<supf><mode>"
3975   [
3976    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3977         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3978                        (match_operand:MVE_2 2 "s_register_operand" "w")
3979                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3980          VCMPNEQ_M))
3981   ]
3982   "TARGET_HAVE_MVE"
3983   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3984   [(set_attr "type" "mve_move")
3985    (set_attr "length""8")])
3986
3987 ;;
3988 ;; [vdupq_m_n_s, vdupq_m_n_u])
3989 ;;
3990 (define_insn "mve_vdupq_m_n_<supf><mode>"
3991   [
3992    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3993         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3994                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3995                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3996          VDUPQ_M_N))
3997   ]
3998   "TARGET_HAVE_MVE"
3999   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4000   [(set_attr "type" "mve_move")
4001    (set_attr "length""8")])
4002
4003 ;;
4004 ;; [vmaxaq_m_s])
4005 ;;
4006 (define_insn "mve_vmaxaq_m_s<mode>"
4007   [
4008    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4009         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4010                        (match_operand:MVE_2 2 "s_register_operand" "w")
4011                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4012          VMAXAQ_M_S))
4013   ]
4014   "TARGET_HAVE_MVE"
4015   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
4016   [(set_attr "type" "mve_move")
4017    (set_attr "length""8")])
4018
4019 ;;
4020 ;; [vmaxavq_p_s])
4021 ;;
4022 (define_insn "mve_vmaxavq_p_s<mode>"
4023   [
4024    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4025         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4026                        (match_operand:MVE_2 2 "s_register_operand" "w")
4027                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4028          VMAXAVQ_P_S))
4029   ]
4030   "TARGET_HAVE_MVE"
4031   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
4032   [(set_attr "type" "mve_move")
4033    (set_attr "length""8")])
4034
4035 ;;
4036 ;; [vmaxvq_p_u, vmaxvq_p_s])
4037 ;;
4038 (define_insn "mve_vmaxvq_p_<supf><mode>"
4039   [
4040    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4041         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4042                        (match_operand:MVE_2 2 "s_register_operand" "w")
4043                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4044          VMAXVQ_P))
4045   ]
4046   "TARGET_HAVE_MVE"
4047   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
4048   [(set_attr "type" "mve_move")
4049    (set_attr "length""8")])
4050
4051 ;;
4052 ;; [vminaq_m_s])
4053 ;;
4054 (define_insn "mve_vminaq_m_s<mode>"
4055   [
4056    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4057         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4058                        (match_operand:MVE_2 2 "s_register_operand" "w")
4059                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4060          VMINAQ_M_S))
4061   ]
4062   "TARGET_HAVE_MVE"
4063   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
4064   [(set_attr "type" "mve_move")
4065    (set_attr "length""8")])
4066
4067 ;;
4068 ;; [vminavq_p_s])
4069 ;;
4070 (define_insn "mve_vminavq_p_s<mode>"
4071   [
4072    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4073         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4074                        (match_operand:MVE_2 2 "s_register_operand" "w")
4075                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4076          VMINAVQ_P_S))
4077   ]
4078   "TARGET_HAVE_MVE"
4079   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
4080   [(set_attr "type" "mve_move")
4081    (set_attr "length""8")])
4082
4083 ;;
4084 ;; [vminvq_p_s, vminvq_p_u])
4085 ;;
4086 (define_insn "mve_vminvq_p_<supf><mode>"
4087   [
4088    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4089         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4090                        (match_operand:MVE_2 2 "s_register_operand" "w")
4091                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4092          VMINVQ_P))
4093   ]
4094   "TARGET_HAVE_MVE"
4095   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
4096   [(set_attr "type" "mve_move")
4097    (set_attr "length""8")])
4098
4099 ;;
4100 ;; [vmladavaq_u, vmladavaq_s])
4101 ;;
4102 (define_insn "mve_vmladavaq_<supf><mode>"
4103   [
4104    (set (match_operand:SI 0 "s_register_operand" "=e")
4105         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4106                        (match_operand:MVE_2 2 "s_register_operand" "w")
4107                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4108          VMLADAVAQ))
4109   ]
4110   "TARGET_HAVE_MVE"
4111   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
4112   [(set_attr "type" "mve_move")
4113 ])
4114
4115 ;;
4116 ;; [vmladavq_p_u, vmladavq_p_s])
4117 ;;
4118 (define_insn "mve_vmladavq_p_<supf><mode>"
4119   [
4120    (set (match_operand:SI 0 "s_register_operand" "=e")
4121         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4122                        (match_operand:MVE_2 2 "s_register_operand" "w")
4123                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4124          VMLADAVQ_P))
4125   ]
4126   "TARGET_HAVE_MVE"
4127   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
4128   [(set_attr "type" "mve_move")
4129    (set_attr "length""8")])
4130
4131 ;;
4132 ;; [vmladavxq_p_s])
4133 ;;
4134 (define_insn "mve_vmladavxq_p_s<mode>"
4135   [
4136    (set (match_operand:SI 0 "s_register_operand" "=e")
4137         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4138                        (match_operand:MVE_2 2 "s_register_operand" "w")
4139                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4140          VMLADAVXQ_P_S))
4141   ]
4142   "TARGET_HAVE_MVE"
4143   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
4144   [(set_attr "type" "mve_move")
4145    (set_attr "length""8")])
4146
4147 ;;
4148 ;; [vmlaq_n_u, vmlaq_n_s])
4149 ;;
4150 (define_insn "mve_vmlaq_n_<supf><mode>"
4151   [
4152    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4153         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4154                        (match_operand:MVE_2 2 "s_register_operand" "w")
4155                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4156          VMLAQ_N))
4157   ]
4158   "TARGET_HAVE_MVE"
4159   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4160   [(set_attr "type" "mve_move")
4161 ])
4162
4163 ;;
4164 ;; [vmlasq_n_u, vmlasq_n_s])
4165 ;;
4166 (define_insn "mve_vmlasq_n_<supf><mode>"
4167   [
4168    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4169         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4170                        (match_operand:MVE_2 2 "s_register_operand" "w")
4171                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4172          VMLASQ_N))
4173   ]
4174   "TARGET_HAVE_MVE"
4175   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
4176   [(set_attr "type" "mve_move")
4177 ])
4178
4179 ;;
4180 ;; [vmlsdavq_p_s])
4181 ;;
4182 (define_insn "mve_vmlsdavq_p_s<mode>"
4183   [
4184    (set (match_operand:SI 0 "s_register_operand" "=e")
4185         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4186                        (match_operand:MVE_2 2 "s_register_operand" "w")
4187                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4188          VMLSDAVQ_P_S))
4189   ]
4190   "TARGET_HAVE_MVE"
4191   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
4192   [(set_attr "type" "mve_move")
4193    (set_attr "length""8")])
4194
4195 ;;
4196 ;; [vmlsdavxq_p_s])
4197 ;;
4198 (define_insn "mve_vmlsdavxq_p_s<mode>"
4199   [
4200    (set (match_operand:SI 0 "s_register_operand" "=e")
4201         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4202                        (match_operand:MVE_2 2 "s_register_operand" "w")
4203                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4204          VMLSDAVXQ_P_S))
4205   ]
4206   "TARGET_HAVE_MVE"
4207   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
4208   [(set_attr "type" "mve_move")
4209    (set_attr "length""8")])
4210
4211 ;;
4212 ;; [vmvnq_m_s, vmvnq_m_u])
4213 ;;
4214 (define_insn "mve_vmvnq_m_<supf><mode>"
4215   [
4216    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4217         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4218                        (match_operand:MVE_2 2 "s_register_operand" "w")
4219                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4220          VMVNQ_M))
4221   ]
4222   "TARGET_HAVE_MVE"
4223   "vpst\;vmvnt %q0, %q2"
4224   [(set_attr "type" "mve_move")
4225    (set_attr "length""8")])
4226
4227 ;;
4228 ;; [vnegq_m_s])
4229 ;;
4230 (define_insn "mve_vnegq_m_s<mode>"
4231   [
4232    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4233         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4234                        (match_operand:MVE_2 2 "s_register_operand" "w")
4235                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4236          VNEGQ_M_S))
4237   ]
4238   "TARGET_HAVE_MVE"
4239   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
4240   [(set_attr "type" "mve_move")
4241    (set_attr "length""8")])
4242
4243 ;;
4244 ;; [vpselq_u, vpselq_s])
4245 ;;
4246 (define_insn "mve_vpselq_<supf><mode>"
4247   [
4248    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
4249         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
4250                        (match_operand:MVE_1 2 "s_register_operand" "w")
4251                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4252          VPSELQ))
4253   ]
4254   "TARGET_HAVE_MVE"
4255   "vpsel %q0, %q1, %q2"
4256   [(set_attr "type" "mve_move")
4257 ])
4258
4259 ;;
4260 ;; [vqabsq_m_s])
4261 ;;
4262 (define_insn "mve_vqabsq_m_s<mode>"
4263   [
4264    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4265         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4266                        (match_operand:MVE_2 2 "s_register_operand" "w")
4267                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4268          VQABSQ_M_S))
4269   ]
4270   "TARGET_HAVE_MVE"
4271   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
4272   [(set_attr "type" "mve_move")
4273    (set_attr "length""8")])
4274
4275 ;;
4276 ;; [vqdmlahq_n_s, vqdmlahq_n_u])
4277 ;;
4278 (define_insn "mve_vqdmlahq_n_<supf><mode>"
4279   [
4280    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4281         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4282                        (match_operand:MVE_2 2 "s_register_operand" "w")
4283                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4284          VQDMLAHQ_N))
4285   ]
4286   "TARGET_HAVE_MVE"
4287   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4288   [(set_attr "type" "mve_move")
4289 ])
4290
4291 ;;
4292 ;; [vqnegq_m_s])
4293 ;;
4294 (define_insn "mve_vqnegq_m_s<mode>"
4295   [
4296    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4297         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4298                        (match_operand:MVE_2 2 "s_register_operand" "w")
4299                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4300          VQNEGQ_M_S))
4301   ]
4302   "TARGET_HAVE_MVE"
4303   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
4304   [(set_attr "type" "mve_move")
4305    (set_attr "length""8")])
4306
4307 ;;
4308 ;; [vqrdmladhq_s])
4309 ;;
4310 (define_insn "mve_vqrdmladhq_s<mode>"
4311   [
4312    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4313         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4314                        (match_operand:MVE_2 2 "s_register_operand" "w")
4315                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4316          VQRDMLADHQ_S))
4317   ]
4318   "TARGET_HAVE_MVE"
4319   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4320   [(set_attr "type" "mve_move")
4321 ])
4322
4323 ;;
4324 ;; [vqrdmladhxq_s])
4325 ;;
4326 (define_insn "mve_vqrdmladhxq_s<mode>"
4327   [
4328    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4329         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4330                        (match_operand:MVE_2 2 "s_register_operand" "w")
4331                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4332          VQRDMLADHXQ_S))
4333   ]
4334   "TARGET_HAVE_MVE"
4335   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4336   [(set_attr "type" "mve_move")
4337 ])
4338
4339 ;;
4340 ;; [vqrdmlahq_n_s, vqrdmlahq_n_u])
4341 ;;
4342 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
4343   [
4344    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4345         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4346                        (match_operand:MVE_2 2 "s_register_operand" "w")
4347                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4348          VQRDMLAHQ_N))
4349   ]
4350   "TARGET_HAVE_MVE"
4351   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4352   [(set_attr "type" "mve_move")
4353 ])
4354
4355 ;;
4356 ;; [vqrdmlashq_n_s, vqrdmlashq_n_u])
4357 ;;
4358 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
4359   [
4360    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4361         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4362                        (match_operand:MVE_2 2 "s_register_operand" "w")
4363                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4364          VQRDMLASHQ_N))
4365   ]
4366   "TARGET_HAVE_MVE"
4367   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
4368   [(set_attr "type" "mve_move")
4369 ])
4370
4371 ;;
4372 ;; [vqrdmlsdhq_s])
4373 ;;
4374 (define_insn "mve_vqrdmlsdhq_s<mode>"
4375   [
4376    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4377         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4378                        (match_operand:MVE_2 2 "s_register_operand" "w")
4379                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4380          VQRDMLSDHQ_S))
4381   ]
4382   "TARGET_HAVE_MVE"
4383   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4384   [(set_attr "type" "mve_move")
4385 ])
4386
4387 ;;
4388 ;; [vqrdmlsdhxq_s])
4389 ;;
4390 (define_insn "mve_vqrdmlsdhxq_s<mode>"
4391   [
4392    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4393         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4394                        (match_operand:MVE_2 2 "s_register_operand" "w")
4395                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4396          VQRDMLSDHXQ_S))
4397   ]
4398   "TARGET_HAVE_MVE"
4399   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4400   [(set_attr "type" "mve_move")
4401 ])
4402
4403 ;;
4404 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
4405 ;;
4406 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
4407   [
4408    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4409         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4410                        (match_operand:SI 2 "s_register_operand" "r")
4411                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4412          VQRSHLQ_M_N))
4413   ]
4414   "TARGET_HAVE_MVE"
4415   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
4416   [(set_attr "type" "mve_move")
4417    (set_attr "length""8")])
4418
4419 ;;
4420 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
4421 ;;
4422 (define_insn "mve_vqshlq_m_r_<supf><mode>"
4423   [
4424    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4425         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4426                        (match_operand:SI 2 "s_register_operand" "r")
4427                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4428          VQSHLQ_M_R))
4429   ]
4430   "TARGET_HAVE_MVE"
4431   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4432   [(set_attr "type" "mve_move")
4433    (set_attr "length""8")])
4434
4435 ;;
4436 ;; [vrev64q_m_u, vrev64q_m_s])
4437 ;;
4438 (define_insn "mve_vrev64q_m_<supf><mode>"
4439   [
4440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4441         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4442                        (match_operand:MVE_2 2 "s_register_operand" "w")
4443                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4444          VREV64Q_M))
4445   ]
4446   "TARGET_HAVE_MVE"
4447   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
4448   [(set_attr "type" "mve_move")
4449    (set_attr "length""8")])
4450
4451 ;;
4452 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
4453 ;;
4454 (define_insn "mve_vrshlq_m_n_<supf><mode>"
4455   [
4456    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4457         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4458                        (match_operand:SI 2 "s_register_operand" "r")
4459                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4460          VRSHLQ_M_N))
4461   ]
4462   "TARGET_HAVE_MVE"
4463   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4464   [(set_attr "type" "mve_move")
4465    (set_attr "length""8")])
4466
4467 ;;
4468 ;; [vshlq_m_r_u, vshlq_m_r_s])
4469 ;;
4470 (define_insn "mve_vshlq_m_r_<supf><mode>"
4471   [
4472    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4473         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4474                        (match_operand:SI 2 "s_register_operand" "r")
4475                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4476          VSHLQ_M_R))
4477   ]
4478   "TARGET_HAVE_MVE"
4479   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4480   [(set_attr "type" "mve_move")
4481    (set_attr "length""8")])
4482
4483 ;;
4484 ;; [vsliq_n_u, vsliq_n_s])
4485 ;;
4486 (define_insn "mve_vsliq_n_<supf><mode>"
4487   [
4488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4489         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4490                        (match_operand:MVE_2 2 "s_register_operand" "w")
4491                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
4492          VSLIQ_N))
4493   ]
4494   "TARGET_HAVE_MVE"
4495   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
4496   [(set_attr "type" "mve_move")
4497 ])
4498
4499 ;;
4500 ;; [vsriq_n_u, vsriq_n_s])
4501 ;;
4502 (define_insn "mve_vsriq_n_<supf><mode>"
4503   [
4504    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4505         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4506                        (match_operand:MVE_2 2 "s_register_operand" "w")
4507                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
4508          VSRIQ_N))
4509   ]
4510   "TARGET_HAVE_MVE"
4511   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
4512   [(set_attr "type" "mve_move")
4513 ])
4514
4515 ;;
4516 ;; [vqdmlsdhxq_s])
4517 ;;
4518 (define_insn "mve_vqdmlsdhxq_s<mode>"
4519   [
4520    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4521         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4522                        (match_operand:MVE_2 2 "s_register_operand" "w")
4523                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4524          VQDMLSDHXQ_S))
4525   ]
4526   "TARGET_HAVE_MVE"
4527   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4528   [(set_attr "type" "mve_move")
4529 ])
4530
4531 ;;
4532 ;; [vqdmlsdhq_s])
4533 ;;
4534 (define_insn "mve_vqdmlsdhq_s<mode>"
4535   [
4536    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4537         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4538                        (match_operand:MVE_2 2 "s_register_operand" "w")
4539                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4540          VQDMLSDHQ_S))
4541   ]
4542   "TARGET_HAVE_MVE"
4543   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4544   [(set_attr "type" "mve_move")
4545 ])
4546
4547 ;;
4548 ;; [vqdmladhxq_s])
4549 ;;
4550 (define_insn "mve_vqdmladhxq_s<mode>"
4551   [
4552    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4553         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4554                        (match_operand:MVE_2 2 "s_register_operand" "w")
4555                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4556          VQDMLADHXQ_S))
4557   ]
4558   "TARGET_HAVE_MVE"
4559   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4560   [(set_attr "type" "mve_move")
4561 ])
4562
4563 ;;
4564 ;; [vqdmladhq_s])
4565 ;;
4566 (define_insn "mve_vqdmladhq_s<mode>"
4567   [
4568    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4569         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4570                        (match_operand:MVE_2 2 "s_register_operand" "w")
4571                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4572          VQDMLADHQ_S))
4573   ]
4574   "TARGET_HAVE_MVE"
4575   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4576   [(set_attr "type" "mve_move")
4577 ])
4578
4579 ;;
4580 ;; [vmlsdavaxq_s])
4581 ;;
4582 (define_insn "mve_vmlsdavaxq_s<mode>"
4583   [
4584    (set (match_operand:SI 0 "s_register_operand" "=e")
4585         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4586                     (match_operand:MVE_2 2 "s_register_operand" "w")
4587                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4588          VMLSDAVAXQ_S))
4589   ]
4590   "TARGET_HAVE_MVE"
4591   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4592   [(set_attr "type" "mve_move")
4593 ])
4594
4595 ;;
4596 ;; [vmlsdavaq_s])
4597 ;;
4598 (define_insn "mve_vmlsdavaq_s<mode>"
4599   [
4600    (set (match_operand:SI 0 "s_register_operand" "=e")
4601         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4602                     (match_operand:MVE_2 2 "s_register_operand" "w")
4603                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4604          VMLSDAVAQ_S))
4605   ]
4606   "TARGET_HAVE_MVE"
4607   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4608   [(set_attr "type" "mve_move")
4609 ])
4610
4611 ;;
4612 ;; [vmladavaxq_s])
4613 ;;
4614 (define_insn "mve_vmladavaxq_s<mode>"
4615   [
4616    (set (match_operand:SI 0 "s_register_operand" "=e")
4617         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4618                     (match_operand:MVE_2 2 "s_register_operand" "w")
4619                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4620          VMLADAVAXQ_S))
4621   ]
4622   "TARGET_HAVE_MVE"
4623   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4624   [(set_attr "type" "mve_move")
4625 ])
4626 ;;
4627 ;; [vabsq_m_f])
4628 ;;
4629 (define_insn "mve_vabsq_m_f<mode>"
4630   [
4631    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4632         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4633                        (match_operand:MVE_0 2 "s_register_operand" "w")
4634                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4635          VABSQ_M_F))
4636   ]
4637   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4638   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4639   [(set_attr "type" "mve_move")
4640    (set_attr "length""8")])
4641
4642 ;;
4643 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4644 ;;
4645 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4646   [
4647    (set (match_operand:DI 0 "s_register_operand" "=r")
4648         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4649                        (match_operand:V4SI 2 "s_register_operand" "w")
4650                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4651          VADDLVAQ_P))
4652   ]
4653   "TARGET_HAVE_MVE"
4654   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4655   [(set_attr "type" "mve_move")
4656    (set_attr "length""8")])
4657 ;;
4658 ;; [vcmlaq_f])
4659 ;;
4660 (define_insn "mve_vcmlaq_f<mode>"
4661   [
4662    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4663         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4664                        (match_operand:MVE_0 2 "s_register_operand" "w")
4665                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4666          VCMLAQ_F))
4667   ]
4668   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4669   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4670   [(set_attr "type" "mve_move")
4671 ])
4672
4673 ;;
4674 ;; [vcmlaq_rot180_f])
4675 ;;
4676 (define_insn "mve_vcmlaq_rot180_f<mode>"
4677   [
4678    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4679         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4680                        (match_operand:MVE_0 2 "s_register_operand" "w")
4681                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4682          VCMLAQ_ROT180_F))
4683   ]
4684   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4685   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4686   [(set_attr "type" "mve_move")
4687 ])
4688
4689 ;;
4690 ;; [vcmlaq_rot270_f])
4691 ;;
4692 (define_insn "mve_vcmlaq_rot270_f<mode>"
4693   [
4694    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4695         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4696                        (match_operand:MVE_0 2 "s_register_operand" "w")
4697                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4698          VCMLAQ_ROT270_F))
4699   ]
4700   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4701   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4702   [(set_attr "type" "mve_move")
4703 ])
4704
4705 ;;
4706 ;; [vcmlaq_rot90_f])
4707 ;;
4708 (define_insn "mve_vcmlaq_rot90_f<mode>"
4709   [
4710    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4711         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4712                        (match_operand:MVE_0 2 "s_register_operand" "w")
4713                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4714          VCMLAQ_ROT90_F))
4715   ]
4716   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4717   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4718   [(set_attr "type" "mve_move")
4719 ])
4720
4721 ;;
4722 ;; [vcmpeqq_m_n_f])
4723 ;;
4724 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4725   [
4726    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4727         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4728                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4729                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4730          VCMPEQQ_M_N_F))
4731   ]
4732   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4733   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4734   [(set_attr "type" "mve_move")
4735    (set_attr "length""8")])
4736
4737 ;;
4738 ;; [vcmpgeq_m_f])
4739 ;;
4740 (define_insn "mve_vcmpgeq_m_f<mode>"
4741   [
4742    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4743         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4744                        (match_operand:MVE_0 2 "s_register_operand" "w")
4745                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4746          VCMPGEQ_M_F))
4747   ]
4748   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4749   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4750   [(set_attr "type" "mve_move")
4751    (set_attr "length""8")])
4752
4753 ;;
4754 ;; [vcmpgeq_m_n_f])
4755 ;;
4756 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4757   [
4758    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4759         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4760                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4761                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4762          VCMPGEQ_M_N_F))
4763   ]
4764   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4765   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4766   [(set_attr "type" "mve_move")
4767    (set_attr "length""8")])
4768
4769 ;;
4770 ;; [vcmpgtq_m_f])
4771 ;;
4772 (define_insn "mve_vcmpgtq_m_f<mode>"
4773   [
4774    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4775         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4776                        (match_operand:MVE_0 2 "s_register_operand" "w")
4777                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4778          VCMPGTQ_M_F))
4779   ]
4780   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4781   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4782   [(set_attr "type" "mve_move")
4783    (set_attr "length""8")])
4784
4785 ;;
4786 ;; [vcmpgtq_m_n_f])
4787 ;;
4788 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4789   [
4790    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4791         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4792                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4793                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4794          VCMPGTQ_M_N_F))
4795   ]
4796   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4797   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4798   [(set_attr "type" "mve_move")
4799    (set_attr "length""8")])
4800
4801 ;;
4802 ;; [vcmpleq_m_f])
4803 ;;
4804 (define_insn "mve_vcmpleq_m_f<mode>"
4805   [
4806    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4807         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4808                        (match_operand:MVE_0 2 "s_register_operand" "w")
4809                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4810          VCMPLEQ_M_F))
4811   ]
4812   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4813   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4814   [(set_attr "type" "mve_move")
4815    (set_attr "length""8")])
4816
4817 ;;
4818 ;; [vcmpleq_m_n_f])
4819 ;;
4820 (define_insn "mve_vcmpleq_m_n_f<mode>"
4821   [
4822    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4823         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4824                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4825                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4826          VCMPLEQ_M_N_F))
4827   ]
4828   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4829   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4830   [(set_attr "type" "mve_move")
4831    (set_attr "length""8")])
4832
4833 ;;
4834 ;; [vcmpltq_m_f])
4835 ;;
4836 (define_insn "mve_vcmpltq_m_f<mode>"
4837   [
4838    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4839         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4840                        (match_operand:MVE_0 2 "s_register_operand" "w")
4841                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4842          VCMPLTQ_M_F))
4843   ]
4844   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4845   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4846   [(set_attr "type" "mve_move")
4847    (set_attr "length""8")])
4848
4849 ;;
4850 ;; [vcmpltq_m_n_f])
4851 ;;
4852 (define_insn "mve_vcmpltq_m_n_f<mode>"
4853   [
4854    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4855         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4856                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4857                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4858          VCMPLTQ_M_N_F))
4859   ]
4860   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4861   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4862   [(set_attr "type" "mve_move")
4863    (set_attr "length""8")])
4864
4865 ;;
4866 ;; [vcmpneq_m_f])
4867 ;;
4868 (define_insn "mve_vcmpneq_m_f<mode>"
4869   [
4870    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4871         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4872                        (match_operand:MVE_0 2 "s_register_operand" "w")
4873                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4874          VCMPNEQ_M_F))
4875   ]
4876   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4877   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4878   [(set_attr "type" "mve_move")
4879    (set_attr "length""8")])
4880
4881 ;;
4882 ;; [vcmpneq_m_n_f])
4883 ;;
4884 (define_insn "mve_vcmpneq_m_n_f<mode>"
4885   [
4886    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4887         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4888                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4889                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4890          VCMPNEQ_M_N_F))
4891   ]
4892   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4893   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4894   [(set_attr "type" "mve_move")
4895    (set_attr "length""8")])
4896
4897 ;;
4898 ;; [vcvtbq_m_f16_f32])
4899 ;;
4900 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4901   [
4902    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4903         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4904                        (match_operand:V4SF 2 "s_register_operand" "w")
4905                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4906          VCVTBQ_M_F16_F32))
4907   ]
4908   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4909   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4910   [(set_attr "type" "mve_move")
4911    (set_attr "length""8")])
4912
4913 ;;
4914 ;; [vcvtbq_m_f32_f16])
4915 ;;
4916 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4917   [
4918    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4919         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4920                        (match_operand:V8HF 2 "s_register_operand" "w")
4921                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4922          VCVTBQ_M_F32_F16))
4923   ]
4924   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4925   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4926   [(set_attr "type" "mve_move")
4927    (set_attr "length""8")])
4928
4929 ;;
4930 ;; [vcvttq_m_f16_f32])
4931 ;;
4932 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4933   [
4934    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4935         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4936                        (match_operand:V4SF 2 "s_register_operand" "w")
4937                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4938          VCVTTQ_M_F16_F32))
4939   ]
4940   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4941   "vpst\;vcvttt.f16.f32 %q0, %q2"
4942   [(set_attr "type" "mve_move")
4943    (set_attr "length""8")])
4944
4945 ;;
4946 ;; [vcvttq_m_f32_f16])
4947 ;;
4948 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4949   [
4950    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4951         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4952                        (match_operand:V8HF 2 "s_register_operand" "w")
4953                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4954          VCVTTQ_M_F32_F16))
4955   ]
4956   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4957   "vpst\;vcvttt.f32.f16 %q0, %q2"
4958   [(set_attr "type" "mve_move")
4959    (set_attr "length""8")])
4960
4961 ;;
4962 ;; [vdupq_m_n_f])
4963 ;;
4964 (define_insn "mve_vdupq_m_n_f<mode>"
4965   [
4966    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4967         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4968                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4969                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4970          VDUPQ_M_N_F))
4971   ]
4972   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4973   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4974   [(set_attr "type" "mve_move")
4975    (set_attr "length""8")])
4976
4977 ;;
4978 ;; [vfmaq_f])
4979 ;;
4980 (define_insn "mve_vfmaq_f<mode>"
4981   [
4982    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4983         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4984                        (match_operand:MVE_0 2 "s_register_operand" "w")
4985                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4986          VFMAQ_F))
4987   ]
4988   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4989   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4990   [(set_attr "type" "mve_move")
4991 ])
4992
4993 ;;
4994 ;; [vfmaq_n_f])
4995 ;;
4996 (define_insn "mve_vfmaq_n_f<mode>"
4997   [
4998    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4999         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5000                        (match_operand:MVE_0 2 "s_register_operand" "w")
5001                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5002          VFMAQ_N_F))
5003   ]
5004   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5005   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
5006   [(set_attr "type" "mve_move")
5007 ])
5008
5009 ;;
5010 ;; [vfmasq_n_f])
5011 ;;
5012 (define_insn "mve_vfmasq_n_f<mode>"
5013   [
5014    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5015         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5016                        (match_operand:MVE_0 2 "s_register_operand" "w")
5017                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5018          VFMASQ_N_F))
5019   ]
5020   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5021   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
5022   [(set_attr "type" "mve_move")
5023 ])
5024 ;;
5025 ;; [vfmsq_f])
5026 ;;
5027 (define_insn "mve_vfmsq_f<mode>"
5028   [
5029    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5030         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5031                        (match_operand:MVE_0 2 "s_register_operand" "w")
5032                        (match_operand:MVE_0 3 "s_register_operand" "w")]
5033          VFMSQ_F))
5034   ]
5035   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5036   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
5037   [(set_attr "type" "mve_move")
5038 ])
5039
5040 ;;
5041 ;; [vmaxnmaq_m_f])
5042 ;;
5043 (define_insn "mve_vmaxnmaq_m_f<mode>"
5044   [
5045    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5046         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5047                        (match_operand:MVE_0 2 "s_register_operand" "w")
5048                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5049          VMAXNMAQ_M_F))
5050   ]
5051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5052   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
5053   [(set_attr "type" "mve_move")
5054    (set_attr "length""8")])
5055 ;;
5056 ;; [vmaxnmavq_p_f])
5057 ;;
5058 (define_insn "mve_vmaxnmavq_p_f<mode>"
5059   [
5060    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5061         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5062                        (match_operand:MVE_0 2 "s_register_operand" "w")
5063                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5064          VMAXNMAVQ_P_F))
5065   ]
5066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5067   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
5068   [(set_attr "type" "mve_move")
5069    (set_attr "length""8")])
5070
5071 ;;
5072 ;; [vmaxnmvq_p_f])
5073 ;;
5074 (define_insn "mve_vmaxnmvq_p_f<mode>"
5075   [
5076    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5077         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5078                        (match_operand:MVE_0 2 "s_register_operand" "w")
5079                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5080          VMAXNMVQ_P_F))
5081   ]
5082   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5083   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
5084   [(set_attr "type" "mve_move")
5085    (set_attr "length""8")])
5086 ;;
5087 ;; [vminnmaq_m_f])
5088 ;;
5089 (define_insn "mve_vminnmaq_m_f<mode>"
5090   [
5091    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5092         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5093                        (match_operand:MVE_0 2 "s_register_operand" "w")
5094                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5095          VMINNMAQ_M_F))
5096   ]
5097   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5098   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
5099   [(set_attr "type" "mve_move")
5100    (set_attr "length""8")])
5101
5102 ;;
5103 ;; [vminnmavq_p_f])
5104 ;;
5105 (define_insn "mve_vminnmavq_p_f<mode>"
5106   [
5107    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5108         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5109                        (match_operand:MVE_0 2 "s_register_operand" "w")
5110                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5111          VMINNMAVQ_P_F))
5112   ]
5113   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5114   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
5115   [(set_attr "type" "mve_move")
5116    (set_attr "length""8")])
5117 ;;
5118 ;; [vminnmvq_p_f])
5119 ;;
5120 (define_insn "mve_vminnmvq_p_f<mode>"
5121   [
5122    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5123         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5124                        (match_operand:MVE_0 2 "s_register_operand" "w")
5125                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5126          VMINNMVQ_P_F))
5127   ]
5128   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5129   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
5130   [(set_attr "type" "mve_move")
5131    (set_attr "length""8")])
5132
5133 ;;
5134 ;; [vmlaldavaq_s, vmlaldavaq_u])
5135 ;;
5136 (define_insn "mve_vmlaldavaq_<supf><mode>"
5137   [
5138    (set (match_operand:DI 0 "s_register_operand" "=r")
5139         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5140                        (match_operand:MVE_5 2 "s_register_operand" "w")
5141                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5142          VMLALDAVAQ))
5143   ]
5144   "TARGET_HAVE_MVE"
5145   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5146   [(set_attr "type" "mve_move")
5147 ])
5148
5149 ;;
5150 ;; [vmlaldavaxq_s])
5151 ;;
5152 (define_insn "mve_vmlaldavaxq_s<mode>"
5153   [
5154    (set (match_operand:DI 0 "s_register_operand" "=r")
5155         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5156                        (match_operand:MVE_5 2 "s_register_operand" "w")
5157                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5158          VMLALDAVAXQ_S))
5159   ]
5160   "TARGET_HAVE_MVE"
5161   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5162   [(set_attr "type" "mve_move")
5163 ])
5164
5165 ;;
5166 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
5167 ;;
5168 (define_insn "mve_vmlaldavq_p_<supf><mode>"
5169   [
5170    (set (match_operand:DI 0 "s_register_operand" "=r")
5171         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5172                        (match_operand:MVE_5 2 "s_register_operand" "w")
5173                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5174          VMLALDAVQ_P))
5175   ]
5176   "TARGET_HAVE_MVE"
5177   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5178   [(set_attr "type" "mve_move")
5179    (set_attr "length""8")])
5180
5181 ;;
5182 ;; [vmlaldavxq_p_s])
5183 ;;
5184 (define_insn "mve_vmlaldavxq_p_s<mode>"
5185   [
5186    (set (match_operand:DI 0 "s_register_operand" "=r")
5187         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5188                        (match_operand:MVE_5 2 "s_register_operand" "w")
5189                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5190          VMLALDAVXQ_P_S))
5191   ]
5192   "TARGET_HAVE_MVE"
5193   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
5194   [(set_attr "type" "mve_move")
5195    (set_attr "length""8")])
5196 ;;
5197 ;; [vmlsldavaq_s])
5198 ;;
5199 (define_insn "mve_vmlsldavaq_s<mode>"
5200   [
5201    (set (match_operand:DI 0 "s_register_operand" "=r")
5202         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5203                        (match_operand:MVE_5 2 "s_register_operand" "w")
5204                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5205          VMLSLDAVAQ_S))
5206   ]
5207   "TARGET_HAVE_MVE"
5208   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5209   [(set_attr "type" "mve_move")
5210 ])
5211
5212 ;;
5213 ;; [vmlsldavaxq_s])
5214 ;;
5215 (define_insn "mve_vmlsldavaxq_s<mode>"
5216   [
5217    (set (match_operand:DI 0 "s_register_operand" "=r")
5218         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5219                        (match_operand:MVE_5 2 "s_register_operand" "w")
5220                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5221          VMLSLDAVAXQ_S))
5222   ]
5223   "TARGET_HAVE_MVE"
5224   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5225   [(set_attr "type" "mve_move")
5226 ])
5227
5228 ;;
5229 ;; [vmlsldavq_p_s])
5230 ;;
5231 (define_insn "mve_vmlsldavq_p_s<mode>"
5232   [
5233    (set (match_operand:DI 0 "s_register_operand" "=r")
5234         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5235                        (match_operand:MVE_5 2 "s_register_operand" "w")
5236                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5237          VMLSLDAVQ_P_S))
5238   ]
5239   "TARGET_HAVE_MVE"
5240   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5241   [(set_attr "type" "mve_move")
5242    (set_attr "length""8")])
5243
5244 ;;
5245 ;; [vmlsldavxq_p_s])
5246 ;;
5247 (define_insn "mve_vmlsldavxq_p_s<mode>"
5248   [
5249    (set (match_operand:DI 0 "s_register_operand" "=r")
5250         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5251                        (match_operand:MVE_5 2 "s_register_operand" "w")
5252                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5253          VMLSLDAVXQ_P_S))
5254   ]
5255   "TARGET_HAVE_MVE"
5256   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5257   [(set_attr "type" "mve_move")
5258    (set_attr "length""8")])
5259 ;;
5260 ;; [vmovlbq_m_u, vmovlbq_m_s])
5261 ;;
5262 (define_insn "mve_vmovlbq_m_<supf><mode>"
5263   [
5264    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5265         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5266                        (match_operand:MVE_3 2 "s_register_operand" "w")
5267                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5268          VMOVLBQ_M))
5269   ]
5270   "TARGET_HAVE_MVE"
5271   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
5272   [(set_attr "type" "mve_move")
5273    (set_attr "length""8")])
5274 ;;
5275 ;; [vmovltq_m_u, vmovltq_m_s])
5276 ;;
5277 (define_insn "mve_vmovltq_m_<supf><mode>"
5278   [
5279    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5280         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5281                        (match_operand:MVE_3 2 "s_register_operand" "w")
5282                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5283          VMOVLTQ_M))
5284   ]
5285   "TARGET_HAVE_MVE"
5286   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
5287   [(set_attr "type" "mve_move")
5288    (set_attr "length""8")])
5289 ;;
5290 ;; [vmovnbq_m_u, vmovnbq_m_s])
5291 ;;
5292 (define_insn "mve_vmovnbq_m_<supf><mode>"
5293   [
5294    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5295         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5296                        (match_operand:MVE_5 2 "s_register_operand" "w")
5297                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5298          VMOVNBQ_M))
5299   ]
5300   "TARGET_HAVE_MVE"
5301   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
5302   [(set_attr "type" "mve_move")
5303    (set_attr "length""8")])
5304
5305 ;;
5306 ;; [vmovntq_m_u, vmovntq_m_s])
5307 ;;
5308 (define_insn "mve_vmovntq_m_<supf><mode>"
5309   [
5310    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5311         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5312                        (match_operand:MVE_5 2 "s_register_operand" "w")
5313                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5314          VMOVNTQ_M))
5315   ]
5316   "TARGET_HAVE_MVE"
5317   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
5318   [(set_attr "type" "mve_move")
5319    (set_attr "length""8")])
5320
5321 ;;
5322 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
5323 ;;
5324 (define_insn "mve_vmvnq_m_n_<supf><mode>"
5325   [
5326    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5327         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5328                        (match_operand:SI 2 "immediate_operand" "i")
5329                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5330          VMVNQ_M_N))
5331   ]
5332   "TARGET_HAVE_MVE"
5333   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
5334   [(set_attr "type" "mve_move")
5335    (set_attr "length""8")])
5336 ;;
5337 ;; [vnegq_m_f])
5338 ;;
5339 (define_insn "mve_vnegq_m_f<mode>"
5340   [
5341    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5342         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5343                        (match_operand:MVE_0 2 "s_register_operand" "w")
5344                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5345          VNEGQ_M_F))
5346   ]
5347   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5348   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
5349   [(set_attr "type" "mve_move")
5350    (set_attr "length""8")])
5351
5352 ;;
5353 ;; [vorrq_m_n_s, vorrq_m_n_u])
5354 ;;
5355 (define_insn "mve_vorrq_m_n_<supf><mode>"
5356   [
5357    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5358         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5359                        (match_operand:SI 2 "immediate_operand" "i")
5360                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5361          VORRQ_M_N))
5362   ]
5363   "TARGET_HAVE_MVE"
5364   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
5365   [(set_attr "type" "mve_move")
5366    (set_attr "length""8")])
5367 ;;
5368 ;; [vpselq_f])
5369 ;;
5370 (define_insn "mve_vpselq_f<mode>"
5371   [
5372    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5373         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
5374                        (match_operand:MVE_0 2 "s_register_operand" "w")
5375                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5376          VPSELQ_F))
5377   ]
5378   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5379   "vpsel %q0, %q1, %q2"
5380   [(set_attr "type" "mve_move")
5381 ])
5382
5383 ;;
5384 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
5385 ;;
5386 (define_insn "mve_vqmovnbq_m_<supf><mode>"
5387   [
5388    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5389         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5390                        (match_operand:MVE_5 2 "s_register_operand" "w")
5391                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5392          VQMOVNBQ_M))
5393   ]
5394   "TARGET_HAVE_MVE"
5395   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
5396   [(set_attr "type" "mve_move")
5397    (set_attr "length""8")])
5398
5399 ;;
5400 ;; [vqmovntq_m_u, vqmovntq_m_s])
5401 ;;
5402 (define_insn "mve_vqmovntq_m_<supf><mode>"
5403   [
5404    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5405         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5406                        (match_operand:MVE_5 2 "s_register_operand" "w")
5407                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5408          VQMOVNTQ_M))
5409   ]
5410   "TARGET_HAVE_MVE"
5411   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
5412   [(set_attr "type" "mve_move")
5413    (set_attr "length""8")])
5414
5415 ;;
5416 ;; [vqmovunbq_m_s])
5417 ;;
5418 (define_insn "mve_vqmovunbq_m_s<mode>"
5419   [
5420    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5421         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5422                        (match_operand:MVE_5 2 "s_register_operand" "w")
5423                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5424          VQMOVUNBQ_M_S))
5425   ]
5426   "TARGET_HAVE_MVE"
5427   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
5428   [(set_attr "type" "mve_move")
5429    (set_attr "length""8")])
5430
5431 ;;
5432 ;; [vqmovuntq_m_s])
5433 ;;
5434 (define_insn "mve_vqmovuntq_m_s<mode>"
5435   [
5436    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5437         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5438                        (match_operand:MVE_5 2 "s_register_operand" "w")
5439                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5440          VQMOVUNTQ_M_S))
5441   ]
5442   "TARGET_HAVE_MVE"
5443   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
5444   [(set_attr "type" "mve_move")
5445    (set_attr "length""8")])
5446
5447 ;;
5448 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
5449 ;;
5450 (define_insn "mve_vqrshrntq_n_<supf><mode>"
5451   [
5452    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5453         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5454                        (match_operand:MVE_5 2 "s_register_operand" "w")
5455                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5456          VQRSHRNTQ_N))
5457   ]
5458   "TARGET_HAVE_MVE"
5459   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
5460   [(set_attr "type" "mve_move")
5461 ])
5462
5463 ;;
5464 ;; [vqrshruntq_n_s])
5465 ;;
5466 (define_insn "mve_vqrshruntq_n_s<mode>"
5467   [
5468    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5469         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5470                        (match_operand:MVE_5 2 "s_register_operand" "w")
5471                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5472          VQRSHRUNTQ_N_S))
5473   ]
5474   "TARGET_HAVE_MVE"
5475   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
5476   [(set_attr "type" "mve_move")
5477 ])
5478
5479 ;;
5480 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
5481 ;;
5482 (define_insn "mve_vqshrnbq_n_<supf><mode>"
5483   [
5484    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5485         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5486                        (match_operand:MVE_5 2 "s_register_operand" "w")
5487                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5488          VQSHRNBQ_N))
5489   ]
5490   "TARGET_HAVE_MVE"
5491   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5492   [(set_attr "type" "mve_move")
5493 ])
5494
5495 ;;
5496 ;; [vqshrntq_n_u, vqshrntq_n_s])
5497 ;;
5498 (define_insn "mve_vqshrntq_n_<supf><mode>"
5499   [
5500    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5501         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5502                        (match_operand:MVE_5 2 "s_register_operand" "w")
5503                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5504          VQSHRNTQ_N))
5505   ]
5506   "TARGET_HAVE_MVE"
5507   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
5508   [(set_attr "type" "mve_move")
5509 ])
5510
5511 ;;
5512 ;; [vqshrunbq_n_s])
5513 ;;
5514 (define_insn "mve_vqshrunbq_n_s<mode>"
5515   [
5516    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5517         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5518                        (match_operand:MVE_5 2 "s_register_operand" "w")
5519                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5520          VQSHRUNBQ_N_S))
5521   ]
5522   "TARGET_HAVE_MVE"
5523   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5524   [(set_attr "type" "mve_move")
5525 ])
5526
5527 ;;
5528 ;; [vqshruntq_n_s])
5529 ;;
5530 (define_insn "mve_vqshruntq_n_s<mode>"
5531   [
5532    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5533         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5534                        (match_operand:MVE_5 2 "s_register_operand" "w")
5535                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5536          VQSHRUNTQ_N_S))
5537   ]
5538   "TARGET_HAVE_MVE"
5539   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5540   [(set_attr "type" "mve_move")
5541 ])
5542
5543 ;;
5544 ;; [vrev32q_m_f])
5545 ;;
5546 (define_insn "mve_vrev32q_m_fv8hf"
5547   [
5548    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5549         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5550                        (match_operand:V8HF 2 "s_register_operand" "w")
5551                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5552          VREV32Q_M_F))
5553   ]
5554   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5555   "vpst\;vrev32t.16 %q0, %q2"
5556   [(set_attr "type" "mve_move")
5557    (set_attr "length""8")])
5558
5559 ;;
5560 ;; [vrev32q_m_s, vrev32q_m_u])
5561 ;;
5562 (define_insn "mve_vrev32q_m_<supf><mode>"
5563   [
5564    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5565         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5566                        (match_operand:MVE_3 2 "s_register_operand" "w")
5567                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5568          VREV32Q_M))
5569   ]
5570   "TARGET_HAVE_MVE"
5571   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5572   [(set_attr "type" "mve_move")
5573    (set_attr "length""8")])
5574
5575 ;;
5576 ;; [vrev64q_m_f])
5577 ;;
5578 (define_insn "mve_vrev64q_m_f<mode>"
5579   [
5580    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5581         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5582                        (match_operand:MVE_0 2 "s_register_operand" "w")
5583                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5584          VREV64Q_M_F))
5585   ]
5586   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5587   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5588   [(set_attr "type" "mve_move")
5589    (set_attr "length""8")])
5590
5591 ;;
5592 ;; [vrmlaldavhaxq_s])
5593 ;;
5594 (define_insn "mve_vrmlaldavhaxq_sv4si"
5595   [
5596    (set (match_operand:DI 0 "s_register_operand" "=r")
5597         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5598                        (match_operand:V4SI 2 "s_register_operand" "w")
5599                        (match_operand:V4SI 3 "s_register_operand" "w")]
5600          VRMLALDAVHAXQ_S))
5601   ]
5602   "TARGET_HAVE_MVE"
5603   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5604   [(set_attr "type" "mve_move")
5605 ])
5606
5607 ;;
5608 ;; [vrmlaldavhxq_p_s])
5609 ;;
5610 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5611   [
5612    (set (match_operand:DI 0 "s_register_operand" "=r")
5613         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5614                        (match_operand:V4SI 2 "s_register_operand" "w")
5615                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5616          VRMLALDAVHXQ_P_S))
5617   ]
5618   "TARGET_HAVE_MVE"
5619   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5620   [(set_attr "type" "mve_move")
5621    (set_attr "length""8")])
5622
5623 ;;
5624 ;; [vrmlsldavhaxq_s])
5625 ;;
5626 (define_insn "mve_vrmlsldavhaxq_sv4si"
5627   [
5628    (set (match_operand:DI 0 "s_register_operand" "=r")
5629         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5630                        (match_operand:V4SI 2 "s_register_operand" "w")
5631                        (match_operand:V4SI 3 "s_register_operand" "w")]
5632          VRMLSLDAVHAXQ_S))
5633   ]
5634   "TARGET_HAVE_MVE"
5635   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5636   [(set_attr "type" "mve_move")
5637 ])
5638
5639 ;;
5640 ;; [vrmlsldavhq_p_s])
5641 ;;
5642 (define_insn "mve_vrmlsldavhq_p_sv4si"
5643   [
5644    (set (match_operand:DI 0 "s_register_operand" "=r")
5645         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5646                        (match_operand:V4SI 2 "s_register_operand" "w")
5647                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5648          VRMLSLDAVHQ_P_S))
5649   ]
5650   "TARGET_HAVE_MVE"
5651   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5652   [(set_attr "type" "mve_move")
5653    (set_attr "length""8")])
5654
5655 ;;
5656 ;; [vrmlsldavhxq_p_s])
5657 ;;
5658 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5659   [
5660    (set (match_operand:DI 0 "s_register_operand" "=r")
5661         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5662                        (match_operand:V4SI 2 "s_register_operand" "w")
5663                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5664          VRMLSLDAVHXQ_P_S))
5665   ]
5666   "TARGET_HAVE_MVE"
5667   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5668   [(set_attr "type" "mve_move")
5669    (set_attr "length""8")])
5670
5671 ;;
5672 ;; [vrndaq_m_f])
5673 ;;
5674 (define_insn "mve_vrndaq_m_f<mode>"
5675   [
5676    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5677         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5678                        (match_operand:MVE_0 2 "s_register_operand" "w")
5679                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5680          VRNDAQ_M_F))
5681   ]
5682   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5683   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5684   [(set_attr "type" "mve_move")
5685    (set_attr "length""8")])
5686
5687 ;;
5688 ;; [vrndmq_m_f])
5689 ;;
5690 (define_insn "mve_vrndmq_m_f<mode>"
5691   [
5692    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5693         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5694                        (match_operand:MVE_0 2 "s_register_operand" "w")
5695                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5696          VRNDMQ_M_F))
5697   ]
5698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5699   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5700   [(set_attr "type" "mve_move")
5701    (set_attr "length""8")])
5702
5703 ;;
5704 ;; [vrndnq_m_f])
5705 ;;
5706 (define_insn "mve_vrndnq_m_f<mode>"
5707   [
5708    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5709         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5710                        (match_operand:MVE_0 2 "s_register_operand" "w")
5711                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5712          VRNDNQ_M_F))
5713   ]
5714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5715   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5716   [(set_attr "type" "mve_move")
5717    (set_attr "length""8")])
5718
5719 ;;
5720 ;; [vrndpq_m_f])
5721 ;;
5722 (define_insn "mve_vrndpq_m_f<mode>"
5723   [
5724    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5725         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5726                        (match_operand:MVE_0 2 "s_register_operand" "w")
5727                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5728          VRNDPQ_M_F))
5729   ]
5730   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5731   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5732   [(set_attr "type" "mve_move")
5733    (set_attr "length""8")])
5734
5735 ;;
5736 ;; [vrndxq_m_f])
5737 ;;
5738 (define_insn "mve_vrndxq_m_f<mode>"
5739   [
5740    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5741         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5742                        (match_operand:MVE_0 2 "s_register_operand" "w")
5743                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5744          VRNDXQ_M_F))
5745   ]
5746   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5747   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5748   [(set_attr "type" "mve_move")
5749    (set_attr "length""8")])
5750
5751 ;;
5752 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5753 ;;
5754 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5755   [
5756    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5757         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5758                        (match_operand:MVE_5 2 "s_register_operand" "w")
5759                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5760          VRSHRNBQ_N))
5761   ]
5762   "TARGET_HAVE_MVE"
5763   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5764   [(set_attr "type" "mve_move")
5765 ])
5766
5767 ;;
5768 ;; [vrshrntq_n_u, vrshrntq_n_s])
5769 ;;
5770 (define_insn "mve_vrshrntq_n_<supf><mode>"
5771   [
5772    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5773         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5774                        (match_operand:MVE_5 2 "s_register_operand" "w")
5775                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5776          VRSHRNTQ_N))
5777   ]
5778   "TARGET_HAVE_MVE"
5779   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5780   [(set_attr "type" "mve_move")
5781 ])
5782
5783 ;;
5784 ;; [vshrnbq_n_u, vshrnbq_n_s])
5785 ;;
5786 (define_insn "mve_vshrnbq_n_<supf><mode>"
5787   [
5788    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5789         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5790                        (match_operand:MVE_5 2 "s_register_operand" "w")
5791                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5792          VSHRNBQ_N))
5793   ]
5794   "TARGET_HAVE_MVE"
5795   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5796   [(set_attr "type" "mve_move")
5797 ])
5798
5799 ;;
5800 ;; [vshrntq_n_s, vshrntq_n_u])
5801 ;;
5802 (define_insn "mve_vshrntq_n_<supf><mode>"
5803   [
5804    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5805         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5806                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5807                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5808          VSHRNTQ_N))
5809   ]
5810   "TARGET_HAVE_MVE"
5811   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5812   [(set_attr "type" "mve_move")
5813 ])
5814
5815 ;;
5816 ;; [vcvtmq_m_s, vcvtmq_m_u])
5817 ;;
5818 (define_insn "mve_vcvtmq_m_<supf><mode>"
5819   [
5820    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5821         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5822                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5823                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5824          VCVTMQ_M))
5825   ]
5826   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5827   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5828   [(set_attr "type" "mve_move")
5829    (set_attr "length""8")])
5830
5831 ;;
5832 ;; [vcvtpq_m_u, vcvtpq_m_s])
5833 ;;
5834 (define_insn "mve_vcvtpq_m_<supf><mode>"
5835   [
5836    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5837         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5838                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5839                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5840          VCVTPQ_M))
5841   ]
5842   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5843   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5844   [(set_attr "type" "mve_move")
5845    (set_attr "length""8")])
5846
5847 ;;
5848 ;; [vcvtnq_m_s, vcvtnq_m_u])
5849 ;;
5850 (define_insn "mve_vcvtnq_m_<supf><mode>"
5851   [
5852    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5853         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5854                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5855                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5856          VCVTNQ_M))
5857   ]
5858   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5859   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5860   [(set_attr "type" "mve_move")
5861    (set_attr "length""8")])
5862
5863 ;;
5864 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5865 ;;
5866 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5867   [
5868    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5869         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5870                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5871                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5872                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5873          VCVTQ_M_N_FROM_F))
5874   ]
5875   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5876   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5877   [(set_attr "type" "mve_move")
5878    (set_attr "length""8")])
5879
5880 ;;
5881 ;; [vrev16q_m_u, vrev16q_m_s])
5882 ;;
5883 (define_insn "mve_vrev16q_m_<supf>v16qi"
5884   [
5885    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5886         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5887                        (match_operand:V16QI 2 "s_register_operand" "w")
5888                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5889          VREV16Q_M))
5890   ]
5891   "TARGET_HAVE_MVE"
5892   "vpst\;vrev16t.8 %q0, %q2"
5893   [(set_attr "type" "mve_move")
5894    (set_attr "length""8")])
5895
5896 ;;
5897 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5898 ;;
5899 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5900   [
5901    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5902         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5903                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5904                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5905          VCVTQ_M_FROM_F))
5906   ]
5907   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5908   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5909   [(set_attr "type" "mve_move")
5910    (set_attr "length""8")])
5911
5912 ;;
5913 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5914 ;;
5915 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5916   [
5917    (set (match_operand:DI 0 "s_register_operand" "=r")
5918         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5919                     (match_operand:V4SI 2 "s_register_operand" "w")
5920                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5921          VRMLALDAVHQ_P))
5922   ]
5923   "TARGET_HAVE_MVE"
5924   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5925   [(set_attr "type" "mve_move")
5926    (set_attr "length""8")])
5927
5928 ;;
5929 ;; [vrmlsldavhaq_s])
5930 ;;
5931 (define_insn "mve_vrmlsldavhaq_sv4si"
5932   [
5933    (set (match_operand:DI 0 "s_register_operand" "=r")
5934         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5935                     (match_operand:V4SI 2 "s_register_operand" "w")
5936                     (match_operand:V4SI 3 "s_register_operand" "w")]
5937          VRMLSLDAVHAQ_S))
5938   ]
5939   "TARGET_HAVE_MVE"
5940   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5941   [(set_attr "type" "mve_move")
5942 ])
5943
5944 ;;
5945 ;; [vabavq_p_s, vabavq_p_u])
5946 ;;
5947 (define_insn "mve_vabavq_p_<supf><mode>"
5948   [
5949    (set (match_operand:SI 0 "s_register_operand" "=r")
5950         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5951                     (match_operand:MVE_2 2 "s_register_operand" "w")
5952                     (match_operand:MVE_2 3 "s_register_operand" "w")
5953                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5954          VABAVQ_P))
5955   ]
5956   "TARGET_HAVE_MVE"
5957   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5958   [(set_attr "type" "mve_move")
5959 ])
5960
5961 ;;
5962 ;; [vqshluq_m_n_s])
5963 ;;
5964 (define_insn "mve_vqshluq_m_n_s<mode>"
5965   [
5966    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5967         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5968                        (match_operand:MVE_2 2 "s_register_operand" "w")
5969                        (match_operand:SI 3 "mve_imm_7" "Ra")
5970                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5971          VQSHLUQ_M_N_S))
5972   ]
5973   "TARGET_HAVE_MVE"
5974   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5975   [(set_attr "type" "mve_move")])
5976
5977 ;;
5978 ;; [vshlq_m_s, vshlq_m_u])
5979 ;;
5980 (define_insn "mve_vshlq_m_<supf><mode>"
5981   [
5982    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5983         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5984                        (match_operand:MVE_2 2 "s_register_operand" "w")
5985                        (match_operand:MVE_2 3 "s_register_operand" "w")
5986                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5987          VSHLQ_M))
5988   ]
5989   "TARGET_HAVE_MVE"
5990   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5991   [(set_attr "type" "mve_move")])
5992
5993 ;;
5994 ;; [vsriq_m_n_s, vsriq_m_n_u])
5995 ;;
5996 (define_insn "mve_vsriq_m_n_<supf><mode>"
5997   [
5998    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5999         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6000                        (match_operand:MVE_2 2 "s_register_operand" "w")
6001                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
6002                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6003          VSRIQ_M_N))
6004   ]
6005   "TARGET_HAVE_MVE"
6006   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
6007   [(set_attr "type" "mve_move")])
6008
6009 ;;
6010 ;; [vsubq_m_u, vsubq_m_s])
6011 ;;
6012 (define_insn "mve_vsubq_m_<supf><mode>"
6013   [
6014    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6015         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6016                        (match_operand:MVE_2 2 "s_register_operand" "w")
6017                        (match_operand:MVE_2 3 "s_register_operand" "w")
6018                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6019          VSUBQ_M))
6020   ]
6021   "TARGET_HAVE_MVE"
6022   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
6023   [(set_attr "type" "mve_move")])
6024
6025 ;;
6026 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
6027 ;;
6028 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
6029   [
6030    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6031         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6032                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
6033                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6034                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6035          VCVTQ_M_N_TO_F))
6036   ]
6037   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6038   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6039   [(set_attr "type" "mve_move")
6040    (set_attr "length""8")])
6041 ;;
6042 ;; [vabdq_m_s, vabdq_m_u])
6043 ;;
6044 (define_insn "mve_vabdq_m_<supf><mode>"
6045   [
6046    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6047         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6048                        (match_operand:MVE_2 2 "s_register_operand" "w")
6049                        (match_operand:MVE_2 3 "s_register_operand" "w")
6050                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6051          VABDQ_M))
6052   ]
6053   "TARGET_HAVE_MVE"
6054   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6055   [(set_attr "type" "mve_move")
6056    (set_attr "length""8")])
6057
6058 ;;
6059 ;; [vaddq_m_n_s, vaddq_m_n_u])
6060 ;;
6061 (define_insn "mve_vaddq_m_n_<supf><mode>"
6062   [
6063    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6064         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6065                        (match_operand:MVE_2 2 "s_register_operand" "w")
6066                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6067                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6068          VADDQ_M_N))
6069   ]
6070   "TARGET_HAVE_MVE"
6071   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
6072   [(set_attr "type" "mve_move")
6073    (set_attr "length""8")])
6074
6075 ;;
6076 ;; [vaddq_m_u, vaddq_m_s])
6077 ;;
6078 (define_insn "mve_vaddq_m_<supf><mode>"
6079   [
6080    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6081         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6082                        (match_operand:MVE_2 2 "s_register_operand" "w")
6083                        (match_operand:MVE_2 3 "s_register_operand" "w")
6084                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6085          VADDQ_M))
6086   ]
6087   "TARGET_HAVE_MVE"
6088   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
6089   [(set_attr "type" "mve_move")
6090    (set_attr "length""8")])
6091
6092 ;;
6093 ;; [vandq_m_u, vandq_m_s])
6094 ;;
6095 (define_insn "mve_vandq_m_<supf><mode>"
6096   [
6097    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6098         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6099                        (match_operand:MVE_2 2 "s_register_operand" "w")
6100                        (match_operand:MVE_2 3 "s_register_operand" "w")
6101                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6102          VANDQ_M))
6103   ]
6104   "TARGET_HAVE_MVE"
6105   "vpst\;vandt %q0, %q2, %q3"
6106   [(set_attr "type" "mve_move")
6107    (set_attr "length""8")])
6108
6109 ;;
6110 ;; [vbicq_m_u, vbicq_m_s])
6111 ;;
6112 (define_insn "mve_vbicq_m_<supf><mode>"
6113   [
6114    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6115         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6116                        (match_operand:MVE_2 2 "s_register_operand" "w")
6117                        (match_operand:MVE_2 3 "s_register_operand" "w")
6118                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6119          VBICQ_M))
6120   ]
6121   "TARGET_HAVE_MVE"
6122   "vpst\;vbict %q0, %q2, %q3"
6123   [(set_attr "type" "mve_move")
6124    (set_attr "length""8")])
6125
6126 ;;
6127 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
6128 ;;
6129 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
6130   [
6131    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6132         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6133                        (match_operand:MVE_2 2 "s_register_operand" "w")
6134                        (match_operand:SI 3 "s_register_operand" "r")
6135                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6136          VBRSRQ_M_N))
6137   ]
6138   "TARGET_HAVE_MVE"
6139   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6140   [(set_attr "type" "mve_move")
6141    (set_attr "length""8")])
6142
6143 ;;
6144 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
6145 ;;
6146 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
6147   [
6148    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6149         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6150                        (match_operand:MVE_2 2 "s_register_operand" "w")
6151                        (match_operand:MVE_2 3 "s_register_operand" "w")
6152                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6153          VCADDQ_ROT270_M))
6154   ]
6155   "TARGET_HAVE_MVE"
6156   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
6157   [(set_attr "type" "mve_move")
6158    (set_attr "length""8")])
6159
6160 ;;
6161 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
6162 ;;
6163 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
6164   [
6165    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6166         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6167                        (match_operand:MVE_2 2 "s_register_operand" "w")
6168                        (match_operand:MVE_2 3 "s_register_operand" "w")
6169                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6170          VCADDQ_ROT90_M))
6171   ]
6172   "TARGET_HAVE_MVE"
6173   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
6174   [(set_attr "type" "mve_move")
6175    (set_attr "length""8")])
6176
6177 ;;
6178 ;; [veorq_m_s, veorq_m_u])
6179 ;;
6180 (define_insn "mve_veorq_m_<supf><mode>"
6181   [
6182    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6183         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6184                        (match_operand:MVE_2 2 "s_register_operand" "w")
6185                        (match_operand:MVE_2 3 "s_register_operand" "w")
6186                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6187          VEORQ_M))
6188   ]
6189   "TARGET_HAVE_MVE"
6190   "vpst\;veort %q0, %q2, %q3"
6191   [(set_attr "type" "mve_move")
6192    (set_attr "length""8")])
6193
6194 ;;
6195 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
6196 ;;
6197 (define_insn "mve_vhaddq_m_n_<supf><mode>"
6198   [
6199    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6200         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6201                        (match_operand:MVE_2 2 "s_register_operand" "w")
6202                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6203                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6204          VHADDQ_M_N))
6205   ]
6206   "TARGET_HAVE_MVE"
6207   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6208   [(set_attr "type" "mve_move")
6209    (set_attr "length""8")])
6210
6211 ;;
6212 ;; [vhaddq_m_s, vhaddq_m_u])
6213 ;;
6214 (define_insn "mve_vhaddq_m_<supf><mode>"
6215   [
6216    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6217         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6218                        (match_operand:MVE_2 2 "s_register_operand" "w")
6219                        (match_operand:MVE_2 3 "s_register_operand" "w")
6220                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6221          VHADDQ_M))
6222   ]
6223   "TARGET_HAVE_MVE"
6224   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6225   [(set_attr "type" "mve_move")
6226    (set_attr "length""8")])
6227
6228 ;;
6229 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
6230 ;;
6231 (define_insn "mve_vhsubq_m_n_<supf><mode>"
6232   [
6233    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6234         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6235                        (match_operand:MVE_2 2 "s_register_operand" "w")
6236                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6237                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6238          VHSUBQ_M_N))
6239   ]
6240   "TARGET_HAVE_MVE"
6241   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6242   [(set_attr "type" "mve_move")
6243    (set_attr "length""8")])
6244
6245 ;;
6246 ;; [vhsubq_m_s, vhsubq_m_u])
6247 ;;
6248 (define_insn "mve_vhsubq_m_<supf><mode>"
6249   [
6250    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6251         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6252                        (match_operand:MVE_2 2 "s_register_operand" "w")
6253                        (match_operand:MVE_2 3 "s_register_operand" "w")
6254                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6255          VHSUBQ_M))
6256   ]
6257   "TARGET_HAVE_MVE"
6258   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6259   [(set_attr "type" "mve_move")
6260    (set_attr "length""8")])
6261
6262 ;;
6263 ;; [vmaxq_m_s, vmaxq_m_u])
6264 ;;
6265 (define_insn "mve_vmaxq_m_<supf><mode>"
6266   [
6267    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6268         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6269                        (match_operand:MVE_2 2 "s_register_operand" "w")
6270                        (match_operand:MVE_2 3 "s_register_operand" "w")
6271                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6272          VMAXQ_M))
6273   ]
6274   "TARGET_HAVE_MVE"
6275   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6276   [(set_attr "type" "mve_move")
6277    (set_attr "length""8")])
6278
6279 ;;
6280 ;; [vminq_m_s, vminq_m_u])
6281 ;;
6282 (define_insn "mve_vminq_m_<supf><mode>"
6283   [
6284    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6285         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6286                        (match_operand:MVE_2 2 "s_register_operand" "w")
6287                        (match_operand:MVE_2 3 "s_register_operand" "w")
6288                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6289          VMINQ_M))
6290   ]
6291   "TARGET_HAVE_MVE"
6292   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6293   [(set_attr "type" "mve_move")
6294    (set_attr "length""8")])
6295
6296 ;;
6297 ;; [vmladavaq_p_u, vmladavaq_p_s])
6298 ;;
6299 (define_insn "mve_vmladavaq_p_<supf><mode>"
6300   [
6301    (set (match_operand:SI 0 "s_register_operand" "=e")
6302         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6303                     (match_operand:MVE_2 2 "s_register_operand" "w")
6304                     (match_operand:MVE_2 3 "s_register_operand" "w")
6305                     (match_operand:HI 4 "vpr_register_operand" "Up")]
6306          VMLADAVAQ_P))
6307   ]
6308   "TARGET_HAVE_MVE"
6309   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
6310   [(set_attr "type" "mve_move")
6311    (set_attr "length""8")])
6312
6313 ;;
6314 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
6315 ;;
6316 (define_insn "mve_vmlaq_m_n_<supf><mode>"
6317   [
6318    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6319         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6320                        (match_operand:MVE_2 2 "s_register_operand" "w")
6321                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6322                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6323          VMLAQ_M_N))
6324   ]
6325   "TARGET_HAVE_MVE"
6326   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
6327   [(set_attr "type" "mve_move")
6328    (set_attr "length""8")])
6329
6330 ;;
6331 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
6332 ;;
6333 (define_insn "mve_vmlasq_m_n_<supf><mode>"
6334   [
6335    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6336         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6337                        (match_operand:MVE_2 2 "s_register_operand" "w")
6338                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6339                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6340          VMLASQ_M_N))
6341   ]
6342   "TARGET_HAVE_MVE"
6343   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6344   [(set_attr "type" "mve_move")
6345    (set_attr "length""8")])
6346
6347 ;;
6348 ;; [vmulhq_m_s, vmulhq_m_u])
6349 ;;
6350 (define_insn "mve_vmulhq_m_<supf><mode>"
6351   [
6352    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6353         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6354                        (match_operand:MVE_2 2 "s_register_operand" "w")
6355                        (match_operand:MVE_2 3 "s_register_operand" "w")
6356                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6357          VMULHQ_M))
6358   ]
6359   "TARGET_HAVE_MVE"
6360   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6361   [(set_attr "type" "mve_move")
6362    (set_attr "length""8")])
6363
6364 ;;
6365 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
6366 ;;
6367 (define_insn "mve_vmullbq_int_m_<supf><mode>"
6368   [
6369    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6370         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6371                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6372                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6373                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6374          VMULLBQ_INT_M))
6375   ]
6376   "TARGET_HAVE_MVE"
6377   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6378   [(set_attr "type" "mve_move")
6379    (set_attr "length""8")])
6380
6381 ;;
6382 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
6383 ;;
6384 (define_insn "mve_vmulltq_int_m_<supf><mode>"
6385   [
6386    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6387         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6388                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6389                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6390                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6391          VMULLTQ_INT_M))
6392   ]
6393   "TARGET_HAVE_MVE"
6394   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6395   [(set_attr "type" "mve_move")
6396    (set_attr "length""8")])
6397
6398 ;;
6399 ;; [vmulq_m_n_u, vmulq_m_n_s])
6400 ;;
6401 (define_insn "mve_vmulq_m_n_<supf><mode>"
6402   [
6403    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6404         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6405                        (match_operand:MVE_2 2 "s_register_operand" "w")
6406                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6407                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6408          VMULQ_M_N))
6409   ]
6410   "TARGET_HAVE_MVE"
6411   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
6412   [(set_attr "type" "mve_move")
6413    (set_attr "length""8")])
6414
6415 ;;
6416 ;; [vmulq_m_s, vmulq_m_u])
6417 ;;
6418 (define_insn "mve_vmulq_m_<supf><mode>"
6419   [
6420    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6421         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6422                        (match_operand:MVE_2 2 "s_register_operand" "w")
6423                        (match_operand:MVE_2 3 "s_register_operand" "w")
6424                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6425          VMULQ_M))
6426   ]
6427   "TARGET_HAVE_MVE"
6428   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
6429   [(set_attr "type" "mve_move")
6430    (set_attr "length""8")])
6431
6432 ;;
6433 ;; [vornq_m_u, vornq_m_s])
6434 ;;
6435 (define_insn "mve_vornq_m_<supf><mode>"
6436   [
6437    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6438         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6439                        (match_operand:MVE_2 2 "s_register_operand" "w")
6440                        (match_operand:MVE_2 3 "s_register_operand" "w")
6441                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6442          VORNQ_M))
6443   ]
6444   "TARGET_HAVE_MVE"
6445   "vpst\;vornt %q0, %q2, %q3"
6446   [(set_attr "type" "mve_move")
6447    (set_attr "length""8")])
6448
6449 ;;
6450 ;; [vorrq_m_s, vorrq_m_u])
6451 ;;
6452 (define_insn "mve_vorrq_m_<supf><mode>"
6453   [
6454    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6455         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6456                        (match_operand:MVE_2 2 "s_register_operand" "w")
6457                        (match_operand:MVE_2 3 "s_register_operand" "w")
6458                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6459          VORRQ_M))
6460   ]
6461   "TARGET_HAVE_MVE"
6462   "vpst\;vorrt %q0, %q2, %q3"
6463   [(set_attr "type" "mve_move")
6464    (set_attr "length""8")])
6465
6466 ;;
6467 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
6468 ;;
6469 (define_insn "mve_vqaddq_m_n_<supf><mode>"
6470   [
6471    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6472         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6473                        (match_operand:MVE_2 2 "s_register_operand" "w")
6474                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6475                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6476          VQADDQ_M_N))
6477   ]
6478   "TARGET_HAVE_MVE"
6479   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6480   [(set_attr "type" "mve_move")
6481    (set_attr "length""8")])
6482
6483 ;;
6484 ;; [vqaddq_m_u, vqaddq_m_s])
6485 ;;
6486 (define_insn "mve_vqaddq_m_<supf><mode>"
6487   [
6488    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6489         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6490                        (match_operand:MVE_2 2 "s_register_operand" "w")
6491                        (match_operand:MVE_2 3 "s_register_operand" "w")
6492                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6493          VQADDQ_M))
6494   ]
6495   "TARGET_HAVE_MVE"
6496   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6497   [(set_attr "type" "mve_move")
6498    (set_attr "length""8")])
6499
6500 ;;
6501 ;; [vqdmlahq_m_n_s])
6502 ;;
6503 (define_insn "mve_vqdmlahq_m_n_s<mode>"
6504   [
6505    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6506         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6507                        (match_operand:MVE_2 2 "s_register_operand" "w")
6508                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6509                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6510          VQDMLAHQ_M_N_S))
6511   ]
6512   "TARGET_HAVE_MVE"
6513   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6514   [(set_attr "type" "mve_move")
6515    (set_attr "length""8")])
6516
6517 ;;
6518 ;; [vqrdmlahq_m_n_s])
6519 ;;
6520 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6521   [
6522    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6523         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6524                        (match_operand:MVE_2 2 "s_register_operand" "w")
6525                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6526                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6527          VQRDMLAHQ_M_N_S))
6528   ]
6529   "TARGET_HAVE_MVE"
6530   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6531   [(set_attr "type" "mve_move")
6532    (set_attr "length""8")])
6533
6534 ;;
6535 ;; [vqrdmlashq_m_n_s])
6536 ;;
6537 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6538   [
6539    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6540         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6541                        (match_operand:MVE_2 2 "s_register_operand" "w")
6542                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6543                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6544          VQRDMLASHQ_M_N_S))
6545   ]
6546   "TARGET_HAVE_MVE"
6547   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6548   [(set_attr "type" "mve_move")
6549    (set_attr "length""8")])
6550
6551 ;;
6552 ;; [vqrshlq_m_u, vqrshlq_m_s])
6553 ;;
6554 (define_insn "mve_vqrshlq_m_<supf><mode>"
6555   [
6556    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6557         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6558                        (match_operand:MVE_2 2 "s_register_operand" "w")
6559                        (match_operand:MVE_2 3 "s_register_operand" "w")
6560                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6561          VQRSHLQ_M))
6562   ]
6563   "TARGET_HAVE_MVE"
6564   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6565   [(set_attr "type" "mve_move")
6566    (set_attr "length""8")])
6567
6568 ;;
6569 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6570 ;;
6571 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6572   [
6573    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6574         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6575                        (match_operand:MVE_2 2 "s_register_operand" "w")
6576                        (match_operand:SI 3 "immediate_operand" "i")
6577                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6578          VQSHLQ_M_N))
6579   ]
6580   "TARGET_HAVE_MVE"
6581   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6582   [(set_attr "type" "mve_move")
6583    (set_attr "length""8")])
6584
6585 ;;
6586 ;; [vqshlq_m_u, vqshlq_m_s])
6587 ;;
6588 (define_insn "mve_vqshlq_m_<supf><mode>"
6589   [
6590    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6591         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6592                        (match_operand:MVE_2 2 "s_register_operand" "w")
6593                        (match_operand:MVE_2 3 "s_register_operand" "w")
6594                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6595          VQSHLQ_M))
6596   ]
6597   "TARGET_HAVE_MVE"
6598   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6599   [(set_attr "type" "mve_move")
6600    (set_attr "length""8")])
6601
6602 ;;
6603 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6604 ;;
6605 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6606   [
6607    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6608         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6609                        (match_operand:MVE_2 2 "s_register_operand" "w")
6610                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6611                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6612          VQSUBQ_M_N))
6613   ]
6614   "TARGET_HAVE_MVE"
6615   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6616   [(set_attr "type" "mve_move")
6617    (set_attr "length""8")])
6618
6619 ;;
6620 ;; [vqsubq_m_u, vqsubq_m_s])
6621 ;;
6622 (define_insn "mve_vqsubq_m_<supf><mode>"
6623   [
6624    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6625         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6626                        (match_operand:MVE_2 2 "s_register_operand" "w")
6627                        (match_operand:MVE_2 3 "s_register_operand" "w")
6628                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6629          VQSUBQ_M))
6630   ]
6631   "TARGET_HAVE_MVE"
6632   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6633   [(set_attr "type" "mve_move")
6634    (set_attr "length""8")])
6635
6636 ;;
6637 ;; [vrhaddq_m_u, vrhaddq_m_s])
6638 ;;
6639 (define_insn "mve_vrhaddq_m_<supf><mode>"
6640   [
6641    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6642         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6643                        (match_operand:MVE_2 2 "s_register_operand" "w")
6644                        (match_operand:MVE_2 3 "s_register_operand" "w")
6645                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6646          VRHADDQ_M))
6647   ]
6648   "TARGET_HAVE_MVE"
6649   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6650   [(set_attr "type" "mve_move")
6651    (set_attr "length""8")])
6652
6653 ;;
6654 ;; [vrmulhq_m_u, vrmulhq_m_s])
6655 ;;
6656 (define_insn "mve_vrmulhq_m_<supf><mode>"
6657   [
6658    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6659         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6660                        (match_operand:MVE_2 2 "s_register_operand" "w")
6661                        (match_operand:MVE_2 3 "s_register_operand" "w")
6662                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6663          VRMULHQ_M))
6664   ]
6665   "TARGET_HAVE_MVE"
6666   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6667   [(set_attr "type" "mve_move")
6668    (set_attr "length""8")])
6669
6670 ;;
6671 ;; [vrshlq_m_s, vrshlq_m_u])
6672 ;;
6673 (define_insn "mve_vrshlq_m_<supf><mode>"
6674   [
6675    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6676         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6677                        (match_operand:MVE_2 2 "s_register_operand" "w")
6678                        (match_operand:MVE_2 3 "s_register_operand" "w")
6679                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6680          VRSHLQ_M))
6681   ]
6682   "TARGET_HAVE_MVE"
6683   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6684   [(set_attr "type" "mve_move")
6685    (set_attr "length""8")])
6686
6687 ;;
6688 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6689 ;;
6690 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6691   [
6692    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6693         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6694                        (match_operand:MVE_2 2 "s_register_operand" "w")
6695                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6696                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6697          VRSHRQ_M_N))
6698   ]
6699   "TARGET_HAVE_MVE"
6700   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6701   [(set_attr "type" "mve_move")
6702    (set_attr "length""8")])
6703
6704 ;;
6705 ;; [vshlq_m_n_s, vshlq_m_n_u])
6706 ;;
6707 (define_insn "mve_vshlq_m_n_<supf><mode>"
6708   [
6709    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6710         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6711                        (match_operand:MVE_2 2 "s_register_operand" "w")
6712                        (match_operand:SI 3 "immediate_operand" "i")
6713                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6714          VSHLQ_M_N))
6715   ]
6716   "TARGET_HAVE_MVE"
6717   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6718   [(set_attr "type" "mve_move")
6719    (set_attr "length""8")])
6720
6721 ;;
6722 ;; [vshrq_m_n_s, vshrq_m_n_u])
6723 ;;
6724 (define_insn "mve_vshrq_m_n_<supf><mode>"
6725   [
6726    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6727         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6728                        (match_operand:MVE_2 2 "s_register_operand" "w")
6729                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6730                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6731          VSHRQ_M_N))
6732   ]
6733   "TARGET_HAVE_MVE"
6734   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6735   [(set_attr "type" "mve_move")
6736    (set_attr "length""8")])
6737
6738 ;;
6739 ;; [vsliq_m_n_u, vsliq_m_n_s])
6740 ;;
6741 (define_insn "mve_vsliq_m_n_<supf><mode>"
6742    [
6743    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6744        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6745                        (match_operand:MVE_2 2 "s_register_operand" "w")
6746                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6747                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6748          VSLIQ_M_N))
6749   ]
6750   "TARGET_HAVE_MVE"
6751   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6752   [(set_attr "type" "mve_move")
6753    (set_attr "length""8")])
6754
6755 ;;
6756 ;; [vsubq_m_n_s, vsubq_m_n_u])
6757 ;;
6758 (define_insn "mve_vsubq_m_n_<supf><mode>"
6759   [
6760    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6761         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6762                        (match_operand:MVE_2 2 "s_register_operand" "w")
6763                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6764                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6765          VSUBQ_M_N))
6766   ]
6767   "TARGET_HAVE_MVE"
6768   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6769   [(set_attr "type" "mve_move")
6770    (set_attr "length""8")])
6771
6772 ;;
6773 ;; [vhcaddq_rot270_m_s])
6774 ;;
6775 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6776   [
6777    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6778         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6779                        (match_operand:MVE_2 2 "s_register_operand" "w")
6780                        (match_operand:MVE_2 3 "s_register_operand" "w")
6781                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6782          VHCADDQ_ROT270_M_S))
6783   ]
6784   "TARGET_HAVE_MVE"
6785   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6786   [(set_attr "type" "mve_move")
6787    (set_attr "length""8")])
6788
6789 ;;
6790 ;; [vhcaddq_rot90_m_s])
6791 ;;
6792 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6793   [
6794    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6795         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6796                        (match_operand:MVE_2 2 "s_register_operand" "w")
6797                        (match_operand:MVE_2 3 "s_register_operand" "w")
6798                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6799          VHCADDQ_ROT90_M_S))
6800   ]
6801   "TARGET_HAVE_MVE"
6802   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6803   [(set_attr "type" "mve_move")
6804    (set_attr "length""8")])
6805
6806 ;;
6807 ;; [vmladavaxq_p_s])
6808 ;;
6809 (define_insn "mve_vmladavaxq_p_s<mode>"
6810   [
6811    (set (match_operand:SI 0 "s_register_operand" "=e")
6812         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6813                        (match_operand:MVE_2 2 "s_register_operand" "w")
6814                        (match_operand:MVE_2 3 "s_register_operand" "w")
6815                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6816          VMLADAVAXQ_P_S))
6817   ]
6818   "TARGET_HAVE_MVE"
6819   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6820   [(set_attr "type" "mve_move")
6821    (set_attr "length""8")])
6822
6823 ;;
6824 ;; [vmlsdavaq_p_s])
6825 ;;
6826 (define_insn "mve_vmlsdavaq_p_s<mode>"
6827   [
6828    (set (match_operand:SI 0 "s_register_operand" "=e")
6829         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6830                        (match_operand:MVE_2 2 "s_register_operand" "w")
6831                        (match_operand:MVE_2 3 "s_register_operand" "w")
6832                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6833          VMLSDAVAQ_P_S))
6834   ]
6835   "TARGET_HAVE_MVE"
6836   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6837   [(set_attr "type" "mve_move")
6838    (set_attr "length""8")])
6839
6840 ;;
6841 ;; [vmlsdavaxq_p_s])
6842 ;;
6843 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6844   [
6845    (set (match_operand:SI 0 "s_register_operand" "=e")
6846         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6847                        (match_operand:MVE_2 2 "s_register_operand" "w")
6848                        (match_operand:MVE_2 3 "s_register_operand" "w")
6849                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6850          VMLSDAVAXQ_P_S))
6851   ]
6852   "TARGET_HAVE_MVE"
6853   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6854   [(set_attr "type" "mve_move")
6855    (set_attr "length""8")])
6856
6857 ;;
6858 ;; [vqdmladhq_m_s])
6859 ;;
6860 (define_insn "mve_vqdmladhq_m_s<mode>"
6861   [
6862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6864                        (match_operand:MVE_2 2 "s_register_operand" "w")
6865                        (match_operand:MVE_2 3 "s_register_operand" "w")
6866                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6867          VQDMLADHQ_M_S))
6868   ]
6869   "TARGET_HAVE_MVE"
6870   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6871   [(set_attr "type" "mve_move")
6872    (set_attr "length""8")])
6873
6874 ;;
6875 ;; [vqdmladhxq_m_s])
6876 ;;
6877 (define_insn "mve_vqdmladhxq_m_s<mode>"
6878   [
6879    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6880         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6881                        (match_operand:MVE_2 2 "s_register_operand" "w")
6882                        (match_operand:MVE_2 3 "s_register_operand" "w")
6883                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6884          VQDMLADHXQ_M_S))
6885   ]
6886   "TARGET_HAVE_MVE"
6887   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6888   [(set_attr "type" "mve_move")
6889    (set_attr "length""8")])
6890
6891 ;;
6892 ;; [vqdmlsdhq_m_s])
6893 ;;
6894 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6895   [
6896    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6897         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6898                        (match_operand:MVE_2 2 "s_register_operand" "w")
6899                        (match_operand:MVE_2 3 "s_register_operand" "w")
6900                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6901          VQDMLSDHQ_M_S))
6902   ]
6903   "TARGET_HAVE_MVE"
6904   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6905   [(set_attr "type" "mve_move")
6906    (set_attr "length""8")])
6907
6908 ;;
6909 ;; [vqdmlsdhxq_m_s])
6910 ;;
6911 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6912   [
6913    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6914         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6915                        (match_operand:MVE_2 2 "s_register_operand" "w")
6916                        (match_operand:MVE_2 3 "s_register_operand" "w")
6917                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6918          VQDMLSDHXQ_M_S))
6919   ]
6920   "TARGET_HAVE_MVE"
6921   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6922   [(set_attr "type" "mve_move")
6923    (set_attr "length""8")])
6924
6925 ;;
6926 ;; [vqdmulhq_m_n_s])
6927 ;;
6928 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6929   [
6930    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6931         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6932                        (match_operand:MVE_2 2 "s_register_operand" "w")
6933                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6934                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6935          VQDMULHQ_M_N_S))
6936   ]
6937   "TARGET_HAVE_MVE"
6938   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6939   [(set_attr "type" "mve_move")
6940    (set_attr "length""8")])
6941
6942 ;;
6943 ;; [vqdmulhq_m_s])
6944 ;;
6945 (define_insn "mve_vqdmulhq_m_s<mode>"
6946   [
6947    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6948         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6949                        (match_operand:MVE_2 2 "s_register_operand" "w")
6950                        (match_operand:MVE_2 3 "s_register_operand" "w")
6951                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6952          VQDMULHQ_M_S))
6953   ]
6954   "TARGET_HAVE_MVE"
6955   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6956   [(set_attr "type" "mve_move")
6957    (set_attr "length""8")])
6958
6959 ;;
6960 ;; [vqrdmladhq_m_s])
6961 ;;
6962 (define_insn "mve_vqrdmladhq_m_s<mode>"
6963   [
6964    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6965         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6966                        (match_operand:MVE_2 2 "s_register_operand" "w")
6967                        (match_operand:MVE_2 3 "s_register_operand" "w")
6968                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6969          VQRDMLADHQ_M_S))
6970   ]
6971   "TARGET_HAVE_MVE"
6972   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6973   [(set_attr "type" "mve_move")
6974    (set_attr "length""8")])
6975
6976 ;;
6977 ;; [vqrdmladhxq_m_s])
6978 ;;
6979 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6980   [
6981    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6982         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6983                        (match_operand:MVE_2 2 "s_register_operand" "w")
6984                        (match_operand:MVE_2 3 "s_register_operand" "w")
6985                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6986          VQRDMLADHXQ_M_S))
6987   ]
6988   "TARGET_HAVE_MVE"
6989   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6990   [(set_attr "type" "mve_move")
6991    (set_attr "length""8")])
6992
6993 ;;
6994 ;; [vqrdmlsdhq_m_s])
6995 ;;
6996 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6997   [
6998    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6999         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7000                        (match_operand:MVE_2 2 "s_register_operand" "w")
7001                        (match_operand:MVE_2 3 "s_register_operand" "w")
7002                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7003          VQRDMLSDHQ_M_S))
7004   ]
7005   "TARGET_HAVE_MVE"
7006   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7007   [(set_attr "type" "mve_move")
7008    (set_attr "length""8")])
7009
7010 ;;
7011 ;; [vqrdmlsdhxq_m_s])
7012 ;;
7013 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
7014   [
7015    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7016         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7017                        (match_operand:MVE_2 2 "s_register_operand" "w")
7018                        (match_operand:MVE_2 3 "s_register_operand" "w")
7019                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7020          VQRDMLSDHXQ_M_S))
7021   ]
7022   "TARGET_HAVE_MVE"
7023   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7024   [(set_attr "type" "mve_move")
7025    (set_attr "length""8")])
7026
7027 ;;
7028 ;; [vqrdmulhq_m_n_s])
7029 ;;
7030 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
7031   [
7032    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7033         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7034                        (match_operand:MVE_2 2 "s_register_operand" "w")
7035                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7036                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7037          VQRDMULHQ_M_N_S))
7038   ]
7039   "TARGET_HAVE_MVE"
7040   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
7041   [(set_attr "type" "mve_move")
7042    (set_attr "length""8")])
7043
7044 ;;
7045 ;; [vqrdmulhq_m_s])
7046 ;;
7047 (define_insn "mve_vqrdmulhq_m_s<mode>"
7048   [
7049    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7050         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7051                        (match_operand:MVE_2 2 "s_register_operand" "w")
7052                        (match_operand:MVE_2 3 "s_register_operand" "w")
7053                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7054          VQRDMULHQ_M_S))
7055   ]
7056   "TARGET_HAVE_MVE"
7057   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7058   [(set_attr "type" "mve_move")
7059    (set_attr "length""8")])
7060
7061 ;;
7062 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
7063 ;;
7064 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
7065   [
7066    (set (match_operand:DI 0 "s_register_operand" "=r")
7067         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7068                        (match_operand:MVE_5 2 "s_register_operand" "w")
7069                        (match_operand:MVE_5 3 "s_register_operand" "w")
7070                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7071          VMLALDAVAQ_P))
7072   ]
7073   "TARGET_HAVE_MVE"
7074   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7075   [(set_attr "type" "mve_move")
7076    (set_attr "length""8")])
7077
7078 ;;
7079 ;; [vmlaldavaxq_p_u, vmlaldavaxq_p_s])
7080 ;;
7081 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
7082   [
7083    (set (match_operand:DI 0 "s_register_operand" "=r")
7084         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7085                        (match_operand:MVE_5 2 "s_register_operand" "w")
7086                        (match_operand:MVE_5 3 "s_register_operand" "w")
7087                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7088          VMLALDAVAXQ_P))
7089   ]
7090   "TARGET_HAVE_MVE"
7091   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7092   [(set_attr "type" "mve_move")
7093    (set_attr "length""8")])
7094
7095 ;;
7096 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
7097 ;;
7098 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
7099   [
7100    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7101         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7102                        (match_operand:MVE_5 2 "s_register_operand" "w")
7103                        (match_operand:SI 3 "mve_imm_8" "Rb")
7104                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7105          VQRSHRNBQ_M_N))
7106   ]
7107   "TARGET_HAVE_MVE"
7108   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7109   [(set_attr "type" "mve_move")
7110    (set_attr "length""8")])
7111
7112 ;;
7113 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
7114 ;;
7115 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
7116   [
7117    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7118         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7119                        (match_operand:MVE_5 2 "s_register_operand" "w")
7120                        (match_operand:SI 3 "mve_imm_8" "Rb")
7121                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7122          VQRSHRNTQ_M_N))
7123   ]
7124   "TARGET_HAVE_MVE"
7125   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7126   [(set_attr "type" "mve_move")
7127    (set_attr "length""8")])
7128
7129 ;;
7130 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
7131 ;;
7132 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
7133   [
7134    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7135         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7136                        (match_operand:MVE_5 2 "s_register_operand" "w")
7137                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7138                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7139          VQSHRNBQ_M_N))
7140   ]
7141   "TARGET_HAVE_MVE"
7142   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7143   [(set_attr "type" "mve_move")
7144    (set_attr "length""8")])
7145
7146 ;;
7147 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
7148 ;;
7149 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
7150   [
7151    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7152         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7153                        (match_operand:MVE_5 2 "s_register_operand" "w")
7154                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7155                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7156          VQSHRNTQ_M_N))
7157   ]
7158   "TARGET_HAVE_MVE"
7159   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7160   [(set_attr "type" "mve_move")
7161    (set_attr "length""8")])
7162
7163 ;;
7164 ;; [vrmlaldavhaq_p_s])
7165 ;;
7166 (define_insn "mve_vrmlaldavhaq_p_sv4si"
7167   [
7168    (set (match_operand:DI 0 "s_register_operand" "=r")
7169         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7170                        (match_operand:V4SI 2 "s_register_operand" "w")
7171                        (match_operand:V4SI 3 "s_register_operand" "w")
7172                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7173          VRMLALDAVHAQ_P_S))
7174   ]
7175   "TARGET_HAVE_MVE"
7176   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
7177   [(set_attr "type" "mve_move")
7178    (set_attr "length""8")])
7179
7180 ;;
7181 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
7182 ;;
7183 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
7184   [
7185    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7186         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7187                        (match_operand:MVE_5 2 "s_register_operand" "w")
7188                        (match_operand:SI 3 "mve_imm_8" "Rb")
7189                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7190          VRSHRNBQ_M_N))
7191   ]
7192   "TARGET_HAVE_MVE"
7193   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7194   [(set_attr "type" "mve_move")
7195    (set_attr "length""8")])
7196
7197 ;;
7198 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
7199 ;;
7200 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
7201   [
7202    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7203         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7204                        (match_operand:MVE_5 2 "s_register_operand" "w")
7205                        (match_operand:SI 3 "mve_imm_8" "Rb")
7206                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7207          VRSHRNTQ_M_N))
7208   ]
7209   "TARGET_HAVE_MVE"
7210   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7211   [(set_attr "type" "mve_move")
7212    (set_attr "length""8")])
7213
7214 ;;
7215 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
7216 ;;
7217 (define_insn "mve_vshllbq_m_n_<supf><mode>"
7218   [
7219    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7220         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7221                        (match_operand:MVE_3 2 "s_register_operand" "w")
7222                        (match_operand:SI 3 "immediate_operand" "i")
7223                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7224          VSHLLBQ_M_N))
7225   ]
7226   "TARGET_HAVE_MVE"
7227   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7228   [(set_attr "type" "mve_move")
7229    (set_attr "length""8")])
7230
7231 ;;
7232 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
7233 ;;
7234 (define_insn "mve_vshlltq_m_n_<supf><mode>"
7235   [
7236    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7237         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7238                        (match_operand:MVE_3 2 "s_register_operand" "w")
7239                        (match_operand:SI 3 "immediate_operand" "i")
7240                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7241          VSHLLTQ_M_N))
7242   ]
7243   "TARGET_HAVE_MVE"
7244   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7245   [(set_attr "type" "mve_move")
7246    (set_attr "length""8")])
7247
7248 ;;
7249 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
7250 ;;
7251 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
7252   [
7253    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7254         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7255                        (match_operand:MVE_5 2 "s_register_operand" "w")
7256                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7257                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7258          VSHRNBQ_M_N))
7259   ]
7260   "TARGET_HAVE_MVE"
7261   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7262   [(set_attr "type" "mve_move")
7263    (set_attr "length""8")])
7264
7265 ;;
7266 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
7267 ;;
7268 (define_insn "mve_vshrntq_m_n_<supf><mode>"
7269   [
7270    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7271         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7272                        (match_operand:MVE_5 2 "s_register_operand" "w")
7273                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7274                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7275          VSHRNTQ_M_N))
7276   ]
7277   "TARGET_HAVE_MVE"
7278   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7279   [(set_attr "type" "mve_move")
7280    (set_attr "length""8")])
7281
7282 ;;
7283 ;; [vmlsldavaq_p_s])
7284 ;;
7285 (define_insn "mve_vmlsldavaq_p_s<mode>"
7286   [
7287    (set (match_operand:DI 0 "s_register_operand" "=r")
7288         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7289                        (match_operand:MVE_5 2 "s_register_operand" "w")
7290                        (match_operand:MVE_5 3 "s_register_operand" "w")
7291                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7292          VMLSLDAVAQ_P_S))
7293   ]
7294   "TARGET_HAVE_MVE"
7295   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7296   [(set_attr "type" "mve_move")
7297    (set_attr "length""8")])
7298
7299 ;;
7300 ;; [vmlsldavaxq_p_s])
7301 ;;
7302 (define_insn "mve_vmlsldavaxq_p_s<mode>"
7303   [
7304    (set (match_operand:DI 0 "s_register_operand" "=r")
7305         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7306                        (match_operand:MVE_5 2 "s_register_operand" "w")
7307                        (match_operand:MVE_5 3 "s_register_operand" "w")
7308                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7309          VMLSLDAVAXQ_P_S))
7310   ]
7311   "TARGET_HAVE_MVE"
7312   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7313   [(set_attr "type" "mve_move")
7314    (set_attr "length""8")])
7315
7316 ;;
7317 ;; [vmullbq_poly_m_p])
7318 ;;
7319 (define_insn "mve_vmullbq_poly_m_p<mode>"
7320   [
7321    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7322         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7323                        (match_operand:MVE_3 2 "s_register_operand" "w")
7324                        (match_operand:MVE_3 3 "s_register_operand" "w")
7325                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7326          VMULLBQ_POLY_M_P))
7327   ]
7328   "TARGET_HAVE_MVE"
7329   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7330   [(set_attr "type" "mve_move")
7331    (set_attr "length""8")])
7332
7333 ;;
7334 ;; [vmulltq_poly_m_p])
7335 ;;
7336 (define_insn "mve_vmulltq_poly_m_p<mode>"
7337   [
7338    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7339         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7340                        (match_operand:MVE_3 2 "s_register_operand" "w")
7341                        (match_operand:MVE_3 3 "s_register_operand" "w")
7342                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7343          VMULLTQ_POLY_M_P))
7344   ]
7345   "TARGET_HAVE_MVE"
7346   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7347   [(set_attr "type" "mve_move")
7348    (set_attr "length""8")])
7349
7350 ;;
7351 ;; [vqdmullbq_m_n_s])
7352 ;;
7353 (define_insn "mve_vqdmullbq_m_n_s<mode>"
7354   [
7355    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7356         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7357                        (match_operand:MVE_5 2 "s_register_operand" "w")
7358                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7359                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7360          VQDMULLBQ_M_N_S))
7361   ]
7362   "TARGET_HAVE_MVE"
7363   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7364   [(set_attr "type" "mve_move")
7365    (set_attr "length""8")])
7366
7367 ;;
7368 ;; [vqdmullbq_m_s])
7369 ;;
7370 (define_insn "mve_vqdmullbq_m_s<mode>"
7371   [
7372    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7373         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7374                        (match_operand:MVE_5 2 "s_register_operand" "w")
7375                        (match_operand:MVE_5 3 "s_register_operand" "w")
7376                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7377          VQDMULLBQ_M_S))
7378   ]
7379   "TARGET_HAVE_MVE"
7380   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7381   [(set_attr "type" "mve_move")
7382    (set_attr "length""8")])
7383
7384 ;;
7385 ;; [vqdmulltq_m_n_s])
7386 ;;
7387 (define_insn "mve_vqdmulltq_m_n_s<mode>"
7388   [
7389    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7390         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7391                        (match_operand:MVE_5 2 "s_register_operand" "w")
7392                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7393                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7394          VQDMULLTQ_M_N_S))
7395   ]
7396   "TARGET_HAVE_MVE"
7397   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
7398   [(set_attr "type" "mve_move")
7399    (set_attr "length""8")])
7400
7401 ;;
7402 ;; [vqdmulltq_m_s])
7403 ;;
7404 (define_insn "mve_vqdmulltq_m_s<mode>"
7405   [
7406    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7407         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7408                        (match_operand:MVE_5 2 "s_register_operand" "w")
7409                        (match_operand:MVE_5 3 "s_register_operand" "w")
7410                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7411          VQDMULLTQ_M_S))
7412   ]
7413   "TARGET_HAVE_MVE"
7414   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7415   [(set_attr "type" "mve_move")
7416    (set_attr "length""8")])
7417
7418 ;;
7419 ;; [vqrshrunbq_m_n_s])
7420 ;;
7421 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
7422   [
7423    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7424         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7425                        (match_operand:MVE_5 2 "s_register_operand" "w")
7426                        (match_operand:SI 3 "mve_imm_8" "Rb")
7427                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7428          VQRSHRUNBQ_M_N_S))
7429   ]
7430   "TARGET_HAVE_MVE"
7431   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7432   [(set_attr "type" "mve_move")
7433    (set_attr "length""8")])
7434
7435 ;;
7436 ;; [vqrshruntq_m_n_s])
7437 ;;
7438 (define_insn "mve_vqrshruntq_m_n_s<mode>"
7439   [
7440    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7441         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7442                        (match_operand:MVE_5 2 "s_register_operand" "w")
7443                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7444                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7445          VQRSHRUNTQ_M_N_S))
7446   ]
7447   "TARGET_HAVE_MVE"
7448   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7449   [(set_attr "type" "mve_move")
7450    (set_attr "length""8")])
7451
7452 ;;
7453 ;; [vqshrunbq_m_n_s])
7454 ;;
7455 (define_insn "mve_vqshrunbq_m_n_s<mode>"
7456   [
7457    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7458         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7459                        (match_operand:MVE_5 2 "s_register_operand" "w")
7460                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7461                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7462          VQSHRUNBQ_M_N_S))
7463   ]
7464   "TARGET_HAVE_MVE"
7465   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7466   [(set_attr "type" "mve_move")
7467    (set_attr "length""8")])
7468
7469 ;;
7470 ;; [vqshruntq_m_n_s])
7471 ;;
7472 (define_insn "mve_vqshruntq_m_n_s<mode>"
7473   [
7474    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7475         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7476                        (match_operand:MVE_5 2 "s_register_operand" "w")
7477                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7478                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7479          VQSHRUNTQ_M_N_S))
7480   ]
7481   "TARGET_HAVE_MVE"
7482   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7483   [(set_attr "type" "mve_move")
7484    (set_attr "length""8")])
7485
7486 ;;
7487 ;; [vrmlaldavhaq_p_u])
7488 ;;
7489 (define_insn "mve_vrmlaldavhaq_p_uv4si"
7490   [
7491    (set (match_operand:DI 0 "s_register_operand" "=r")
7492         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7493                        (match_operand:V4SI 2 "s_register_operand" "w")
7494                        (match_operand:V4SI 3 "s_register_operand" "w")
7495                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7496          VRMLALDAVHAQ_P_U))
7497   ]
7498   "TARGET_HAVE_MVE"
7499   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7500   [(set_attr "type" "mve_move")
7501    (set_attr "length""8")])
7502
7503 ;;
7504 ;; [vrmlaldavhaxq_p_s])
7505 ;;
7506 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7507   [
7508    (set (match_operand:DI 0 "s_register_operand" "=r")
7509         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7510                        (match_operand:V4SI 2 "s_register_operand" "w")
7511                        (match_operand:V4SI 3 "s_register_operand" "w")
7512                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7513          VRMLALDAVHAXQ_P_S))
7514   ]
7515   "TARGET_HAVE_MVE"
7516   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7517   [(set_attr "type" "mve_move")
7518    (set_attr "length""8")])
7519
7520 ;;
7521 ;; [vrmlsldavhaq_p_s])
7522 ;;
7523 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7524   [
7525    (set (match_operand:DI 0 "s_register_operand" "=r")
7526         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7527                        (match_operand:V4SI 2 "s_register_operand" "w")
7528                        (match_operand:V4SI 3 "s_register_operand" "w")
7529                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7530          VRMLSLDAVHAQ_P_S))
7531   ]
7532   "TARGET_HAVE_MVE"
7533   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7534   [(set_attr "type" "mve_move")
7535    (set_attr "length""8")])
7536
7537 ;;
7538 ;; [vrmlsldavhaxq_p_s])
7539 ;;
7540 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7541   [
7542    (set (match_operand:DI 0 "s_register_operand" "=r")
7543         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7544                        (match_operand:V4SI 2 "s_register_operand" "w")
7545                        (match_operand:V4SI 3 "s_register_operand" "w")
7546                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7547          VRMLSLDAVHAXQ_P_S))
7548   ]
7549   "TARGET_HAVE_MVE"
7550   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7551   [(set_attr "type" "mve_move")
7552    (set_attr "length""8")])
7553 ;;
7554 ;; [vabdq_m_f])
7555 ;;
7556 (define_insn "mve_vabdq_m_f<mode>"
7557   [
7558    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7559         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7560                        (match_operand:MVE_0 2 "s_register_operand" "w")
7561                        (match_operand:MVE_0 3 "s_register_operand" "w")
7562                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7563          VABDQ_M_F))
7564   ]
7565   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7566   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7567   [(set_attr "type" "mve_move")
7568    (set_attr "length""8")])
7569
7570 ;;
7571 ;; [vaddq_m_f])
7572 ;;
7573 (define_insn "mve_vaddq_m_f<mode>"
7574   [
7575    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7576         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7577                        (match_operand:MVE_0 2 "s_register_operand" "w")
7578                        (match_operand:MVE_0 3 "s_register_operand" "w")
7579                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7580          VADDQ_M_F))
7581   ]
7582   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7583   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7584   [(set_attr "type" "mve_move")
7585    (set_attr "length""8")])
7586
7587 ;;
7588 ;; [vaddq_m_n_f])
7589 ;;
7590 (define_insn "mve_vaddq_m_n_f<mode>"
7591   [
7592    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7593         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7594                        (match_operand:MVE_0 2 "s_register_operand" "w")
7595                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7596                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7597          VADDQ_M_N_F))
7598   ]
7599   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7600   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7601   [(set_attr "type" "mve_move")
7602    (set_attr "length""8")])
7603
7604 ;;
7605 ;; [vandq_m_f])
7606 ;;
7607 (define_insn "mve_vandq_m_f<mode>"
7608   [
7609    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7610         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7611                        (match_operand:MVE_0 2 "s_register_operand" "w")
7612                        (match_operand:MVE_0 3 "s_register_operand" "w")
7613                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7614          VANDQ_M_F))
7615   ]
7616   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7617   "vpst\;vandt %q0, %q2, %q3"
7618   [(set_attr "type" "mve_move")
7619    (set_attr "length""8")])
7620
7621 ;;
7622 ;; [vbicq_m_f])
7623 ;;
7624 (define_insn "mve_vbicq_m_f<mode>"
7625   [
7626    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7627         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7628                        (match_operand:MVE_0 2 "s_register_operand" "w")
7629                        (match_operand:MVE_0 3 "s_register_operand" "w")
7630                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7631          VBICQ_M_F))
7632   ]
7633   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7634   "vpst\;vbict %q0, %q2, %q3"
7635   [(set_attr "type" "mve_move")
7636    (set_attr "length""8")])
7637
7638 ;;
7639 ;; [vbrsrq_m_n_f])
7640 ;;
7641 (define_insn "mve_vbrsrq_m_n_f<mode>"
7642   [
7643    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7644         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7645                        (match_operand:MVE_0 2 "s_register_operand" "w")
7646                        (match_operand:SI 3 "s_register_operand" "r")
7647                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7648          VBRSRQ_M_N_F))
7649   ]
7650   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7651   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7652   [(set_attr "type" "mve_move")
7653    (set_attr "length""8")])
7654
7655 ;;
7656 ;; [vcaddq_rot270_m_f])
7657 ;;
7658 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7659   [
7660    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7661         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7662                        (match_operand:MVE_0 2 "s_register_operand" "w")
7663                        (match_operand:MVE_0 3 "s_register_operand" "w")
7664                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7665          VCADDQ_ROT270_M_F))
7666   ]
7667   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7668   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7669   [(set_attr "type" "mve_move")
7670    (set_attr "length""8")])
7671
7672 ;;
7673 ;; [vcaddq_rot90_m_f])
7674 ;;
7675 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7676   [
7677    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7678         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7679                        (match_operand:MVE_0 2 "s_register_operand" "w")
7680                        (match_operand:MVE_0 3 "s_register_operand" "w")
7681                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7682          VCADDQ_ROT90_M_F))
7683   ]
7684   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7685   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7686   [(set_attr "type" "mve_move")
7687    (set_attr "length""8")])
7688
7689 ;;
7690 ;; [vcmlaq_m_f])
7691 ;;
7692 (define_insn "mve_vcmlaq_m_f<mode>"
7693   [
7694    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7695         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7696                        (match_operand:MVE_0 2 "s_register_operand" "w")
7697                        (match_operand:MVE_0 3 "s_register_operand" "w")
7698                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7699          VCMLAQ_M_F))
7700   ]
7701   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7702   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7703   [(set_attr "type" "mve_move")
7704    (set_attr "length""8")])
7705
7706 ;;
7707 ;; [vcmlaq_rot180_m_f])
7708 ;;
7709 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7710   [
7711    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7712         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7713                        (match_operand:MVE_0 2 "s_register_operand" "w")
7714                        (match_operand:MVE_0 3 "s_register_operand" "w")
7715                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7716          VCMLAQ_ROT180_M_F))
7717   ]
7718   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7719   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7720   [(set_attr "type" "mve_move")
7721    (set_attr "length""8")])
7722
7723 ;;
7724 ;; [vcmlaq_rot270_m_f])
7725 ;;
7726 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7727   [
7728    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7729         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7730                        (match_operand:MVE_0 2 "s_register_operand" "w")
7731                        (match_operand:MVE_0 3 "s_register_operand" "w")
7732                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7733          VCMLAQ_ROT270_M_F))
7734   ]
7735   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7736   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7737   [(set_attr "type" "mve_move")
7738    (set_attr "length""8")])
7739
7740 ;;
7741 ;; [vcmlaq_rot90_m_f])
7742 ;;
7743 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7744   [
7745    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7746         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7747                        (match_operand:MVE_0 2 "s_register_operand" "w")
7748                        (match_operand:MVE_0 3 "s_register_operand" "w")
7749                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7750          VCMLAQ_ROT90_M_F))
7751   ]
7752   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7753   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7754   [(set_attr "type" "mve_move")
7755    (set_attr "length""8")])
7756
7757 ;;
7758 ;; [vcmulq_m_f])
7759 ;;
7760 (define_insn "mve_vcmulq_m_f<mode>"
7761   [
7762    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7763         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7764                        (match_operand:MVE_0 2 "s_register_operand" "w")
7765                        (match_operand:MVE_0 3 "s_register_operand" "w")
7766                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7767          VCMULQ_M_F))
7768   ]
7769   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7770   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7771   [(set_attr "type" "mve_move")
7772    (set_attr "length""8")])
7773
7774 ;;
7775 ;; [vcmulq_rot180_m_f])
7776 ;;
7777 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7778   [
7779    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7780         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7781                        (match_operand:MVE_0 2 "s_register_operand" "w")
7782                        (match_operand:MVE_0 3 "s_register_operand" "w")
7783                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7784          VCMULQ_ROT180_M_F))
7785   ]
7786   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7787   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7788   [(set_attr "type" "mve_move")
7789    (set_attr "length""8")])
7790
7791 ;;
7792 ;; [vcmulq_rot270_m_f])
7793 ;;
7794 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7795   [
7796    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7797         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7798                        (match_operand:MVE_0 2 "s_register_operand" "w")
7799                        (match_operand:MVE_0 3 "s_register_operand" "w")
7800                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7801          VCMULQ_ROT270_M_F))
7802   ]
7803   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7804   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7805   [(set_attr "type" "mve_move")
7806    (set_attr "length""8")])
7807
7808 ;;
7809 ;; [vcmulq_rot90_m_f])
7810 ;;
7811 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7812   [
7813    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7814         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7815                        (match_operand:MVE_0 2 "s_register_operand" "w")
7816                        (match_operand:MVE_0 3 "s_register_operand" "w")
7817                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7818          VCMULQ_ROT90_M_F))
7819   ]
7820   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7821   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7822   [(set_attr "type" "mve_move")
7823    (set_attr "length""8")])
7824
7825 ;;
7826 ;; [veorq_m_f])
7827 ;;
7828 (define_insn "mve_veorq_m_f<mode>"
7829   [
7830    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7831         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7832                        (match_operand:MVE_0 2 "s_register_operand" "w")
7833                        (match_operand:MVE_0 3 "s_register_operand" "w")
7834                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7835          VEORQ_M_F))
7836   ]
7837   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7838   "vpst\;veort %q0, %q2, %q3"
7839   [(set_attr "type" "mve_move")
7840    (set_attr "length""8")])
7841
7842 ;;
7843 ;; [vfmaq_m_f])
7844 ;;
7845 (define_insn "mve_vfmaq_m_f<mode>"
7846   [
7847    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7848         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7849                        (match_operand:MVE_0 2 "s_register_operand" "w")
7850                        (match_operand:MVE_0 3 "s_register_operand" "w")
7851                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7852          VFMAQ_M_F))
7853   ]
7854   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7855   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7856   [(set_attr "type" "mve_move")
7857    (set_attr "length""8")])
7858
7859 ;;
7860 ;; [vfmaq_m_n_f])
7861 ;;
7862 (define_insn "mve_vfmaq_m_n_f<mode>"
7863   [
7864    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7865         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7866                        (match_operand:MVE_0 2 "s_register_operand" "w")
7867                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7868                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7869          VFMAQ_M_N_F))
7870   ]
7871   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7872   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7873   [(set_attr "type" "mve_move")
7874    (set_attr "length""8")])
7875
7876 ;;
7877 ;; [vfmasq_m_n_f])
7878 ;;
7879 (define_insn "mve_vfmasq_m_n_f<mode>"
7880   [
7881    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7882         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7883                        (match_operand:MVE_0 2 "s_register_operand" "w")
7884                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7885                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7886          VFMASQ_M_N_F))
7887   ]
7888   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7889   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7890   [(set_attr "type" "mve_move")
7891    (set_attr "length""8")])
7892
7893 ;;
7894 ;; [vfmsq_m_f])
7895 ;;
7896 (define_insn "mve_vfmsq_m_f<mode>"
7897   [
7898    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7899         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7900                        (match_operand:MVE_0 2 "s_register_operand" "w")
7901                        (match_operand:MVE_0 3 "s_register_operand" "w")
7902                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7903          VFMSQ_M_F))
7904   ]
7905   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7906   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7907   [(set_attr "type" "mve_move")
7908    (set_attr "length""8")])
7909
7910 ;;
7911 ;; [vmaxnmq_m_f])
7912 ;;
7913 (define_insn "mve_vmaxnmq_m_f<mode>"
7914   [
7915    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7916         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7917                        (match_operand:MVE_0 2 "s_register_operand" "w")
7918                        (match_operand:MVE_0 3 "s_register_operand" "w")
7919                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7920          VMAXNMQ_M_F))
7921   ]
7922   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7923   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7924   [(set_attr "type" "mve_move")
7925    (set_attr "length""8")])
7926
7927 ;;
7928 ;; [vminnmq_m_f])
7929 ;;
7930 (define_insn "mve_vminnmq_m_f<mode>"
7931   [
7932    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7933         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7934                        (match_operand:MVE_0 2 "s_register_operand" "w")
7935                        (match_operand:MVE_0 3 "s_register_operand" "w")
7936                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7937          VMINNMQ_M_F))
7938   ]
7939   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7940   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7941   [(set_attr "type" "mve_move")
7942    (set_attr "length""8")])
7943
7944 ;;
7945 ;; [vmulq_m_f])
7946 ;;
7947 (define_insn "mve_vmulq_m_f<mode>"
7948   [
7949    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7950         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7951                        (match_operand:MVE_0 2 "s_register_operand" "w")
7952                        (match_operand:MVE_0 3 "s_register_operand" "w")
7953                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7954          VMULQ_M_F))
7955   ]
7956   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7957   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7958   [(set_attr "type" "mve_move")
7959    (set_attr "length""8")])
7960
7961 ;;
7962 ;; [vmulq_m_n_f])
7963 ;;
7964 (define_insn "mve_vmulq_m_n_f<mode>"
7965   [
7966    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7967         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7968                        (match_operand:MVE_0 2 "s_register_operand" "w")
7969                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7970                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7971          VMULQ_M_N_F))
7972   ]
7973   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7974   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7975   [(set_attr "type" "mve_move")
7976    (set_attr "length""8")])
7977
7978 ;;
7979 ;; [vornq_m_f])
7980 ;;
7981 (define_insn "mve_vornq_m_f<mode>"
7982   [
7983    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7984         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7985                        (match_operand:MVE_0 2 "s_register_operand" "w")
7986                        (match_operand:MVE_0 3 "s_register_operand" "w")
7987                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7988          VORNQ_M_F))
7989   ]
7990   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7991   "vpst\;vornt %q0, %q2, %q3"
7992   [(set_attr "type" "mve_move")
7993    (set_attr "length""8")])
7994
7995 ;;
7996 ;; [vorrq_m_f])
7997 ;;
7998 (define_insn "mve_vorrq_m_f<mode>"
7999   [
8000    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8001         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8002                        (match_operand:MVE_0 2 "s_register_operand" "w")
8003                        (match_operand:MVE_0 3 "s_register_operand" "w")
8004                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8005          VORRQ_M_F))
8006   ]
8007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8008   "vpst\;vorrt %q0, %q2, %q3"
8009   [(set_attr "type" "mve_move")
8010    (set_attr "length""8")])
8011
8012 ;;
8013 ;; [vsubq_m_f])
8014 ;;
8015 (define_insn "mve_vsubq_m_f<mode>"
8016   [
8017    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8018         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8019                        (match_operand:MVE_0 2 "s_register_operand" "w")
8020                        (match_operand:MVE_0 3 "s_register_operand" "w")
8021                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8022          VSUBQ_M_F))
8023   ]
8024   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8025   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
8026   [(set_attr "type" "mve_move")
8027    (set_attr "length""8")])
8028
8029 ;;
8030 ;; [vsubq_m_n_f])
8031 ;;
8032 (define_insn "mve_vsubq_m_n_f<mode>"
8033   [
8034    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8035         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8036                        (match_operand:MVE_0 2 "s_register_operand" "w")
8037                        (match_operand:<V_elem> 3 "s_register_operand" "r")
8038                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8039          VSUBQ_M_N_F))
8040   ]
8041   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8042   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
8043   [(set_attr "type" "mve_move")
8044    (set_attr "length""8")])
8045
8046 ;;
8047 ;; [vstrbq_s vstrbq_u]
8048 ;;
8049 (define_insn "mve_vstrbq_<supf><mode>"
8050   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8051         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
8052          VSTRBQ))
8053   ]
8054   "TARGET_HAVE_MVE"
8055 {
8056    rtx ops[2];
8057    int regno = REGNO (operands[1]);
8058    ops[1] = gen_rtx_REG (TImode, regno);
8059    ops[0]  = operands[0];
8060    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
8061    return "";
8062 }
8063   [(set_attr "length" "4")])
8064
8065 ;;
8066 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
8067 ;;
8068 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>"
8069   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8070         (unspec:<MVE_B_ELEM>
8071                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8072                  (match_operand:MVE_2 2 "s_register_operand" "w")]
8073          VSTRBSOQ))
8074   ]
8075   "TARGET_HAVE_MVE"
8076 {
8077    rtx ops[3];
8078    ops[0] = operands[0];
8079    ops[1] = operands[1];
8080    ops[2] = operands[2];
8081    output_asm_insn("vstrb.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8082    return "";
8083 }
8084   [(set_attr "length" "4")])
8085
8086 ;;
8087 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
8088 ;;
8089 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
8090   [(set (mem:BLK (scratch))
8091         (unspec:BLK
8092                 [(match_operand:V4SI 0 "s_register_operand" "w")
8093                  (match_operand:SI 1 "immediate_operand" "i")
8094                  (match_operand:V4SI 2 "s_register_operand" "w")]
8095          VSTRWSBQ))
8096   ]
8097   "TARGET_HAVE_MVE"
8098 {
8099    rtx ops[3];
8100    ops[0] = operands[0];
8101    ops[1] = operands[1];
8102    ops[2] = operands[2];
8103    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
8104    return "";
8105 }
8106   [(set_attr "length" "4")])
8107
8108 ;;
8109 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
8110 ;;
8111 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
8112   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8113         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8114                        (match_operand:MVE_2 2 "s_register_operand" "w")]
8115          VLDRBGOQ))
8116   ]
8117   "TARGET_HAVE_MVE"
8118 {
8119    rtx ops[3];
8120    ops[0] = operands[0];
8121    ops[1] = operands[1];
8122    ops[2] = operands[2];
8123    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8124      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
8125    else
8126      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8127    return "";
8128 }
8129   [(set_attr "length" "4")])
8130
8131 ;;
8132 ;; [vldrbq_s vldrbq_u]
8133 ;;
8134 (define_insn "mve_vldrbq_<supf><mode>"
8135   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8136         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")]
8137          VLDRBQ))
8138   ]
8139   "TARGET_HAVE_MVE"
8140 {
8141    rtx ops[2];
8142    int regno = REGNO (operands[0]);
8143    ops[0] = gen_rtx_REG (TImode, regno);
8144    ops[1]  = operands[1];
8145    output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
8146    return "";
8147 }
8148   [(set_attr "length" "4")])
8149
8150 ;;
8151 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
8152 ;;
8153 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
8154   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8155         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8156                       (match_operand:SI 2 "immediate_operand" "i")]
8157          VLDRWGBQ))
8158   ]
8159   "TARGET_HAVE_MVE"
8160 {
8161    rtx ops[3];
8162    ops[0] = operands[0];
8163    ops[1] = operands[1];
8164    ops[2] = operands[2];
8165    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8166    return "";
8167 }
8168   [(set_attr "length" "4")])
8169
8170 ;;
8171 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
8172 ;;
8173 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>"
8174   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8175         (unspec:<MVE_B_ELEM>
8176                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8177                  (match_operand:MVE_2 2 "s_register_operand" "w")
8178                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8179          VSTRBSOQ))
8180   ]
8181   "TARGET_HAVE_MVE"
8182 {
8183    rtx ops[3];
8184    ops[0] = operands[0];
8185    ops[1] = operands[1];
8186    ops[2] = operands[2];
8187    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8188    return "";
8189 }
8190   [(set_attr "length" "8")])
8191
8192 ;;
8193 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
8194 ;;
8195 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
8196   [(set (mem:BLK (scratch))
8197         (unspec:BLK
8198                 [(match_operand:V4SI 0 "s_register_operand" "w")
8199                  (match_operand:SI 1 "immediate_operand" "i")
8200                  (match_operand:V4SI 2 "s_register_operand" "w")
8201                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8202          VSTRWSBQ))
8203   ]
8204   "TARGET_HAVE_MVE"
8205 {
8206    rtx ops[3];
8207    ops[0] = operands[0];
8208    ops[1] = operands[1];
8209    ops[2] = operands[2];
8210    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8211    return "";
8212 }
8213   [(set_attr "length" "8")])
8214
8215 ;;
8216 ;; [vstrbq_p_s vstrbq_p_u]
8217 ;;
8218 (define_insn "mve_vstrbq_p_<supf><mode>"
8219   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8220         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
8221                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8222          VSTRBQ))
8223   ]
8224   "TARGET_HAVE_MVE"
8225 {
8226    rtx ops[2];
8227    int regno = REGNO (operands[1]);
8228    ops[1] = gen_rtx_REG (TImode, regno);
8229    ops[0]  = operands[0];
8230    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q1, %E0",ops);
8231    return "";
8232 }
8233   [(set_attr "length" "8")])
8234
8235 ;;
8236 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
8237 ;;
8238 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
8239   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8240         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8241                        (match_operand:MVE_2 2 "s_register_operand" "w")
8242                        (match_operand:HI 3 "vpr_register_operand" "Up")]
8243          VLDRBGOQ))
8244   ]
8245   "TARGET_HAVE_MVE"
8246 {
8247    rtx ops[4];
8248    ops[0] = operands[0];
8249    ops[1] = operands[1];
8250    ops[2] = operands[2];
8251    ops[3] = operands[3];
8252    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8253      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
8254    else
8255      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8256    return "";
8257 }
8258   [(set_attr "length" "8")])
8259
8260 ;;
8261 ;; [vldrbq_z_s vldrbq_z_u]
8262 ;;
8263 (define_insn "mve_vldrbq_z_<supf><mode>"
8264   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8265         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8266                        (match_operand:HI 2 "vpr_register_operand" "Up")]
8267          VLDRBQ))
8268   ]
8269   "TARGET_HAVE_MVE"
8270 {
8271    rtx ops[2];
8272    int regno = REGNO (operands[0]);
8273    ops[0] = gen_rtx_REG (TImode, regno);
8274    ops[1]  = operands[1];
8275    output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
8276    return "";
8277 }
8278   [(set_attr "length" "8")])
8279
8280 ;;
8281 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
8282 ;;
8283 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
8284   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8285         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8286                       (match_operand:SI 2 "immediate_operand" "i")
8287                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8288          VLDRWGBQ))
8289   ]
8290   "TARGET_HAVE_MVE"
8291 {
8292    rtx ops[3];
8293    ops[0] = operands[0];
8294    ops[1] = operands[1];
8295    ops[2] = operands[2];
8296    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8297    return "";
8298 }
8299   [(set_attr "length" "8")])
8300
8301 ;;
8302 ;; [vldrhq_f]
8303 ;;
8304 (define_insn "mve_vldrhq_fv8hf"
8305   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8306         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")]
8307          VLDRHQ_F))
8308   ]
8309   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8310 {
8311    rtx ops[2];
8312    int regno = REGNO (operands[0]);
8313    ops[0] = gen_rtx_REG (TImode, regno);
8314    ops[1]  = operands[1];
8315    output_asm_insn ("vldrh.f16\t%q0, %E1",ops);
8316    return "";
8317 }
8318   [(set_attr "length" "4")])
8319
8320 ;;
8321 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
8322 ;;
8323 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
8324   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8325         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8326                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8327         VLDRHGOQ))
8328   ]
8329   "TARGET_HAVE_MVE"
8330 {
8331    rtx ops[3];
8332    ops[0] = operands[0];
8333    ops[1] = operands[1];
8334    ops[2] = operands[2];
8335    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8336      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
8337    else
8338      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8339    return "";
8340 }
8341   [(set_attr "length" "4")])
8342
8343 ;;
8344 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
8345 ;;
8346 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
8347   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8348         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8349                        (match_operand:MVE_6 2 "s_register_operand" "w")
8350                        (match_operand:HI 3 "vpr_register_operand" "Up")
8351         ]VLDRHGOQ))
8352   ]
8353   "TARGET_HAVE_MVE"
8354 {
8355    rtx ops[4];
8356    ops[0] = operands[0];
8357    ops[1] = operands[1];
8358    ops[2] = operands[2];
8359    ops[3] = operands[3];
8360    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8361      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
8362    else
8363      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8364    return "";
8365 }
8366  [(set_attr "length" "8")])
8367
8368 ;;
8369 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
8370 ;;
8371 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
8372   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8373         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8374                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8375         VLDRHGSOQ))
8376   ]
8377   "TARGET_HAVE_MVE"
8378 {
8379    rtx ops[3];
8380    ops[0] = operands[0];
8381    ops[1] = operands[1];
8382    ops[2] = operands[2];
8383       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8384      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8385    else
8386      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8387    return "";
8388 }
8389   [(set_attr "length" "4")])
8390
8391 ;;
8392 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
8393 ;;
8394 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
8395   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8396         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8397                        (match_operand:MVE_6 2 "s_register_operand" "w")
8398                        (match_operand:HI 3 "vpr_register_operand" "Up")
8399         ]VLDRHGSOQ))
8400   ]
8401   "TARGET_HAVE_MVE"
8402 {
8403    rtx ops[4];
8404    ops[0] = operands[0];
8405    ops[1] = operands[1];
8406    ops[2] = operands[2];
8407    ops[3] = operands[3];
8408    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8409      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8410    else
8411      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8412    return "";
8413 }
8414   [(set_attr "length" "8")])
8415
8416 ;;
8417 ;;
8418 ;; [vldrhq_s, vldrhq_u]
8419 ;;
8420 (define_insn "mve_vldrhq_<supf><mode>"
8421   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8422         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")]
8423          VLDRHQ))
8424   ]
8425   "TARGET_HAVE_MVE"
8426 {
8427    rtx ops[2];
8428    int regno = REGNO (operands[0]);
8429    ops[0] = gen_rtx_REG (TImode, regno);
8430    ops[1]  = operands[1];
8431    output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
8432    return "";
8433 }
8434   [(set_attr "length" "4")])
8435
8436 ;;
8437 ;; [vldrhq_z_f]
8438 ;;
8439 (define_insn "mve_vldrhq_z_fv8hf"
8440   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8441         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8442         (match_operand:HI 2 "vpr_register_operand" "Up")]
8443          VLDRHQ_F))
8444   ]
8445   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8446 {
8447    rtx ops[2];
8448    int regno = REGNO (operands[0]);
8449    ops[0] = gen_rtx_REG (TImode, regno);
8450    ops[1]  = operands[1];
8451    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, %E1",ops);
8452    return "";
8453 }
8454   [(set_attr "length" "8")])
8455
8456 ;;
8457 ;; [vldrhq_z_s vldrhq_z_u]
8458 ;;
8459 (define_insn "mve_vldrhq_z_<supf><mode>"
8460   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8461         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8462         (match_operand:HI 2 "vpr_register_operand" "Up")]
8463          VLDRHQ))
8464   ]
8465   "TARGET_HAVE_MVE"
8466 {
8467    rtx ops[2];
8468    int regno = REGNO (operands[0]);
8469    ops[0] = gen_rtx_REG (TImode, regno);
8470    ops[1]  = operands[1];
8471    output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8472    return "";
8473 }
8474   [(set_attr "length" "8")])
8475
8476 ;;
8477 ;; [vldrwq_f]
8478 ;;
8479 (define_insn "mve_vldrwq_fv4sf"
8480   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8481         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")]
8482          VLDRWQ_F))
8483   ]
8484   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8485 {
8486    rtx ops[2];
8487    int regno = REGNO (operands[0]);
8488    ops[0] = gen_rtx_REG (TImode, regno);
8489    ops[1]  = operands[1];
8490    output_asm_insn ("vldrw.f32\t%q0, %E1",ops);
8491    return "";
8492 }
8493   [(set_attr "length" "4")])
8494
8495 ;;
8496 ;; [vldrwq_s vldrwq_u]
8497 ;;
8498 (define_insn "mve_vldrwq_<supf>v4si"
8499   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8500         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")]
8501          VLDRWQ))
8502   ]
8503   "TARGET_HAVE_MVE"
8504 {
8505    rtx ops[2];
8506    int regno = REGNO (operands[0]);
8507    ops[0] = gen_rtx_REG (TImode, regno);
8508    ops[1]  = operands[1];
8509    output_asm_insn ("vldrw.<supf>32\t%q0, %E1",ops);
8510    return "";
8511 }
8512   [(set_attr "length" "4")])
8513
8514 ;;
8515 ;; [vldrwq_z_f]
8516 ;;
8517 (define_insn "mve_vldrwq_z_fv4sf"
8518   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8519         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8520         (match_operand:HI 2 "vpr_register_operand" "Up")]
8521          VLDRWQ_F))
8522   ]
8523   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8524 {
8525    rtx ops[2];
8526    int regno = REGNO (operands[0]);
8527    ops[0] = gen_rtx_REG (TImode, regno);
8528    ops[1]  = operands[1];
8529    output_asm_insn ("vpst\n\tvldrwt.f32\t%q0, %E1",ops);
8530    return "";
8531 }
8532   [(set_attr "length" "8")])
8533
8534 ;;
8535 ;; [vldrwq_z_s vldrwq_z_u]
8536 ;;
8537 (define_insn "mve_vldrwq_z_<supf>v4si"
8538   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8539         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8540         (match_operand:HI 2 "vpr_register_operand" "Up")]
8541          VLDRWQ))
8542   ]
8543   "TARGET_HAVE_MVE"
8544 {
8545    rtx ops[2];
8546    int regno = REGNO (operands[0]);
8547    ops[0] = gen_rtx_REG (TImode, regno);
8548    ops[1]  = operands[1];
8549    output_asm_insn ("vpst\n\tvldrwt.<supf>32\t%q0, %E1",ops);
8550    return "";
8551 }
8552   [(set_attr "length" "8")])
8553
8554 (define_expand "mve_vld1q_f<mode>"
8555   [(match_operand:MVE_0 0 "s_register_operand")
8556    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "memory_operand")] VLD1Q_F)
8557   ]
8558   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8559 {
8560   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8561   DONE;
8562 })
8563
8564 (define_expand "mve_vld1q_<supf><mode>"
8565   [(match_operand:MVE_2 0 "s_register_operand")
8566    (unspec:MVE_2 [(match_operand:MVE_2 1 "memory_operand")] VLD1Q)
8567   ]
8568   "TARGET_HAVE_MVE"
8569 {
8570   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8571   DONE;
8572 })
8573
8574 ;;
8575 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8576 ;;
8577 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8578   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8579         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8580                       (match_operand:SI 2 "immediate_operand" "i")]
8581          VLDRDGBQ))
8582   ]
8583   "TARGET_HAVE_MVE"
8584 {
8585    rtx ops[3];
8586    ops[0] = operands[0];
8587    ops[1] = operands[1];
8588    ops[2] = operands[2];
8589    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8590    return "";
8591 }
8592   [(set_attr "length" "4")])
8593
8594 ;;
8595 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8596 ;;
8597 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8598   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8599         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8600                       (match_operand:SI 2 "immediate_operand" "i")
8601                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8602          VLDRDGBQ))
8603   ]
8604   "TARGET_HAVE_MVE"
8605 {
8606    rtx ops[3];
8607    ops[0] = operands[0];
8608    ops[1] = operands[1];
8609    ops[2] = operands[2];
8610    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8611    return "";
8612 }
8613   [(set_attr "length" "8")])
8614
8615 ;;
8616 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8617 ;;
8618 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8619  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8620        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8621                      (match_operand:V2DI 2 "s_register_operand" "w")]
8622         VLDRDGOQ))
8623  ]
8624  "TARGET_HAVE_MVE"
8625 {
8626   rtx ops[3];
8627   ops[0] = operands[0];
8628   ops[1] = operands[1];
8629   ops[2] = operands[2];
8630   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8631   return "";
8632 }
8633  [(set_attr "length" "4")])
8634
8635 ;;
8636 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8637 ;;
8638 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8639  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8640        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8641                      (match_operand:V2DI 2 "s_register_operand" "w")
8642                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8643         VLDRDGOQ))
8644  ]
8645  "TARGET_HAVE_MVE"
8646 {
8647   rtx ops[3];
8648   ops[0] = operands[0];
8649   ops[1] = operands[1];
8650   ops[2] = operands[2];
8651   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8652   return "";
8653 }
8654  [(set_attr "length" "8")])
8655
8656 ;;
8657 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8658 ;;
8659 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8660   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8661         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8662                       (match_operand:V2DI 2 "s_register_operand" "w")]
8663          VLDRDGSOQ))
8664   ]
8665   "TARGET_HAVE_MVE"
8666 {
8667    rtx ops[3];
8668    ops[0] = operands[0];
8669    ops[1] = operands[1];
8670    ops[2] = operands[2];
8671    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8672    return "";
8673 }
8674   [(set_attr "length" "4")])
8675
8676 ;;
8677 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8678 ;;
8679 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8680   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8681         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8682                       (match_operand:V2DI 2 "s_register_operand" "w")
8683                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8684          VLDRDGSOQ))
8685   ]
8686   "TARGET_HAVE_MVE"
8687 {
8688    rtx ops[3];
8689    ops[0] = operands[0];
8690    ops[1] = operands[1];
8691    ops[2] = operands[2];
8692    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8693    return "";
8694 }
8695   [(set_attr "length" "8")])
8696
8697 ;;
8698 ;; [vldrhq_gather_offset_f]
8699 ;;
8700 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8701   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8702         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8703                       (match_operand:V8HI 2 "s_register_operand" "w")]
8704          VLDRHQGO_F))
8705   ]
8706   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8707 {
8708    rtx ops[3];
8709    ops[0] = operands[0];
8710    ops[1] = operands[1];
8711    ops[2] = operands[2];
8712    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8713    return "";
8714 }
8715   [(set_attr "length" "4")])
8716
8717 ;;
8718 ;; [vldrhq_gather_offset_z_f]
8719 ;;
8720 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8721   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8722         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8723                       (match_operand:V8HI 2 "s_register_operand" "w")
8724                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8725          VLDRHQGO_F))
8726   ]
8727   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8728 {
8729    rtx ops[4];
8730    ops[0] = operands[0];
8731    ops[1] = operands[1];
8732    ops[2] = operands[2];
8733    ops[3] = operands[3];
8734    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8735    return "";
8736 }
8737   [(set_attr "length" "8")])
8738
8739 ;;
8740 ;; [vldrhq_gather_shifted_offset_f]
8741 ;;
8742 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8743   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8744         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8745                       (match_operand:V8HI 2 "s_register_operand" "w")]
8746          VLDRHQGSO_F))
8747   ]
8748   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8749 {
8750    rtx ops[3];
8751    ops[0] = operands[0];
8752    ops[1] = operands[1];
8753    ops[2] = operands[2];
8754    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8755    return "";
8756 }
8757   [(set_attr "length" "4")])
8758
8759 ;;
8760 ;; [vldrhq_gather_shifted_offset_z_f]
8761 ;;
8762 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8763   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8764         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8765                       (match_operand:V8HI 2 "s_register_operand" "w")
8766                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8767          VLDRHQGSO_F))
8768   ]
8769   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8770 {
8771    rtx ops[4];
8772    ops[0] = operands[0];
8773    ops[1] = operands[1];
8774    ops[2] = operands[2];
8775    ops[3] = operands[3];
8776    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8777    return "";
8778 }
8779   [(set_attr "length" "8")])
8780
8781 ;;
8782 ;; [vldrwq_gather_base_f]
8783 ;;
8784 (define_insn "mve_vldrwq_gather_base_fv4sf"
8785   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8786         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8787                       (match_operand:SI 2 "immediate_operand" "i")]
8788          VLDRWQGB_F))
8789   ]
8790   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8791 {
8792    rtx ops[3];
8793    ops[0] = operands[0];
8794    ops[1] = operands[1];
8795    ops[2] = operands[2];
8796    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8797    return "";
8798 }
8799   [(set_attr "length" "4")])
8800
8801 ;;
8802 ;; [vldrwq_gather_base_z_f]
8803 ;;
8804 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8805   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8806         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8807                       (match_operand:SI 2 "immediate_operand" "i")
8808                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8809          VLDRWQGB_F))
8810   ]
8811   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8812 {
8813    rtx ops[3];
8814    ops[0] = operands[0];
8815    ops[1] = operands[1];
8816    ops[2] = operands[2];
8817    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8818    return "";
8819 }
8820   [(set_attr "length" "8")])
8821
8822 ;;
8823 ;; [vldrwq_gather_offset_f]
8824 ;;
8825 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8826   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8827         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8828                        (match_operand:V4SI 2 "s_register_operand" "w")]
8829          VLDRWQGO_F))
8830   ]
8831   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8832 {
8833    rtx ops[3];
8834    ops[0] = operands[0];
8835    ops[1] = operands[1];
8836    ops[2] = operands[2];
8837    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8838    return "";
8839 }
8840   [(set_attr "length" "4")])
8841
8842 ;;
8843 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8844 ;;
8845 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8846   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8847         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8848                        (match_operand:V4SI 2 "s_register_operand" "w")]
8849          VLDRWGOQ))
8850   ]
8851   "TARGET_HAVE_MVE"
8852 {
8853    rtx ops[3];
8854    ops[0] = operands[0];
8855    ops[1] = operands[1];
8856    ops[2] = operands[2];
8857    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8858    return "";
8859 }
8860   [(set_attr "length" "4")])
8861
8862 ;;
8863 ;; [vldrwq_gather_offset_z_f]
8864 ;;
8865 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8866   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8867         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8868                       (match_operand:V4SI 2 "s_register_operand" "w")
8869                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8870          VLDRWQGO_F))
8871   ]
8872   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8873 {
8874    rtx ops[4];
8875    ops[0] = operands[0];
8876    ops[1] = operands[1];
8877    ops[2] = operands[2];
8878    ops[3] = operands[3];
8879    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8880    return "";
8881 }
8882   [(set_attr "length" "8")])
8883
8884 ;;
8885 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8886 ;;
8887 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8888   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8889         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8890                       (match_operand:V4SI 2 "s_register_operand" "w")
8891                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8892          VLDRWGOQ))
8893   ]
8894   "TARGET_HAVE_MVE"
8895 {
8896    rtx ops[4];
8897    ops[0] = operands[0];
8898    ops[1] = operands[1];
8899    ops[2] = operands[2];
8900    ops[3] = operands[3];
8901    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8902    return "";
8903 }
8904   [(set_attr "length" "8")])
8905
8906 ;;
8907 ;; [vldrwq_gather_shifted_offset_f]
8908 ;;
8909 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8910   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8911         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8912                       (match_operand:V4SI 2 "s_register_operand" "w")]
8913          VLDRWQGSO_F))
8914   ]
8915   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8916 {
8917    rtx ops[3];
8918    ops[0] = operands[0];
8919    ops[1] = operands[1];
8920    ops[2] = operands[2];
8921    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8922    return "";
8923 }
8924   [(set_attr "length" "4")])
8925
8926 ;;
8927 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8928 ;;
8929 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8930   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8931         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8932                       (match_operand:V4SI 2 "s_register_operand" "w")]
8933          VLDRWGSOQ))
8934   ]
8935   "TARGET_HAVE_MVE"
8936 {
8937    rtx ops[3];
8938    ops[0] = operands[0];
8939    ops[1] = operands[1];
8940    ops[2] = operands[2];
8941    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8942    return "";
8943 }
8944   [(set_attr "length" "4")])
8945
8946 ;;
8947 ;; [vldrwq_gather_shifted_offset_z_f]
8948 ;;
8949 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8950   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8951         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8952                       (match_operand:V4SI 2 "s_register_operand" "w")
8953                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8954          VLDRWQGSO_F))
8955   ]
8956   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8957 {
8958    rtx ops[4];
8959    ops[0] = operands[0];
8960    ops[1] = operands[1];
8961    ops[2] = operands[2];
8962    ops[3] = operands[3];
8963    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8964    return "";
8965 }
8966   [(set_attr "length" "8")])
8967
8968 ;;
8969 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8970 ;;
8971 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8972   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8973         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8974                       (match_operand:V4SI 2 "s_register_operand" "w")
8975                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8976          VLDRWGSOQ))
8977   ]
8978   "TARGET_HAVE_MVE"
8979 {
8980    rtx ops[4];
8981    ops[0] = operands[0];
8982    ops[1] = operands[1];
8983    ops[2] = operands[2];
8984    ops[3] = operands[3];
8985    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8986    return "";
8987 }
8988   [(set_attr "length" "8")])
8989
8990 ;;
8991 ;; [vstrhq_f]
8992 ;;
8993 (define_insn "mve_vstrhq_fv8hf"
8994   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
8995         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8996          VSTRHQ_F))
8997   ]
8998   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8999 {
9000    rtx ops[2];
9001    int regno = REGNO (operands[1]);
9002    ops[1] = gen_rtx_REG (TImode, regno);
9003    ops[0]  = operands[0];
9004    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
9005    return "";
9006 }
9007   [(set_attr "length" "4")])
9008
9009 ;;
9010 ;; [vstrhq_p_f]
9011 ;;
9012 (define_insn "mve_vstrhq_p_fv8hf"
9013   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9014         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
9015                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9016          VSTRHQ_F))
9017   ]
9018   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9019 {
9020    rtx ops[2];
9021    int regno = REGNO (operands[1]);
9022    ops[1] = gen_rtx_REG (TImode, regno);
9023    ops[0]  = operands[0];
9024    output_asm_insn ("vpst\n\tvstrht.16\t%q1, %E0",ops);
9025    return "";
9026 }
9027   [(set_attr "length" "8")])
9028
9029 ;;
9030 ;; [vstrhq_p_s vstrhq_p_u]
9031 ;;
9032 (define_insn "mve_vstrhq_p_<supf><mode>"
9033   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9034         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
9035                               (match_operand:HI 2 "vpr_register_operand" "Up")]
9036          VSTRHQ))
9037   ]
9038   "TARGET_HAVE_MVE"
9039 {
9040    rtx ops[2];
9041    int regno = REGNO (operands[1]);
9042    ops[1] = gen_rtx_REG (TImode, regno);
9043    ops[0]  = operands[0];
9044    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q1, %E0",ops);
9045    return "";
9046 }
9047   [(set_attr "length" "8")])
9048
9049 ;;
9050 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
9051 ;;
9052 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>"
9053   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9054         (unspec:<MVE_H_ELEM>
9055                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9056                  (match_operand:MVE_6 2 "s_register_operand" "w")
9057                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9058          VSTRHSOQ))
9059   ]
9060   "TARGET_HAVE_MVE"
9061 {
9062    rtx ops[3];
9063    ops[0] = operands[0];
9064    ops[1] = operands[1];
9065    ops[2] = operands[2];
9066    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9067    return "";
9068 }
9069   [(set_attr "length" "8")])
9070
9071 ;;
9072 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
9073 ;;
9074 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>"
9075   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9076         (unspec:<MVE_H_ELEM>
9077                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9078                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9079          VSTRHSOQ))
9080   ]
9081   "TARGET_HAVE_MVE"
9082 {
9083    rtx ops[3];
9084    ops[0] = operands[0];
9085    ops[1] = operands[1];
9086    ops[2] = operands[2];
9087    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9088    return "";
9089 }
9090   [(set_attr "length" "4")])
9091
9092 ;;
9093 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
9094 ;;
9095 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
9096   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9097         (unspec:<MVE_H_ELEM>
9098                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9099                  (match_operand:MVE_6 2 "s_register_operand" "w")
9100                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9101          VSTRHSSOQ))
9102   ]
9103   "TARGET_HAVE_MVE"
9104 {
9105    rtx ops[3];
9106    ops[0] = operands[0];
9107    ops[1] = operands[1];
9108    ops[2] = operands[2];
9109    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9110    return "";
9111 }
9112   [(set_attr "length" "8")])
9113
9114 ;;
9115 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
9116 ;;
9117 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
9118   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9119         (unspec:<MVE_H_ELEM>
9120                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9121                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9122          VSTRHSSOQ))
9123   ]
9124   "TARGET_HAVE_MVE"
9125 {
9126    rtx ops[3];
9127    ops[0] = operands[0];
9128    ops[1] = operands[1];
9129    ops[2] = operands[2];
9130    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9131    return "";
9132 }
9133   [(set_attr "length" "4")])
9134
9135 ;;
9136 ;; [vstrhq_s, vstrhq_u]
9137 ;;
9138 (define_insn "mve_vstrhq_<supf><mode>"
9139   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9140         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
9141          VSTRHQ))
9142   ]
9143   "TARGET_HAVE_MVE"
9144 {
9145    rtx ops[2];
9146    int regno = REGNO (operands[1]);
9147    ops[1] = gen_rtx_REG (TImode, regno);
9148    ops[0]  = operands[0];
9149    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
9150    return "";
9151 }
9152   [(set_attr "length" "4")])
9153
9154 ;;
9155 ;; [vstrwq_f]
9156 ;;
9157 (define_insn "mve_vstrwq_fv4sf"
9158   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9159         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
9160          VSTRWQ_F))
9161   ]
9162   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9163 {
9164    rtx ops[2];
9165    int regno = REGNO (operands[1]);
9166    ops[1] = gen_rtx_REG (TImode, regno);
9167    ops[0]  = operands[0];
9168    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9169    return "";
9170 }
9171   [(set_attr "length" "4")])
9172
9173 ;;
9174 ;; [vstrwq_p_f]
9175 ;;
9176 (define_insn "mve_vstrwq_p_fv4sf"
9177   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9178         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
9179                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9180          VSTRWQ_F))
9181   ]
9182   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9183 {
9184    rtx ops[2];
9185    int regno = REGNO (operands[1]);
9186    ops[1] = gen_rtx_REG (TImode, regno);
9187    ops[0]  = operands[0];
9188    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9189    return "";
9190 }
9191   [(set_attr "length" "8")])
9192
9193 ;;
9194 ;; [vstrwq_p_s vstrwq_p_u]
9195 ;;
9196 (define_insn "mve_vstrwq_p_<supf>v4si"
9197   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9198         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9199                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9200          VSTRWQ))
9201   ]
9202   "TARGET_HAVE_MVE"
9203 {
9204    rtx ops[2];
9205    int regno = REGNO (operands[1]);
9206    ops[1] = gen_rtx_REG (TImode, regno);
9207    ops[0]  = operands[0];
9208    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9209    return "";
9210 }
9211   [(set_attr "length" "8")])
9212
9213 ;;
9214 ;; [vstrwq_s vstrwq_u]
9215 ;;
9216 (define_insn "mve_vstrwq_<supf>v4si"
9217   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9218         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
9219          VSTRWQ))
9220   ]
9221   "TARGET_HAVE_MVE"
9222 {
9223    rtx ops[2];
9224    int regno = REGNO (operands[1]);
9225    ops[1] = gen_rtx_REG (TImode, regno);
9226    ops[0]  = operands[0];
9227    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9228    return "";
9229 }
9230   [(set_attr "length" "4")])
9231
9232 (define_expand "mve_vst1q_f<mode>"
9233   [(match_operand:<MVE_CNVT> 0 "memory_operand")
9234    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
9235   ]
9236   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
9237 {
9238   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
9239   DONE;
9240 })
9241
9242 (define_expand "mve_vst1q_<supf><mode>"
9243   [(match_operand:MVE_2 0 "memory_operand")
9244    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
9245   ]
9246   "TARGET_HAVE_MVE"
9247 {
9248   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
9249   DONE;
9250 })
9251
9252 ;;
9253 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
9254 ;;
9255 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
9256   [(set (mem:BLK (scratch))
9257         (unspec:BLK
9258                 [(match_operand:V2DI 0 "s_register_operand" "w")
9259                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9260                  (match_operand:V2DI 2 "s_register_operand" "w")
9261                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9262          VSTRDSBQ))
9263   ]
9264   "TARGET_HAVE_MVE"
9265 {
9266    rtx ops[3];
9267    ops[0] = operands[0];
9268    ops[1] = operands[1];
9269    ops[2] = operands[2];
9270    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
9271    return "";
9272 }
9273   [(set_attr "length" "8")])
9274
9275 ;;
9276 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
9277 ;;
9278 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
9279   [(set (mem:BLK (scratch))
9280         (unspec:BLK
9281                 [(match_operand:V2DI 0 "s_register_operand" "=w")
9282                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9283                  (match_operand:V2DI 2 "s_register_operand" "w")]
9284          VSTRDSBQ))
9285   ]
9286   "TARGET_HAVE_MVE"
9287 {
9288    rtx ops[3];
9289    ops[0] = operands[0];
9290    ops[1] = operands[1];
9291    ops[2] = operands[2];
9292    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
9293    return "";
9294 }
9295   [(set_attr "length" "4")])
9296
9297 ;;
9298 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
9299 ;;
9300 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di"
9301   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9302         (unspec:V2DI
9303                 [(match_operand:V2DI 1 "s_register_operand" "w")
9304                  (match_operand:V2DI 2 "s_register_operand" "w")
9305                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9306          VSTRDSOQ))
9307   ]
9308   "TARGET_HAVE_MVE"
9309 {
9310    rtx ops[3];
9311    ops[0] = operands[0];
9312    ops[1] = operands[1];
9313    ops[2] = operands[2];
9314    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1]",ops);
9315    return "";
9316 }
9317   [(set_attr "length" "8")])
9318
9319 ;;
9320 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
9321 ;;
9322 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di"
9323   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9324         (unspec:V2DI
9325                 [(match_operand:V2DI 1 "s_register_operand" "w")
9326                  (match_operand:V2DI 2 "s_register_operand" "w")]
9327          VSTRDSOQ))
9328   ]
9329   "TARGET_HAVE_MVE"
9330 {
9331    rtx ops[3];
9332    ops[0] = operands[0];
9333    ops[1] = operands[1];
9334    ops[2] = operands[2];
9335    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1]",ops);
9336    return "";
9337 }
9338   [(set_attr "length" "4")])
9339
9340 ;;
9341 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
9342 ;;
9343 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
9344   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9345         (unspec:V2DI
9346                 [(match_operand:V2DI 1 "s_register_operand" "w")
9347                  (match_operand:V2DI 2 "s_register_operand" "w")
9348                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9349          VSTRDSSOQ))
9350   ]
9351   "TARGET_HAVE_MVE"
9352 {
9353    rtx ops[3];
9354    ops[0] = operands[0];
9355    ops[1] = operands[1];
9356    ops[2] = operands[2];
9357    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9358    return "";
9359 }
9360   [(set_attr "length" "8")])
9361
9362 ;;
9363 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
9364 ;;
9365 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
9366   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9367         (unspec:V2DI
9368                 [(match_operand:V2DI 1 "s_register_operand" "w")
9369                  (match_operand:V2DI 2 "s_register_operand" "w")]
9370          VSTRDSSOQ))
9371   ]
9372   "TARGET_HAVE_MVE"
9373 {
9374    rtx ops[3];
9375    ops[0] = operands[0];
9376    ops[1] = operands[1];
9377    ops[2] = operands[2];
9378    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9379    return "";
9380 }
9381   [(set_attr "length" "4")])
9382
9383 ;;
9384 ;; [vstrhq_scatter_offset_f]
9385 ;;
9386 (define_insn "mve_vstrhq_scatter_offset_fv8hf"
9387   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9388         (unspec:V8HI
9389                 [(match_operand:V8HI 1 "s_register_operand" "w")
9390                  (match_operand:V8HF 2 "s_register_operand" "w")]
9391          VSTRHQSO_F))
9392   ]
9393   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9394 {
9395    rtx ops[3];
9396    ops[0] = operands[0];
9397    ops[1] = operands[1];
9398    ops[2] = operands[2];
9399    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1]",ops);
9400    return "";
9401 }
9402   [(set_attr "length" "4")])
9403
9404 ;;
9405 ;; [vstrhq_scatter_offset_p_f]
9406 ;;
9407 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf"
9408   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9409         (unspec:V8HI
9410                 [(match_operand:V8HI 1 "s_register_operand" "w")
9411                  (match_operand:V8HF 2 "s_register_operand" "w")
9412                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9413          VSTRHQSO_F))
9414   ]
9415   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9416 {
9417    rtx ops[3];
9418    ops[0] = operands[0];
9419    ops[1] = operands[1];
9420    ops[2] = operands[2];
9421    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1]",ops);
9422    return "";
9423 }
9424   [(set_attr "length" "8")])
9425
9426 ;;
9427 ;; [vstrhq_scatter_shifted_offset_f]
9428 ;;
9429 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf"
9430   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9431         (unspec:V8HI
9432                 [(match_operand:V8HI 1 "s_register_operand" "w")
9433                  (match_operand:V8HF 2 "s_register_operand" "w")]
9434          VSTRHQSSO_F))
9435   ]
9436   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9437 {
9438    rtx ops[3];
9439    ops[0] = operands[0];
9440    ops[1] = operands[1];
9441    ops[2] = operands[2];
9442    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9443    return "";
9444 }
9445   [(set_attr "length" "4")])
9446
9447 ;;
9448 ;; [vstrhq_scatter_shifted_offset_p_f]
9449 ;;
9450 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9451   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9452         (unspec:V8HI
9453                 [(match_operand:V8HI 1 "s_register_operand" "w")
9454                  (match_operand:V8HF 2 "s_register_operand" "w")
9455                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9456          VSTRHQSSO_F))
9457   ]
9458   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9459 {
9460    rtx ops[3];
9461    ops[0] = operands[0];
9462    ops[1] = operands[1];
9463    ops[2] = operands[2];
9464    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9465    return "";
9466 }
9467   [(set_attr "length" "8")])
9468
9469 ;;
9470 ;; [vstrwq_scatter_base_f]
9471 ;;
9472 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9473   [(set (mem:BLK (scratch))
9474         (unspec:BLK
9475                 [(match_operand:V4SI 0 "s_register_operand" "w")
9476                  (match_operand:SI 1 "immediate_operand" "i")
9477                  (match_operand:V4SF 2 "s_register_operand" "w")]
9478          VSTRWQSB_F))
9479   ]
9480   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9481 {
9482    rtx ops[3];
9483    ops[0] = operands[0];
9484    ops[1] = operands[1];
9485    ops[2] = operands[2];
9486    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9487    return "";
9488 }
9489   [(set_attr "length" "4")])
9490
9491 ;;
9492 ;; [vstrwq_scatter_base_p_f]
9493 ;;
9494 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9495   [(set (mem:BLK (scratch))
9496         (unspec:BLK
9497                 [(match_operand:V4SI 0 "s_register_operand" "w")
9498                  (match_operand:SI 1 "immediate_operand" "i")
9499                  (match_operand:V4SF 2 "s_register_operand" "w")
9500                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9501          VSTRWQSB_F))
9502   ]
9503   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9504 {
9505    rtx ops[3];
9506    ops[0] = operands[0];
9507    ops[1] = operands[1];
9508    ops[2] = operands[2];
9509    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9510    return "";
9511 }
9512   [(set_attr "length" "8")])
9513
9514 ;;
9515 ;; [vstrwq_scatter_offset_f]
9516 ;;
9517 (define_insn "mve_vstrwq_scatter_offset_fv4sf"
9518   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9519         (unspec:V4SI
9520                 [(match_operand:V4SI 1 "s_register_operand" "w")
9521                  (match_operand:V4SF 2 "s_register_operand" "w")]
9522          VSTRWQSO_F))
9523   ]
9524   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9525 {
9526    rtx ops[3];
9527    ops[0] = operands[0];
9528    ops[1] = operands[1];
9529    ops[2] = operands[2];
9530    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9531    return "";
9532 }
9533   [(set_attr "length" "4")])
9534
9535 ;;
9536 ;; [vstrwq_scatter_offset_p_f]
9537 ;;
9538 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf"
9539   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9540         (unspec:V4SI
9541                 [(match_operand:V4SI 1 "s_register_operand" "w")
9542                  (match_operand:V4SF 2 "s_register_operand" "w")
9543                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9544          VSTRWQSO_F))
9545   ]
9546   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9547 {
9548    rtx ops[3];
9549    ops[0] = operands[0];
9550    ops[1] = operands[1];
9551    ops[2] = operands[2];
9552    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9553    return "";
9554 }
9555   [(set_attr "length" "8")])
9556
9557 ;;
9558 ;; [vstrwq_scatter_offset_p_s vstrwq_scatter_offset_p_u]
9559 ;;
9560 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si"
9561   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9562         (unspec:V4SI
9563                 [(match_operand:V4SI 1 "s_register_operand" "w")
9564                  (match_operand:V4SI 2 "s_register_operand" "w")
9565                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9566          VSTRWSOQ))
9567   ]
9568   "TARGET_HAVE_MVE"
9569 {
9570    rtx ops[3];
9571    ops[0] = operands[0];
9572    ops[1] = operands[1];
9573    ops[2] = operands[2];
9574    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9575    return "";
9576 }
9577   [(set_attr "length" "8")])
9578
9579 ;;
9580 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9581 ;;
9582 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si"
9583   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9584         (unspec:V4SI
9585                 [(match_operand:V4SI 1 "s_register_operand" "w")
9586                  (match_operand:V4SI 2 "s_register_operand" "w")]
9587          VSTRWSOQ))
9588   ]
9589   "TARGET_HAVE_MVE"
9590 {
9591    rtx ops[3];
9592    ops[0] = operands[0];
9593    ops[1] = operands[1];
9594    ops[2] = operands[2];
9595    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9596    return "";
9597 }
9598   [(set_attr "length" "4")])
9599
9600 ;;
9601 ;; [vstrwq_scatter_shifted_offset_f]
9602 ;;
9603 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf"
9604   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9605         (unspec:V4SI
9606                 [(match_operand:V4SI 1 "s_register_operand" "w")
9607                  (match_operand:V4SF 2 "s_register_operand" "w")]
9608          VSTRWQSSO_F))
9609   ]
9610   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9611 {
9612    rtx ops[3];
9613    ops[0] = operands[0];
9614    ops[1] = operands[1];
9615    ops[2] = operands[2];
9616    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9617    return "";
9618 }
9619   [(set_attr "length" "4")])
9620
9621 ;;
9622 ;; [vstrwq_scatter_shifted_offset_p_f]
9623 ;;
9624 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9625   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9626         (unspec:V4SI
9627                 [(match_operand:V4SI 1 "s_register_operand" "w")
9628                  (match_operand:V4SF 2 "s_register_operand" "w")
9629                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9630          VSTRWQSSO_F))
9631   ]
9632   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9633 {
9634    rtx ops[3];
9635    ops[0] = operands[0];
9636    ops[1] = operands[1];
9637    ops[2] = operands[2];
9638    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9639    return "";
9640 }
9641   [(set_attr "length" "8")])
9642
9643 ;;
9644 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9645 ;;
9646 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9647   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9648         (unspec:V4SI
9649                 [(match_operand:V4SI 1 "s_register_operand" "w")
9650                  (match_operand:V4SI 2 "s_register_operand" "w")
9651                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9652          VSTRWSSOQ))
9653   ]
9654   "TARGET_HAVE_MVE"
9655 {
9656    rtx ops[3];
9657    ops[0] = operands[0];
9658    ops[1] = operands[1];
9659    ops[2] = operands[2];
9660    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9661    return "";
9662 }
9663   [(set_attr "length" "8")])
9664
9665 ;;
9666 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9667 ;;
9668 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9669   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9670         (unspec:V4SI
9671                 [(match_operand:V4SI 1 "s_register_operand" "w")
9672                  (match_operand:V4SI 2 "s_register_operand" "w")]
9673          VSTRWSSOQ))
9674   ]
9675   "TARGET_HAVE_MVE"
9676 {
9677    rtx ops[3];
9678    ops[0] = operands[0];
9679    ops[1] = operands[1];
9680    ops[2] = operands[2];
9681    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9682    return "";
9683 }
9684   [(set_attr "length" "4")])
9685
9686 ;;
9687 ;; [vaddq_s, vaddq_u])
9688 ;;
9689 (define_insn "mve_vaddq<mode>"
9690   [
9691    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9692         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9693                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9694   ]
9695   "TARGET_HAVE_MVE"
9696   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9697   [(set_attr "type" "mve_move")
9698 ])
9699
9700 ;;
9701 ;; [vaddq_f])
9702 ;;
9703 (define_insn "mve_vaddq_f<mode>"
9704   [
9705    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9706         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9707                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9708   ]
9709   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9710   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9711   [(set_attr "type" "mve_move")
9712 ])
9713
9714 ;;
9715 ;; [vidupq_n_u])
9716 ;;
9717 (define_expand "mve_vidupq_n_u<mode>"
9718  [(match_operand:MVE_2 0 "s_register_operand")
9719   (match_operand:SI 1 "s_register_operand")
9720   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9721  "TARGET_HAVE_MVE"
9722 {
9723   rtx temp = gen_reg_rtx (SImode);
9724   emit_move_insn (temp, operands[1]);
9725   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9726   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9727                                           operands[2], inc));
9728   DONE;
9729 })
9730
9731 ;;
9732 ;; [vidupq_u_insn])
9733 ;;
9734 (define_insn "mve_vidupq_u<mode>_insn"
9735  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9736        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9737                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9738          VIDUPQ))
9739   (set (match_operand:SI 1 "s_register_operand" "=e")
9740        (plus:SI (match_dup 2)
9741                 (match_operand:SI 4 "immediate_operand" "i")))]
9742  "TARGET_HAVE_MVE"
9743  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9744
9745 ;;
9746 ;; [vidupq_m_n_u])
9747 ;;
9748 (define_expand "mve_vidupq_m_n_u<mode>"
9749   [(match_operand:MVE_2 0 "s_register_operand")
9750    (match_operand:MVE_2 1 "s_register_operand")
9751    (match_operand:SI 2 "s_register_operand")
9752    (match_operand:SI 3 "mve_imm_selective_upto_8")
9753    (match_operand:HI 4 "vpr_register_operand")]
9754   "TARGET_HAVE_MVE"
9755 {
9756   rtx temp = gen_reg_rtx (SImode);
9757   emit_move_insn (temp, operands[2]);
9758   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9759   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9760                                              operands[2], operands[3],
9761                                              operands[4], inc));
9762   DONE;
9763 })
9764
9765 ;;
9766 ;; [vidupq_m_wb_u_insn])
9767 ;;
9768 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9769  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9770        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9771                       (match_operand:SI 3 "s_register_operand" "2")
9772                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9773                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9774         VIDUPQ_M))
9775   (set (match_operand:SI 2 "s_register_operand" "=e")
9776        (plus:SI (match_dup 3)
9777                 (match_operand:SI 6 "immediate_operand" "i")))]
9778  "TARGET_HAVE_MVE"
9779  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9780  [(set_attr "length""8")])
9781
9782 ;;
9783 ;; [vddupq_n_u])
9784 ;;
9785 (define_expand "mve_vddupq_n_u<mode>"
9786  [(match_operand:MVE_2 0 "s_register_operand")
9787   (match_operand:SI 1 "s_register_operand")
9788   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9789  "TARGET_HAVE_MVE"
9790 {
9791   rtx temp = gen_reg_rtx (SImode);
9792   emit_move_insn (temp, operands[1]);
9793   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9794   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9795                                           operands[2], inc));
9796   DONE;
9797 })
9798
9799 ;;
9800 ;; [vddupq_u_insn])
9801 ;;
9802 (define_insn "mve_vddupq_u<mode>_insn"
9803  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9804        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9805                       (match_operand:SI 3 "immediate_operand" "i")]
9806         VDDUPQ))
9807   (set (match_operand:SI 1 "s_register_operand" "=e")
9808        (minus:SI (match_dup 2)
9809                  (match_operand:SI 4 "immediate_operand" "i")))]
9810  "TARGET_HAVE_MVE"
9811  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9812
9813 ;;
9814 ;; [vddupq_m_n_u])
9815 ;;
9816 (define_expand "mve_vddupq_m_n_u<mode>"
9817   [(match_operand:MVE_2 0 "s_register_operand")
9818    (match_operand:MVE_2 1 "s_register_operand")
9819    (match_operand:SI 2 "s_register_operand")
9820    (match_operand:SI 3 "mve_imm_selective_upto_8")
9821    (match_operand:HI 4 "vpr_register_operand")]
9822   "TARGET_HAVE_MVE"
9823 {
9824   rtx temp = gen_reg_rtx (SImode);
9825   emit_move_insn (temp, operands[2]);
9826   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9827   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9828                                              operands[2], operands[3],
9829                                              operands[4], inc));
9830   DONE;
9831 })
9832
9833 ;;
9834 ;; [vddupq_m_wb_u_insn])
9835 ;;
9836 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9837  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9838        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9839                       (match_operand:SI 3 "s_register_operand" "2")
9840                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9841                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9842         VDDUPQ_M))
9843   (set (match_operand:SI 2 "s_register_operand" "=e")
9844        (minus:SI (match_dup 3)
9845                  (match_operand:SI 6 "immediate_operand" "i")))]
9846  "TARGET_HAVE_MVE"
9847  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9848  [(set_attr "length""8")])
9849
9850 ;;
9851 ;; [vdwdupq_n_u])
9852 ;;
9853 (define_expand "mve_vdwdupq_n_u<mode>"
9854  [(match_operand:MVE_2 0 "s_register_operand")
9855   (match_operand:SI 1 "s_register_operand")
9856   (match_operand:DI 2 "s_register_operand")
9857   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9858  "TARGET_HAVE_MVE"
9859 {
9860   rtx ignore_wb = gen_reg_rtx (SImode);
9861   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9862                                               operands[1], operands[2],
9863                                               operands[3]));
9864   DONE;
9865 })
9866
9867 ;;
9868 ;; [vdwdupq_wb_u])
9869 ;;
9870 (define_expand "mve_vdwdupq_wb_u<mode>"
9871  [(match_operand:SI 0 "s_register_operand")
9872   (match_operand:SI 1 "s_register_operand")
9873   (match_operand:DI 2 "s_register_operand")
9874   (match_operand:SI 3 "mve_imm_selective_upto_8")
9875   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9876  "TARGET_HAVE_MVE"
9877 {
9878   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9879   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9880                                               operands[1], operands[2],
9881                                               operands[3]));
9882   DONE;
9883 })
9884
9885 ;;
9886 ;; [vdwdupq_wb_u_insn])
9887 ;;
9888 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9889   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9890         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9891                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9892                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9893          VDWDUPQ))
9894    (set (match_operand:SI 1 "s_register_operand" "=e")
9895         (unspec:SI [(match_dup 2)
9896                     (subreg:SI (match_dup 3) 4)
9897                     (match_dup 4)]
9898          VDWDUPQ))]
9899   "TARGET_HAVE_MVE"
9900   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9901 )
9902
9903 ;;
9904 ;; [vdwdupq_m_n_u])
9905 ;;
9906 (define_expand "mve_vdwdupq_m_n_u<mode>"
9907  [(match_operand:MVE_2 0 "s_register_operand")
9908   (match_operand:MVE_2 1 "s_register_operand")
9909   (match_operand:SI 2 "s_register_operand")
9910   (match_operand:DI 3 "s_register_operand")
9911   (match_operand:SI 4 "mve_imm_selective_upto_8")
9912   (match_operand:HI 5 "vpr_register_operand")]
9913  "TARGET_HAVE_MVE"
9914 {
9915   rtx ignore_wb = gen_reg_rtx (SImode);
9916   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9917                                                 operands[1], operands[2],
9918                                                 operands[3], operands[4],
9919                                                 operands[5]));
9920   DONE;
9921 })
9922
9923 ;;
9924 ;; [vdwdupq_m_wb_u])
9925 ;;
9926 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9927  [(match_operand:SI 0 "s_register_operand")
9928   (match_operand:MVE_2 1 "s_register_operand")
9929   (match_operand:SI 2 "s_register_operand")
9930   (match_operand:DI 3 "s_register_operand")
9931   (match_operand:SI 4 "mve_imm_selective_upto_8")
9932   (match_operand:HI 5 "vpr_register_operand")]
9933  "TARGET_HAVE_MVE"
9934 {
9935   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9936   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9937                                                 operands[1], operands[2],
9938                                                 operands[3], operands[4],
9939                                                 operands[5]));
9940   DONE;
9941 })
9942
9943 ;;
9944 ;; [vdwdupq_m_wb_u_insn])
9945 ;;
9946 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9947   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9948         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9949                        (match_operand:SI 3 "s_register_operand" "1")
9950                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9951                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9952                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9953          VDWDUPQ_M))
9954    (set (match_operand:SI 1 "s_register_operand" "=e")
9955         (unspec:SI [(match_dup 2)
9956                     (match_dup 3)
9957                     (subreg:SI (match_dup 4) 4)
9958                     (match_dup 5)
9959                     (match_dup 6)]
9960          VDWDUPQ_M))
9961   ]
9962   "TARGET_HAVE_MVE"
9963   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9964   [(set_attr "type" "mve_move")
9965    (set_attr "length""8")])
9966
9967 ;;
9968 ;; [viwdupq_n_u])
9969 ;;
9970 (define_expand "mve_viwdupq_n_u<mode>"
9971  [(match_operand:MVE_2 0 "s_register_operand")
9972   (match_operand:SI 1 "s_register_operand")
9973   (match_operand:DI 2 "s_register_operand")
9974   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9975  "TARGET_HAVE_MVE"
9976 {
9977   rtx ignore_wb = gen_reg_rtx (SImode);
9978   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9979                                               operands[1], operands[2],
9980                                               operands[3]));
9981   DONE;
9982 })
9983
9984 ;;
9985 ;; [viwdupq_wb_u])
9986 ;;
9987 (define_expand "mve_viwdupq_wb_u<mode>"
9988  [(match_operand:SI 0 "s_register_operand")
9989   (match_operand:SI 1 "s_register_operand")
9990   (match_operand:DI 2 "s_register_operand")
9991   (match_operand:SI 3 "mve_imm_selective_upto_8")
9992   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9993  "TARGET_HAVE_MVE"
9994 {
9995   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9996   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9997                                               operands[1], operands[2],
9998                                               operands[3]));
9999   DONE;
10000 })
10001
10002 ;;
10003 ;; [viwdupq_wb_u_insn])
10004 ;;
10005 (define_insn "mve_viwdupq_wb_u<mode>_insn"
10006   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10007         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10008                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
10009                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
10010          VIWDUPQ))
10011    (set (match_operand:SI 1 "s_register_operand" "=e")
10012         (unspec:SI [(match_dup 2)
10013                     (subreg:SI (match_dup 3) 4)
10014                     (match_dup 4)]
10015          VIWDUPQ))]
10016   "TARGET_HAVE_MVE"
10017   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
10018 )
10019
10020 ;;
10021 ;; [viwdupq_m_n_u])
10022 ;;
10023 (define_expand "mve_viwdupq_m_n_u<mode>"
10024  [(match_operand:MVE_2 0 "s_register_operand")
10025   (match_operand:MVE_2 1 "s_register_operand")
10026   (match_operand:SI 2 "s_register_operand")
10027   (match_operand:DI 3 "s_register_operand")
10028   (match_operand:SI 4 "mve_imm_selective_upto_8")
10029   (match_operand:HI 5 "vpr_register_operand")]
10030  "TARGET_HAVE_MVE"
10031 {
10032   rtx ignore_wb = gen_reg_rtx (SImode);
10033   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
10034                                                 operands[1], operands[2],
10035                                                 operands[3], operands[4],
10036                                                 operands[5]));
10037   DONE;
10038 })
10039
10040 ;;
10041 ;; [viwdupq_m_wb_u])
10042 ;;
10043 (define_expand "mve_viwdupq_m_wb_u<mode>"
10044  [(match_operand:SI 0 "s_register_operand")
10045   (match_operand:MVE_2 1 "s_register_operand")
10046   (match_operand:SI 2 "s_register_operand")
10047   (match_operand:DI 3 "s_register_operand")
10048   (match_operand:SI 4 "mve_imm_selective_upto_8")
10049   (match_operand:HI 5 "vpr_register_operand")]
10050  "TARGET_HAVE_MVE"
10051 {
10052   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10053   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
10054                                                 operands[1], operands[2],
10055                                                 operands[3], operands[4],
10056                                                 operands[5]));
10057   DONE;
10058 })
10059
10060 ;;
10061 ;; [viwdupq_m_wb_u_insn])
10062 ;;
10063 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
10064   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10065         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10066                        (match_operand:SI 3 "s_register_operand" "1")
10067                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
10068                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
10069                        (match_operand:HI 6 "vpr_register_operand" "Up")]
10070          VIWDUPQ_M))
10071    (set (match_operand:SI 1 "s_register_operand" "=e")
10072         (unspec:SI [(match_dup 2)
10073                     (match_dup 3)
10074                     (subreg:SI (match_dup 4) 4)
10075                     (match_dup 5)
10076                     (match_dup 6)]
10077          VIWDUPQ_M))
10078   ]
10079   "TARGET_HAVE_MVE"
10080   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
10081   [(set_attr "type" "mve_move")
10082    (set_attr "length""8")])
10083
10084 (define_expand "mve_vstrwq_scatter_base_wb_<supf>v4si"
10085   [(match_operand:V4SI 0 "s_register_operand" "=w")
10086    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10087    (match_operand:V4SI 2 "s_register_operand" "w")
10088    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10089   "TARGET_HAVE_MVE"
10090 {
10091   rtx ignore_wb = gen_reg_rtx (V4SImode);
10092   emit_insn (
10093   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (ignore_wb, operands[0],
10094                                                   operands[1], operands[2]));
10095   DONE;
10096 })
10097
10098 (define_expand "mve_vstrwq_scatter_base_wb_add_<supf>v4si"
10099   [(match_operand:V4SI 0 "s_register_operand" "=w")
10100    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10101    (match_operand:V4SI 2 "s_register_operand" "0")
10102    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10103   "TARGET_HAVE_MVE"
10104 {
10105   rtx ignore_vec = gen_reg_rtx (V4SImode);
10106   emit_insn (
10107   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (operands[0], operands[2],
10108                                                   operands[1], ignore_vec));
10109   DONE;
10110 })
10111
10112 ;;
10113 ;; [vstrwq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10114 ;;
10115 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si_insn"
10116   [(set (mem:BLK (scratch))
10117         (unspec:BLK
10118                 [(match_operand:V4SI 1 "s_register_operand" "0")
10119                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10120                  (match_operand:V4SI 3 "s_register_operand" "w")]
10121          VSTRWSBWBQ))
10122    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10123         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10124          VSTRWSBWBQ))
10125   ]
10126   "TARGET_HAVE_MVE"
10127 {
10128    rtx ops[3];
10129    ops[0] = operands[1];
10130    ops[1] = operands[2];
10131    ops[2] = operands[3];
10132    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10133    return "";
10134 }
10135   [(set_attr "length" "4")])
10136
10137 (define_expand "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
10138   [(match_operand:V4SI 0 "s_register_operand" "=w")
10139    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10140    (match_operand:V4SI 2 "s_register_operand" "w")
10141    (match_operand:HI 3 "vpr_register_operand")
10142    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10143   "TARGET_HAVE_MVE"
10144 {
10145   rtx ignore_wb = gen_reg_rtx (V4SImode);
10146   emit_insn (
10147   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (ignore_wb, operands[0],
10148                                                     operands[1], operands[2],
10149                                                     operands[3]));
10150   DONE;
10151 })
10152
10153 (define_expand "mve_vstrwq_scatter_base_wb_p_add_<supf>v4si"
10154   [(match_operand:V4SI 0 "s_register_operand" "=w")
10155    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10156    (match_operand:V4SI 2 "s_register_operand" "0")
10157    (match_operand:HI 3 "vpr_register_operand")
10158    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10159   "TARGET_HAVE_MVE"
10160 {
10161   rtx ignore_vec = gen_reg_rtx (V4SImode);
10162   emit_insn (
10163   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (operands[0], operands[2],
10164                                                     operands[1], ignore_vec,
10165                                                     operands[3]));
10166   DONE;
10167 })
10168
10169 ;;
10170 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
10171 ;;
10172 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn"
10173  [(set (mem:BLK (scratch))
10174        (unspec:BLK
10175                 [(match_operand:V4SI 1 "s_register_operand" "0")
10176                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10177                  (match_operand:V4SI 3 "s_register_operand" "w")
10178                  (match_operand:HI 4 "vpr_register_operand")]
10179         VSTRWSBWBQ))
10180    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10181         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10182          VSTRWSBWBQ))
10183   ]
10184   "TARGET_HAVE_MVE"
10185 {
10186    rtx ops[3];
10187    ops[0] = operands[1];
10188    ops[1] = operands[2];
10189    ops[2] = operands[3];
10190    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10191    return "";
10192 }
10193   [(set_attr "length" "8")])
10194
10195 (define_expand "mve_vstrwq_scatter_base_wb_fv4sf"
10196   [(match_operand:V4SI 0 "s_register_operand" "=w")
10197    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10198    (match_operand:V4SF 2 "s_register_operand" "w")
10199    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10201 {
10202   rtx ignore_wb = gen_reg_rtx (V4SImode);
10203   emit_insn (
10204   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (ignore_wb,operands[0],
10205                                              operands[1], operands[2]));
10206   DONE;
10207 })
10208
10209 (define_expand "mve_vstrwq_scatter_base_wb_add_fv4sf"
10210   [(match_operand:V4SI 0 "s_register_operand" "=w")
10211    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10212    (match_operand:V4SI 2 "s_register_operand" "0")
10213    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10214   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10215 {
10216   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10217   emit_insn (
10218   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (operands[0], operands[2],
10219                                              operands[1], ignore_vec));
10220   DONE;
10221 })
10222
10223 ;;
10224 ;; [vstrwq_scatter_base_wb_f]
10225 ;;
10226 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf_insn"
10227  [(set (mem:BLK (scratch))
10228        (unspec:BLK
10229                 [(match_operand:V4SI 1 "s_register_operand" "0")
10230                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10231                  (match_operand:V4SF 3 "s_register_operand" "w")]
10232          VSTRWQSBWB_F))
10233    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10234         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10235          VSTRWQSBWB_F))
10236   ]
10237   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10238 {
10239    rtx ops[3];
10240    ops[0] = operands[1];
10241    ops[1] = operands[2];
10242    ops[2] = operands[3];
10243    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10244    return "";
10245 }
10246   [(set_attr "length" "4")])
10247
10248 (define_expand "mve_vstrwq_scatter_base_wb_p_fv4sf"
10249   [(match_operand:V4SI 0 "s_register_operand" "=w")
10250    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10251    (match_operand:V4SF 2 "s_register_operand" "w")
10252    (match_operand:HI 3 "vpr_register_operand")
10253    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10254   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10255 {
10256   rtx ignore_wb = gen_reg_rtx (V4SImode);
10257   emit_insn (
10258   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (ignore_wb, operands[0],
10259                                                operands[1], operands[2],
10260                                                operands[3]));
10261   DONE;
10262 })
10263
10264 (define_expand "mve_vstrwq_scatter_base_wb_p_add_fv4sf"
10265   [(match_operand:V4SI 0 "s_register_operand" "=w")
10266    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10267    (match_operand:V4SI 2 "s_register_operand" "0")
10268    (match_operand:HI 3 "vpr_register_operand")
10269    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10270   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10271 {
10272   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10273   emit_insn (
10274   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (operands[0], operands[2],
10275                                                operands[1], ignore_vec,
10276                                                operands[3]));
10277   DONE;
10278 })
10279
10280 ;;
10281 ;; [vstrwq_scatter_base_wb_p_f]
10282 ;;
10283 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf_insn"
10284  [(set (mem:BLK (scratch))
10285        (unspec:BLK
10286                 [(match_operand:V4SI 1 "s_register_operand" "0")
10287                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10288                  (match_operand:V4SF 3 "s_register_operand" "w")
10289                  (match_operand:HI 4 "vpr_register_operand")]
10290         VSTRWQSBWB_F))
10291    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10292         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10293          VSTRWQSBWB_F))
10294   ]
10295   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10296 {
10297    rtx ops[3];
10298    ops[0] = operands[1];
10299    ops[1] = operands[2];
10300    ops[2] = operands[3];
10301    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10302    return "";
10303 }
10304   [(set_attr "length" "8")])
10305
10306 (define_expand "mve_vstrdq_scatter_base_wb_<supf>v2di"
10307   [(match_operand:V2DI 0 "s_register_operand" "=w")
10308    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10309    (match_operand:V2DI 2 "s_register_operand" "w")
10310    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10311   "TARGET_HAVE_MVE"
10312 {
10313   rtx ignore_wb = gen_reg_rtx (V2DImode);
10314   emit_insn (
10315   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (ignore_wb, operands[0],
10316                                                   operands[1], operands[2]));
10317   DONE;
10318 })
10319
10320 (define_expand "mve_vstrdq_scatter_base_wb_add_<supf>v2di"
10321   [(match_operand:V2DI 0 "s_register_operand" "=w")
10322    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10323    (match_operand:V2DI 2 "s_register_operand" "0")
10324    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10325   "TARGET_HAVE_MVE"
10326 {
10327   rtx ignore_vec = gen_reg_rtx (V2DImode);
10328   emit_insn (
10329   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (operands[0], operands[2],
10330                                                   operands[1], ignore_vec));
10331   DONE;
10332 })
10333
10334 ;;
10335 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10336 ;;
10337 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di_insn"
10338   [(set (mem:BLK (scratch))
10339         (unspec:BLK
10340                 [(match_operand:V2DI 1 "s_register_operand" "0")
10341                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10342                  (match_operand:V2DI 3 "s_register_operand" "w")]
10343          VSTRDSBWBQ))
10344    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
10345         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10346          VSTRDSBWBQ))
10347   ]
10348   "TARGET_HAVE_MVE"
10349 {
10350    rtx ops[3];
10351    ops[0] = operands[1];
10352    ops[1] = operands[2];
10353    ops[2] = operands[3];
10354    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
10355    return "";
10356 }
10357   [(set_attr "length" "4")])
10358
10359 (define_expand "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
10360   [(match_operand:V2DI 0 "s_register_operand" "=w")
10361    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10362    (match_operand:V2DI 2 "s_register_operand" "w")
10363    (match_operand:HI 3 "vpr_register_operand")
10364    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10365   "TARGET_HAVE_MVE"
10366 {
10367   rtx ignore_wb = gen_reg_rtx (V2DImode);
10368   emit_insn (
10369   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (ignore_wb, operands[0],
10370                                                     operands[1], operands[2],
10371                                                     operands[3]));
10372   DONE;
10373 })
10374
10375 (define_expand "mve_vstrdq_scatter_base_wb_p_add_<supf>v2di"
10376   [(match_operand:V2DI 0 "s_register_operand" "=w")
10377    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10378    (match_operand:V2DI 2 "s_register_operand" "0")
10379    (match_operand:HI 3 "vpr_register_operand")
10380    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10381   "TARGET_HAVE_MVE"
10382 {
10383   rtx ignore_vec = gen_reg_rtx (V2DImode);
10384   emit_insn (
10385   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (operands[0], operands[2],
10386                                                     operands[1], ignore_vec,
10387                                                     operands[3]));
10388   DONE;
10389 })
10390
10391 ;;
10392 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
10393 ;;
10394 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn"
10395   [(set (mem:BLK (scratch))
10396         (unspec:BLK
10397                 [(match_operand:V2DI 1 "s_register_operand" "0")
10398                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10399                  (match_operand:V2DI 3 "s_register_operand" "w")
10400                  (match_operand:HI 4 "vpr_register_operand")]
10401          VSTRDSBWBQ))
10402    (set (match_operand:V2DI 0 "s_register_operand" "=w")
10403         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10404          VSTRDSBWBQ))
10405   ]
10406   "TARGET_HAVE_MVE"
10407 {
10408    rtx ops[3];
10409    ops[0] = operands[1];
10410    ops[1] = operands[2];
10411    ops[2] = operands[3];
10412    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]!",ops);
10413    return "";
10414 }
10415   [(set_attr "length" "8")])
10416
10417 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
10418   [(match_operand:V4SI 0 "s_register_operand")
10419    (match_operand:V4SI 1 "s_register_operand")
10420    (match_operand:SI 2 "mve_vldrd_immediate")
10421    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10422   "TARGET_HAVE_MVE"
10423 {
10424   rtx ignore_result = gen_reg_rtx (V4SImode);
10425   emit_insn (
10426   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
10427                                                  operands[1], operands[2]));
10428   DONE;
10429 })
10430
10431 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
10432   [(match_operand:V4SI 0 "s_register_operand")
10433    (match_operand:V4SI 1 "s_register_operand")
10434    (match_operand:SI 2 "mve_vldrd_immediate")
10435    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10436   "TARGET_HAVE_MVE"
10437 {
10438   rtx ignore_wb = gen_reg_rtx (V4SImode);
10439   emit_insn (
10440   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
10441                                                  operands[1], operands[2]));
10442   DONE;
10443 })
10444
10445 ;;
10446 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
10447 ;;
10448 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
10449   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10450         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10451                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10452                       (mem:BLK (scratch))]
10453          VLDRWGBWBQ))
10454    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10455         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10456          VLDRWGBWBQ))
10457   ]
10458   "TARGET_HAVE_MVE"
10459 {
10460    rtx ops[3];
10461    ops[0] = operands[0];
10462    ops[1] = operands[2];
10463    ops[2] = operands[3];
10464    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10465    return "";
10466 }
10467   [(set_attr "length" "4")])
10468
10469 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
10470   [(match_operand:V4SI 0 "s_register_operand")
10471    (match_operand:V4SI 1 "s_register_operand")
10472    (match_operand:SI 2 "mve_vldrd_immediate")
10473    (match_operand:HI 3 "vpr_register_operand")
10474    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10475   "TARGET_HAVE_MVE"
10476 {
10477   rtx ignore_result = gen_reg_rtx (V4SImode);
10478   emit_insn (
10479   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
10480                                                    operands[1], operands[2],
10481                                                    operands[3]));
10482   DONE;
10483 })
10484 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
10485   [(match_operand:V4SI 0 "s_register_operand")
10486    (match_operand:V4SI 1 "s_register_operand")
10487    (match_operand:SI 2 "mve_vldrd_immediate")
10488    (match_operand:HI 3 "vpr_register_operand")
10489    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10490   "TARGET_HAVE_MVE"
10491 {
10492   rtx ignore_wb = gen_reg_rtx (V4SImode);
10493   emit_insn (
10494   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10495                                                    operands[1], operands[2],
10496                                                    operands[3]));
10497   DONE;
10498 })
10499
10500 ;;
10501 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10502 ;;
10503 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10504   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10505         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10506                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10507                       (match_operand:HI 4 "vpr_register_operand" "Up")
10508                       (mem:BLK (scratch))]
10509          VLDRWGBWBQ))
10510    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10511         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10512          VLDRWGBWBQ))
10513   ]
10514   "TARGET_HAVE_MVE"
10515 {
10516    rtx ops[3];
10517    ops[0] = operands[0];
10518    ops[1] = operands[2];
10519    ops[2] = operands[3];
10520    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10521    return "";
10522 }
10523   [(set_attr "length" "8")])
10524
10525 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10526   [(match_operand:V4SI 0 "s_register_operand")
10527    (match_operand:V4SI 1 "s_register_operand")
10528    (match_operand:SI 2 "mve_vldrd_immediate")
10529    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10530   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10531 {
10532   rtx ignore_result = gen_reg_rtx (V4SFmode);
10533   emit_insn (
10534   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10535                                             operands[1], operands[2]));
10536   DONE;
10537 })
10538
10539 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10540   [(match_operand:V4SF 0 "s_register_operand")
10541    (match_operand:V4SI 1 "s_register_operand")
10542    (match_operand:SI 2 "mve_vldrd_immediate")
10543    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10544   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10545 {
10546   rtx ignore_wb = gen_reg_rtx (V4SImode);
10547   emit_insn (
10548   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10549                                             operands[1], operands[2]));
10550   DONE;
10551 })
10552
10553 ;;
10554 ;; [vldrwq_gather_base_wb_f]
10555 ;;
10556 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10557   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10558         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10559                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10560                       (mem:BLK (scratch))]
10561          VLDRWQGBWB_F))
10562    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10563         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10564          VLDRWQGBWB_F))
10565   ]
10566   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10567 {
10568    rtx ops[3];
10569    ops[0] = operands[0];
10570    ops[1] = operands[2];
10571    ops[2] = operands[3];
10572    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10573    return "";
10574 }
10575   [(set_attr "length" "4")])
10576
10577 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10578   [(match_operand:V4SI 0 "s_register_operand")
10579    (match_operand:V4SI 1 "s_register_operand")
10580    (match_operand:SI 2 "mve_vldrd_immediate")
10581    (match_operand:HI 3 "vpr_register_operand")
10582    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10583   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10584 {
10585   rtx ignore_result = gen_reg_rtx (V4SFmode);
10586   emit_insn (
10587   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10588                                               operands[1], operands[2],
10589                                               operands[3]));
10590   DONE;
10591 })
10592
10593 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10594   [(match_operand:V4SF 0 "s_register_operand")
10595    (match_operand:V4SI 1 "s_register_operand")
10596    (match_operand:SI 2 "mve_vldrd_immediate")
10597    (match_operand:HI 3 "vpr_register_operand")
10598    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10599   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10600 {
10601   rtx ignore_wb = gen_reg_rtx (V4SImode);
10602   emit_insn (
10603   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10604                                               operands[1], operands[2],
10605                                               operands[3]));
10606   DONE;
10607 })
10608
10609 ;;
10610 ;; [vldrwq_gather_base_wb_z_f]
10611 ;;
10612 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10613   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10614         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10615                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10616                       (match_operand:HI 4 "vpr_register_operand" "Up")
10617                       (mem:BLK (scratch))]
10618          VLDRWQGBWB_F))
10619    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10620         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10621          VLDRWQGBWB_F))
10622   ]
10623   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10624 {
10625    rtx ops[3];
10626    ops[0] = operands[0];
10627    ops[1] = operands[2];
10628    ops[2] = operands[3];
10629    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10630    return "";
10631 }
10632   [(set_attr "length" "8")])
10633
10634 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10635   [(match_operand:V2DI 0 "s_register_operand")
10636    (match_operand:V2DI 1 "s_register_operand")
10637    (match_operand:SI 2 "mve_vldrd_immediate")
10638    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10639   "TARGET_HAVE_MVE"
10640 {
10641   rtx ignore_result = gen_reg_rtx (V2DImode);
10642   emit_insn (
10643   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10644                                                  operands[1], operands[2]));
10645   DONE;
10646 })
10647
10648 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10649   [(match_operand:V2DI 0 "s_register_operand")
10650    (match_operand:V2DI 1 "s_register_operand")
10651    (match_operand:SI 2 "mve_vldrd_immediate")
10652    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10653   "TARGET_HAVE_MVE"
10654 {
10655   rtx ignore_wb = gen_reg_rtx (V2DImode);
10656   emit_insn (
10657   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10658                                                  operands[1], operands[2]));
10659   DONE;
10660 })
10661
10662
10663 ;;
10664 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10665 ;;
10666 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10667   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10668         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10669                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10670                       (mem:BLK (scratch))]
10671          VLDRDGBWBQ))
10672    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10673         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10674          VLDRDGBWBQ))
10675   ]
10676   "TARGET_HAVE_MVE"
10677 {
10678    rtx ops[3];
10679    ops[0] = operands[0];
10680    ops[1] = operands[2];
10681    ops[2] = operands[3];
10682    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10683    return "";
10684 }
10685   [(set_attr "length" "4")])
10686
10687 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10688   [(match_operand:V2DI 0 "s_register_operand")
10689    (match_operand:V2DI 1 "s_register_operand")
10690    (match_operand:SI 2 "mve_vldrd_immediate")
10691    (match_operand:HI 3 "vpr_register_operand")
10692    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10693   "TARGET_HAVE_MVE"
10694 {
10695   rtx ignore_result = gen_reg_rtx (V2DImode);
10696   emit_insn (
10697   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10698                                                    operands[1], operands[2],
10699                                                    operands[3]));
10700   DONE;
10701 })
10702
10703 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10704   [(match_operand:V2DI 0 "s_register_operand")
10705    (match_operand:V2DI 1 "s_register_operand")
10706    (match_operand:SI 2 "mve_vldrd_immediate")
10707    (match_operand:HI 3 "vpr_register_operand")
10708    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10709   "TARGET_HAVE_MVE"
10710 {
10711   rtx ignore_wb = gen_reg_rtx (V2DImode);
10712   emit_insn (
10713   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10714                                                    operands[1], operands[2],
10715                                                    operands[3]));
10716   DONE;
10717 })
10718
10719 (define_insn "get_fpscr_nzcvqc"
10720  [(set (match_operand:SI 0 "register_operand" "=r")
10721    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10722  "TARGET_HAVE_MVE"
10723  "vmrs\\t%0, FPSCR_nzcvqc"
10724   [(set_attr "type" "mve_move")])
10725
10726 (define_insn "set_fpscr_nzcvqc"
10727  [(set (reg:SI VFPCC_REGNUM)
10728    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10729     VUNSPEC_SET_FPSCR_NZCVQC))]
10730  "TARGET_HAVE_MVE"
10731  "vmsr\\tFPSCR_nzcvqc, %0"
10732   [(set_attr "type" "mve_move")])
10733
10734 ;;
10735 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10736 ;;
10737 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10738   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10739         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10740                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10741                       (match_operand:HI 4 "vpr_register_operand" "Up")
10742                       (mem:BLK (scratch))]
10743          VLDRDGBWBQ))
10744    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10745         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10746          VLDRDGBWBQ))
10747   ]
10748   "TARGET_HAVE_MVE"
10749 {
10750    rtx ops[3];
10751    ops[0] = operands[0];
10752    ops[1] = operands[2];
10753    ops[2] = operands[3];
10754    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10755    return "";
10756 }
10757   [(set_attr "length" "8")])
10758 ;;
10759 ;; [vadciq_m_s, vadciq_m_u])
10760 ;;
10761 (define_insn "mve_vadciq_m_<supf>v4si"
10762   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10763         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10764                       (match_operand:V4SI 2 "s_register_operand" "w")
10765                       (match_operand:V4SI 3 "s_register_operand" "w")
10766                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10767          VADCIQ_M))
10768    (set (reg:SI VFPCC_REGNUM)
10769         (unspec:SI [(const_int 0)]
10770          VADCIQ_M))
10771   ]
10772   "TARGET_HAVE_MVE"
10773   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10774   [(set_attr "type" "mve_move")
10775    (set_attr "length" "8")])
10776
10777 ;;
10778 ;; [vadciq_u, vadciq_s])
10779 ;;
10780 (define_insn "mve_vadciq_<supf>v4si"
10781   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10782         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10783                       (match_operand:V4SI 2 "s_register_operand" "w")]
10784          VADCIQ))
10785    (set (reg:SI VFPCC_REGNUM)
10786         (unspec:SI [(const_int 0)]
10787          VADCIQ))
10788   ]
10789   "TARGET_HAVE_MVE"
10790   "vadci.i32\t%q0, %q1, %q2"
10791   [(set_attr "type" "mve_move")
10792    (set_attr "length" "4")])
10793
10794 ;;
10795 ;; [vadcq_m_s, vadcq_m_u])
10796 ;;
10797 (define_insn "mve_vadcq_m_<supf>v4si"
10798   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10799         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10800                       (match_operand:V4SI 2 "s_register_operand" "w")
10801                       (match_operand:V4SI 3 "s_register_operand" "w")
10802                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10803          VADCQ_M))
10804    (set (reg:SI VFPCC_REGNUM)
10805         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10806          VADCQ_M))
10807   ]
10808   "TARGET_HAVE_MVE"
10809   "vpst\;vadct.i32\t%q0, %q2, %q3"
10810   [(set_attr "type" "mve_move")
10811    (set_attr "length" "8")])
10812
10813 ;;
10814 ;; [vadcq_u, vadcq_s])
10815 ;;
10816 (define_insn "mve_vadcq_<supf>v4si"
10817   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10818         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10819                        (match_operand:V4SI 2 "s_register_operand" "w")]
10820          VADCQ))
10821    (set (reg:SI VFPCC_REGNUM)
10822         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10823          VADCQ))
10824   ]
10825   "TARGET_HAVE_MVE"
10826   "vadc.i32\t%q0, %q1, %q2"
10827   [(set_attr "type" "mve_move")
10828    (set_attr "length" "4")
10829    (set_attr "conds" "set")])
10830
10831 ;;
10832 ;; [vsbciq_m_u, vsbciq_m_s])
10833 ;;
10834 (define_insn "mve_vsbciq_m_<supf>v4si"
10835   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10836         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10837                       (match_operand:V4SI 2 "s_register_operand" "w")
10838                       (match_operand:V4SI 3 "s_register_operand" "w")
10839                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10840          VSBCIQ_M))
10841    (set (reg:SI VFPCC_REGNUM)
10842         (unspec:SI [(const_int 0)]
10843          VSBCIQ_M))
10844   ]
10845   "TARGET_HAVE_MVE"
10846   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10847   [(set_attr "type" "mve_move")
10848    (set_attr "length" "8")])
10849
10850 ;;
10851 ;; [vsbciq_s, vsbciq_u])
10852 ;;
10853 (define_insn "mve_vsbciq_<supf>v4si"
10854   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10855         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10856                       (match_operand:V4SI 2 "s_register_operand" "w")]
10857          VSBCIQ))
10858    (set (reg:SI VFPCC_REGNUM)
10859         (unspec:SI [(const_int 0)]
10860          VSBCIQ))
10861   ]
10862   "TARGET_HAVE_MVE"
10863   "vsbci.i32\t%q0, %q1, %q2"
10864   [(set_attr "type" "mve_move")
10865    (set_attr "length" "4")])
10866
10867 ;;
10868 ;; [vsbcq_m_u, vsbcq_m_s])
10869 ;;
10870 (define_insn "mve_vsbcq_m_<supf>v4si"
10871   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10872         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10873                       (match_operand:V4SI 2 "s_register_operand" "w")
10874                       (match_operand:V4SI 3 "s_register_operand" "w")
10875                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10876          VSBCQ_M))
10877    (set (reg:SI VFPCC_REGNUM)
10878         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10879          VSBCQ_M))
10880   ]
10881   "TARGET_HAVE_MVE"
10882   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10883   [(set_attr "type" "mve_move")
10884    (set_attr "length" "8")])
10885
10886 ;;
10887 ;; [vsbcq_s, vsbcq_u])
10888 ;;
10889 (define_insn "mve_vsbcq_<supf>v4si"
10890   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10891         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10892                       (match_operand:V4SI 2 "s_register_operand" "w")]
10893          VSBCQ))
10894    (set (reg:SI VFPCC_REGNUM)
10895         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10896          VSBCQ))
10897   ]
10898   "TARGET_HAVE_MVE"
10899   "vsbc.i32\t%q0, %q1, %q2"
10900   [(set_attr "type" "mve_move")
10901    (set_attr "length" "4")])
10902
10903 ;;
10904 ;; [vst2q])
10905 ;;
10906 (define_insn "mve_vst2q<mode>"
10907   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10908         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10909                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10910          VST2Q))
10911   ]
10912   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10913    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10914 {
10915    rtx ops[4];
10916    int regno = REGNO (operands[1]);
10917    ops[0] = gen_rtx_REG (TImode, regno);
10918    ops[1] = gen_rtx_REG (TImode, regno + 4);
10919    rtx reg  = operands[0];
10920    while (reg && !REG_P (reg))
10921     reg = XEXP (reg, 0);
10922    gcc_assert (REG_P (reg));
10923    ops[2] = reg;
10924    ops[3] = operands[0];
10925    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10926                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10927    return "";
10928 }
10929   [(set_attr "length" "8")])
10930
10931 ;;
10932 ;; [vld2q])
10933 ;;
10934 (define_insn "mve_vld2q<mode>"
10935   [(set (match_operand:OI 0 "s_register_operand" "=w")
10936         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10937                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10938          VLD2Q))
10939   ]
10940   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10941    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10942 {
10943    rtx ops[4];
10944    int regno = REGNO (operands[0]);
10945    ops[0] = gen_rtx_REG (TImode, regno);
10946    ops[1] = gen_rtx_REG (TImode, regno + 4);
10947    rtx reg  = operands[1];
10948    while (reg && !REG_P (reg))
10949     reg = XEXP (reg, 0);
10950    gcc_assert (REG_P (reg));
10951    ops[2] = reg;
10952    ops[3] = operands[1];
10953    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10954                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10955    return "";
10956 }
10957   [(set_attr "length" "8")])
10958
10959 ;;
10960 ;; [vld4q])
10961 ;;
10962 (define_insn "mve_vld4q<mode>"
10963   [(set (match_operand:XI 0 "s_register_operand" "=w")
10964         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10965                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10966          VLD4Q))
10967   ]
10968   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10969    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10970 {
10971    rtx ops[6];
10972    int regno = REGNO (operands[0]);
10973    ops[0] = gen_rtx_REG (TImode, regno);
10974    ops[1] = gen_rtx_REG (TImode, regno+4);
10975    ops[2] = gen_rtx_REG (TImode, regno+8);
10976    ops[3] = gen_rtx_REG (TImode, regno + 12);
10977    rtx reg  = operands[1];
10978    while (reg && !REG_P (reg))
10979     reg = XEXP (reg, 0);
10980    gcc_assert (REG_P (reg));
10981    ops[4] = reg;
10982    ops[5] = operands[1];
10983    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10984                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10985                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10986                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10987    return "";
10988 }
10989   [(set_attr "length" "16")])
10990 ;;
10991 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10992 ;;
10993 (define_insn "mve_vec_extract<mode><V_elem_l>"
10994  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10995    (vec_select:<V_elem>
10996     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10997     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10998   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10999    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11000 {
11001   if (BYTES_BIG_ENDIAN)
11002     {
11003       int elt = INTVAL (operands[2]);
11004       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11005       operands[2] = GEN_INT (elt);
11006     }
11007   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
11008 }
11009  [(set_attr "type" "mve_move")])
11010
11011 (define_insn "mve_vec_extractv2didi"
11012  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
11013    (vec_select:DI
11014     (match_operand:V2DI 1 "s_register_operand" "w")
11015     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
11016   "TARGET_HAVE_MVE"
11017 {
11018   int elt = INTVAL (operands[2]);
11019   if (BYTES_BIG_ENDIAN)
11020     elt = 1 - elt;
11021
11022   if (elt == 0)
11023    return "vmov\t%Q0, %R0, %e1";
11024   else
11025    return "vmov\t%Q0, %R0, %f1";
11026 }
11027  [(set_attr "type" "mve_move")])
11028
11029 (define_insn "*mve_vec_extract_sext_internal<mode>"
11030  [(set (match_operand:SI 0 "s_register_operand" "=r")
11031    (sign_extend:SI
11032     (vec_select:<V_elem>
11033      (match_operand:MVE_2 1 "s_register_operand" "w")
11034      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11035   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11036    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11037 {
11038   if (BYTES_BIG_ENDIAN)
11039     {
11040       int elt = INTVAL (operands[2]);
11041       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11042       operands[2] = GEN_INT (elt);
11043     }
11044   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
11045 }
11046  [(set_attr "type" "mve_move")])
11047
11048 (define_insn "*mve_vec_extract_zext_internal<mode>"
11049  [(set (match_operand:SI 0 "s_register_operand" "=r")
11050    (zero_extend:SI
11051     (vec_select:<V_elem>
11052      (match_operand:MVE_2 1 "s_register_operand" "w")
11053      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11054   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11055    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11056 {
11057   if (BYTES_BIG_ENDIAN)
11058     {
11059       int elt = INTVAL (operands[2]);
11060       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11061       operands[2] = GEN_INT (elt);
11062     }
11063   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
11064 }
11065  [(set_attr "type" "mve_move")])
11066
11067 ;;
11068 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
11069 ;;
11070 (define_insn "mve_vec_set<mode>_internal"
11071  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
11072        (vec_merge:VQ2
11073         (vec_duplicate:VQ2
11074           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
11075         (match_operand:VQ2 3 "s_register_operand" "0")
11076         (match_operand:SI 2 "immediate_operand" "i")))]
11077   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11078    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11079 {
11080   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11081   if (BYTES_BIG_ENDIAN)
11082     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11083   operands[2] = GEN_INT (elt);
11084
11085   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
11086 }
11087  [(set_attr "type" "mve_move")])
11088
11089 (define_insn "mve_vec_setv2di_internal"
11090  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
11091        (vec_merge:V2DI
11092         (vec_duplicate:V2DI
11093           (match_operand:DI 1 "nonimmediate_operand" "r"))
11094         (match_operand:V2DI 3 "s_register_operand" "0")
11095         (match_operand:SI 2 "immediate_operand" "i")))]
11096  "TARGET_HAVE_MVE"
11097 {
11098   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11099   if (BYTES_BIG_ENDIAN)
11100     elt = 1 - elt;
11101
11102   if (elt == 0)
11103    return "vmov\t%e0, %Q1, %R1";
11104   else
11105    return "vmov\t%f0, %J1, %K1";
11106 }
11107  [(set_attr "type" "mve_move")])
11108
11109 ;;
11110 ;; [uqrshll_di]
11111 ;;
11112 (define_insn "mve_uqrshll_sat<supf>_di"
11113   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11114         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11115                     (match_operand:SI 2 "s_register_operand" "r")]
11116          UQRSHLLQ))]
11117   "TARGET_HAVE_MVE"
11118   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
11119   [(set_attr "predicable" "yes")])
11120
11121 ;;
11122 ;; [sqrshrl_di]
11123 ;;
11124 (define_insn "mve_sqrshrl_sat<supf>_di"
11125   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11126         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11127                     (match_operand:SI 2 "s_register_operand" "r")]
11128          SQRSHRLQ))]
11129   "TARGET_HAVE_MVE"
11130   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
11131   [(set_attr "predicable" "yes")])
11132
11133 ;;
11134 ;; [uqrshl_si]
11135 ;;
11136 (define_insn "mve_uqrshl_si"
11137   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11138         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11139                     (match_operand:SI 2 "s_register_operand" "r")]
11140          UQRSHL))]
11141   "TARGET_HAVE_MVE"
11142   "uqrshl%?\\t%1, %2"
11143   [(set_attr "predicable" "yes")])
11144
11145 ;;
11146 ;; [sqrshr_si]
11147 ;;
11148 (define_insn "mve_sqrshr_si"
11149   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11150         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11151                     (match_operand:SI 2 "s_register_operand" "r")]
11152          SQRSHR))]
11153   "TARGET_HAVE_MVE"
11154   "sqrshr%?\\t%1, %2"
11155   [(set_attr "predicable" "yes")])
11156
11157 ;;
11158 ;; [uqshll_di]
11159 ;;
11160 (define_insn "mve_uqshll_di"
11161   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11162         (us_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11163                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11164   "TARGET_HAVE_MVE"
11165   "uqshll%?\\t%Q1, %R1, %2"
11166   [(set_attr "predicable" "yes")])
11167
11168 ;;
11169 ;; [urshrl_di]
11170 ;;
11171 (define_insn "mve_urshrl_di"
11172   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11173         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11174                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11175          URSHRL))]
11176   "TARGET_HAVE_MVE"
11177   "urshrl%?\\t%Q1, %R1, %2"
11178   [(set_attr "predicable" "yes")])
11179
11180 ;;
11181 ;; [uqshl_si]
11182 ;;
11183 (define_insn "mve_uqshl_si"
11184   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11185         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "r")
11186                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11187   "TARGET_HAVE_MVE"
11188   "uqshl%?\\t%1, %2"
11189   [(set_attr "predicable" "yes")])
11190
11191 ;;
11192 ;; [urshr_si]
11193 ;;
11194 (define_insn "mve_urshr_si"
11195   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11196         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11197                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11198          URSHR))]
11199   "TARGET_HAVE_MVE"
11200   "urshr%?\\t%1, %2"
11201   [(set_attr "predicable" "yes")])
11202
11203 ;;
11204 ;; [sqshl_si]
11205 ;;
11206 (define_insn "mve_sqshl_si"
11207   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11208         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "r")
11209                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11210   "TARGET_HAVE_MVE"
11211   "sqshl%?\\t%1, %2"
11212   [(set_attr "predicable" "yes")])
11213
11214 ;;
11215 ;; [srshr_si]
11216 ;;
11217 (define_insn "mve_srshr_si"
11218   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11219         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "r")
11220                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11221          SRSHR))]
11222   "TARGET_HAVE_MVE"
11223   "srshr%?\\t%1, %2"
11224   [(set_attr "predicable" "yes")])
11225
11226 ;;
11227 ;; [srshrl_di]
11228 ;;
11229 (define_insn "mve_srshrl_di"
11230   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11231         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11232                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11233          SRSHRL))]
11234   "TARGET_HAVE_MVE"
11235   "srshrl%?\\t%Q1, %R1, %2"
11236   [(set_attr "predicable" "yes")])
11237
11238 ;;
11239 ;; [sqshll_di]
11240 ;;
11241 (define_insn "mve_sqshll_di"
11242   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11243         (ss_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11244                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11245   "TARGET_HAVE_MVE"
11246   "sqshll%?\\t%Q1, %R1, %2"
11247   [(set_attr "predicable" "yes")])
11248
11249 ;;
11250 ;; [vshlcq_m_u vshlcq_m_s]
11251 ;;
11252 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
11253  [(match_operand:MVE_2 0 "s_register_operand")
11254   (match_operand:MVE_2 1 "s_register_operand")
11255   (match_operand:SI 2 "s_register_operand")
11256   (match_operand:SI 3 "mve_imm_32")
11257   (match_operand:HI 4 "vpr_register_operand")
11258   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11259  "TARGET_HAVE_MVE"
11260 {
11261   rtx ignore_wb = gen_reg_rtx (SImode);
11262   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
11263                                             operands[2], operands[3],
11264                                             operands[4]));
11265   DONE;
11266 })
11267
11268 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
11269  [(match_operand:SI 0 "s_register_operand")
11270   (match_operand:MVE_2 1 "s_register_operand")
11271   (match_operand:SI 2 "s_register_operand")
11272   (match_operand:SI 3 "mve_imm_32")
11273   (match_operand:HI 4 "vpr_register_operand")
11274   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11275  "TARGET_HAVE_MVE"
11276 {
11277   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
11278   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
11279                                             operands[1], operands[2],
11280                                             operands[3], operands[4]));
11281   DONE;
11282 })
11283
11284 (define_insn "mve_vshlcq_m_<supf><mode>"
11285  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
11286        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
11287                       (match_operand:SI 3 "s_register_operand" "1")
11288                       (match_operand:SI 4 "mve_imm_32" "Rf")
11289                       (match_operand:HI 5 "vpr_register_operand" "Up")]
11290         VSHLCQ_M))
11291   (set (match_operand:SI  1 "s_register_operand" "=r")
11292        (unspec:SI [(match_dup 2)
11293                    (match_dup 3)
11294                    (match_dup 4)
11295                    (match_dup 5)]
11296         VSHLCQ_M))
11297  ]
11298  "TARGET_HAVE_MVE"
11299  "vpst\;vshlct\t%q0, %1, %4"
11300  [(set_attr "type" "mve_move")
11301   (set_attr "length" "8")])
11302
11303 (define_insn "*mve_vec_duplicate<mode>"
11304  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
11305        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
11306  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
11307  "vdup.<V_sz_elem>\t%q0, %1"
11308  [(set_attr "type" "mve_move")])
11309
11310 ;; CDE instructions on MVE registers.
11311
11312 (define_insn "arm_vcx1qv16qi"
11313   [(set (match_operand:V16QI 0 "register_operand" "=t")
11314         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11315                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
11316          UNSPEC_VCDE))]
11317   "TARGET_CDE && TARGET_HAVE_MVE"
11318   "vcx1\\tp%c1, %q0, #%c2"
11319   [(set_attr "type" "coproc")]
11320 )
11321
11322 (define_insn "arm_vcx1qav16qi"
11323   [(set (match_operand:V16QI 0 "register_operand" "=t")
11324         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11325                             (match_operand:V16QI 2 "register_operand" "0")
11326                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
11327          UNSPEC_VCDEA))]
11328   "TARGET_CDE && TARGET_HAVE_MVE"
11329   "vcx1a\\tp%c1, %q0, #%c3"
11330   [(set_attr "type" "coproc")]
11331 )
11332
11333 (define_insn "arm_vcx2qv16qi"
11334   [(set (match_operand:V16QI 0 "register_operand" "=t")
11335         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11336                           (match_operand:V16QI 2 "register_operand" "t")
11337                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
11338          UNSPEC_VCDE))]
11339   "TARGET_CDE && TARGET_HAVE_MVE"
11340   "vcx2\\tp%c1, %q0, %q2, #%c3"
11341   [(set_attr "type" "coproc")]
11342 )
11343
11344 (define_insn "arm_vcx2qav16qi"
11345   [(set (match_operand:V16QI 0 "register_operand" "=t")
11346         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11347                           (match_operand:V16QI 2 "register_operand" "0")
11348                           (match_operand:V16QI 3 "register_operand" "t")
11349                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
11350          UNSPEC_VCDEA))]
11351   "TARGET_CDE && TARGET_HAVE_MVE"
11352   "vcx2a\\tp%c1, %q0, %q3, #%c4"
11353   [(set_attr "type" "coproc")]
11354 )
11355
11356 (define_insn "arm_vcx3qv16qi"
11357   [(set (match_operand:V16QI 0 "register_operand" "=t")
11358         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11359                           (match_operand:V16QI 2 "register_operand" "t")
11360                           (match_operand:V16QI 3 "register_operand" "t")
11361                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
11362          UNSPEC_VCDE))]
11363   "TARGET_CDE && TARGET_HAVE_MVE"
11364   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
11365   [(set_attr "type" "coproc")]
11366 )
11367
11368 (define_insn "arm_vcx3qav16qi"
11369   [(set (match_operand:V16QI 0 "register_operand" "=t")
11370         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11371                           (match_operand:V16QI 2 "register_operand" "0")
11372                           (match_operand:V16QI 3 "register_operand" "t")
11373                           (match_operand:V16QI 4 "register_operand" "t")
11374                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
11375          UNSPEC_VCDEA))]
11376   "TARGET_CDE && TARGET_HAVE_MVE"
11377   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
11378   [(set_attr "type" "coproc")]
11379 )
11380
11381 (define_insn "arm_vcx1q<a>_p_v16qi"
11382   [(set (match_operand:V16QI 0 "register_operand" "=t")
11383         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11384                            (match_operand:V16QI 2 "register_operand" "0")
11385                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
11386                            (match_operand:HI 4 "vpr_register_operand" "Up")]
11387          CDE_VCX))]
11388   "TARGET_CDE && TARGET_HAVE_MVE"
11389   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
11390   [(set_attr "type" "coproc")
11391    (set_attr "length" "8")]
11392 )
11393
11394 (define_insn "arm_vcx2q<a>_p_v16qi"
11395   [(set (match_operand:V16QI 0 "register_operand" "=t")
11396         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11397                           (match_operand:V16QI 2 "register_operand" "0")
11398                           (match_operand:V16QI 3 "register_operand" "t")
11399                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
11400                           (match_operand:HI 5 "vpr_register_operand" "Up")]
11401          CDE_VCX))]
11402   "TARGET_CDE && TARGET_HAVE_MVE"
11403   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
11404   [(set_attr "type" "coproc")
11405    (set_attr "length" "8")]
11406 )
11407
11408 (define_insn "arm_vcx3q<a>_p_v16qi"
11409   [(set (match_operand:V16QI 0 "register_operand" "=t")
11410         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
11411                           (match_operand:V16QI 2 "register_operand" "0")
11412                           (match_operand:V16QI 3 "register_operand" "t")
11413                           (match_operand:V16QI 4 "register_operand" "t")
11414                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
11415                           (match_operand:HI 6 "vpr_register_operand" "Up")]
11416          CDE_VCX))]
11417   "TARGET_CDE && TARGET_HAVE_MVE"
11418   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
11419   [(set_attr "type" "coproc")
11420    (set_attr "length" "8")]
11421 )