gcc/config/rs6000/vector.md

   1 ;; Expander definitions for vector support between altivec & vsx.  No
   2 ;; instructions are in this file, this file provides the generic vector
   3 ;; expander, and the actual vector instructions will be in altivec.md and
   4 ;; vsx.md
   5
   6 ;; Copyright (C) 2009-2023 Free Software Foundation, Inc.
   7 ;; Contributed by Michael Meissner <meissner@linux.vnet.ibm.com>
   8
   9 ;; This file is part of GCC.
  10
  11 ;; GCC is free software; you can redistribute it and/or modify it
  12 ;; under the terms of the GNU General Public License as published
  13 ;; by the Free Software Foundation; either version 3, or (at your
  14 ;; option) any later version.
  15
  16 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  17 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  18 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  19 ;; License for more details.
  20
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GCC; see the file COPYING3.  If not see
  23 ;; <http://www.gnu.org/licenses/>.
  24
  25
  26 ;; Vector int modes
  27 (define_mode_iterator VEC_I [V16QI V8HI V4SI V2DI])
  28
  29 ;; Vector int modes for comparison, shift and rotation
  30 (define_mode_iterator VEC_IC [V16QI V8HI V4SI V2DI (V1TI "TARGET_POWER10")])
  31
  32 ;; 128-bit int modes
  33 (define_mode_iterator VEC_TI [V1TI TI])
  34
  35 ;; Vector int modes for parity
  36 (define_mode_iterator VEC_IP [V8HI
  37                               V4SI
  38                               V2DI
  39                               V1TI
  40                               TI])
  41
  42 ;; Vector float modes
  43 (define_mode_iterator VEC_F [V4SF V2DF])
  44
  45 ;; Vector arithmetic modes
  46 (define_mode_iterator VEC_A [V16QI V8HI V4SI V2DI V4SF V2DF])
  47
  48 ;; Vector modes that need alginment via permutes
  49 (define_mode_iterator VEC_K [V16QI V8HI V4SI V4SF])
  50
  51 ;; Vector logical modes
  52 (define_mode_iterator VEC_L [V16QI V8HI V4SI V2DI V4SF V2DF V1TI TI KF TF])
  53
  54 ;; Vector modes for moves.  Don't do TImode or TFmode here, since their
  55 ;; moves are handled elsewhere.
  56 (define_mode_iterator VEC_M [V16QI V8HI V4SI V2DI V4SF V2DF V1TI KF])
  57
  58 ;; Vector modes for types that don't need a realignment under VSX
  59 (define_mode_iterator VEC_N [V4SI V4SF V2DI V2DF V1TI KF TF])
  60
  61 ;; Vector comparison modes
  62 (define_mode_iterator VEC_C [V16QI V8HI V4SI V2DI V4SF V2DF V1TI])
  63
  64 ;; Vector init/extract modes
  65 (define_mode_iterator VEC_E [V16QI V8HI V4SI V2DI V4SF V2DF])
  66
  67 ;; Vector modes for 64-bit base types
  68 (define_mode_iterator VEC_64 [V2DI V2DF])
  69
  70 ;; Vector integer modes
  71 (define_mode_iterator VI [V4SI V8HI V16QI])
  72
  73 ;; Base type from vector mode
  74 (define_mode_attr VEC_base [(V16QI "QI")
  75                             (V8HI  "HI")
  76                             (V4SI  "SI")
  77                             (V2DI  "DI")
  78                             (V4SF  "SF")
  79                             (V2DF  "DF")
  80                             (V1TI  "TI")
  81                             (TI    "TI")])
  82
  83 ;; As above, but in lower case
  84 (define_mode_attr VEC_base_l [(V16QI "qi")
  85                               (V8HI  "hi")
  86                               (V4SI  "si")
  87                               (V2DI  "di")
  88                               (V4SF  "sf")
  89                               (V2DF  "df")
  90                               (V1TI  "ti")
  91                               (TI    "ti")])
  92
  93 ;; Same size integer type for floating point data
  94 (define_mode_attr VEC_int [(V4SF  "v4si")
  95                            (V2DF  "v2di")])
  96
  97 (define_mode_attr VEC_INT [(V4SF  "V4SI")
  98                            (V2DF  "V2DI")])
  99
 100 ;; constants for unspec
 101 (define_c_enum "unspec" [UNSPEC_PREDICATE
 102                          UNSPEC_REDUC
 103                          UNSPEC_NEZ_P])
 104
 105 ;; Vector reduction code iterators
 106 (define_code_iterator VEC_reduc [plus smin smax])
 107
 108 (define_code_attr VEC_reduc_name [(plus "plus")
 109                                   (smin "smin")
 110                                   (smax "smax")])
 111
 112 (define_code_attr VEC_reduc_rtx [(plus "add")
 113                                  (smin "smin")
 114                                  (smax "smax")])
 115
 116 ;; code iterators and attributes for vector FP comparison operators:
 117 (define_code_iterator
 118   vector_fp_comparison_simple [lt le ne ungt unge unlt unle])
 119 (define_code_iterator
 120   vector_fp_comparison_complex [ltgt uneq unordered ordered])
 121
 122 \f
 123 ;; Vector move instructions.  Little-endian VSX loads and stores require
 124 ;; special handling to circumvent "element endianness."
 125 (define_expand "mov<mode>"
 126   [(set (match_operand:VEC_M 0 "nonimmediate_operand")
 127         (match_operand:VEC_M 1 "any_operand"))]
 128   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 129 {
 130   if (can_create_pseudo_p ())
 131     {
 132       if (CONSTANT_P (operands[1]))
 133         {
 134           if (FLOAT128_VECTOR_P (<MODE>mode))
 135             {
 136               if (!easy_fp_constant (operands[1], <MODE>mode))
 137                 operands[1] = force_const_mem (<MODE>mode, operands[1]);
 138             }
 139           else if (!easy_vector_constant (operands[1], <MODE>mode))
 140             operands[1] = force_const_mem (<MODE>mode, operands[1]);
 141         }
 142
 143       if (!vlogical_operand (operands[0], <MODE>mode)
 144           && !vlogical_operand (operands[1], <MODE>mode))
 145         operands[1] = force_reg (<MODE>mode, operands[1]);
 146     }
 147   /* When generating load/store instructions to/from VSX registers on
 148      pre-power9 hardware in little endian mode, we need to emit register
 149      permute instructions to byte swap the contents, since the VSX load/store
 150      instructions do not include a byte swap as part of their operation.
 151      Altivec loads and stores have no such problem, so we skip them below.  */
 152   if (!BYTES_BIG_ENDIAN
 153       && VECTOR_MEM_VSX_P (<MODE>mode)
 154       && !TARGET_P9_VECTOR
 155       && !gpr_or_gpr_p (operands[0], operands[1])
 156       && ((memory_operand (operands[0], <MODE>mode)
 157            && !altivec_indexed_or_indirect_operand(operands[0], <MODE>mode))
 158           ^ (memory_operand (operands[1], <MODE>mode)
 159              && !altivec_indexed_or_indirect_operand(operands[1], <MODE>mode))))
 160     {
 161       rs6000_emit_le_vsx_move (operands[0], operands[1], <MODE>mode);
 162       DONE;
 163     }
 164 })
 165
 166 ;; Generic vector floating point load/store instructions.  These will match
 167 ;; insns defined in vsx.md or altivec.md depending on the switches.
 168 (define_expand "vector_load_<mode>"
 169   [(set (match_operand:VEC_M 0 "vfloat_operand")
 170         (match_operand:VEC_M 1 "memory_operand"))]
 171   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 172   "")
 173
 174 (define_expand "vector_store_<mode>"
 175   [(set (match_operand:VEC_M 0 "memory_operand")
 176         (match_operand:VEC_M 1 "vfloat_operand"))]
 177   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 178   "")
 179
 180 ;; Splits if a GPR register was chosen for the move
 181 (define_split
 182   [(set (match_operand:VEC_L 0 "nonimmediate_operand")
 183         (match_operand:VEC_L 1 "input_operand"))]
 184   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)
 185    && reload_completed
 186    && gpr_or_gpr_p (operands[0], operands[1])
 187    && !direct_move_p (operands[0], operands[1])
 188    && !quad_load_store_p (operands[0], operands[1])"
 189   [(pc)]
 190 {
 191   rs6000_split_multireg_move (operands[0], operands[1]);
 192   DONE;
 193 })
 194
 195 \f
 196 ;; Generic floating point vector arithmetic support
 197 (define_expand "add<mode>3"
 198   [(set (match_operand:VEC_F 0 "vfloat_operand")
 199         (plus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 200                     (match_operand:VEC_F 2 "vfloat_operand")))]
 201   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 202   "")
 203
 204 (define_expand "sub<mode>3"
 205   [(set (match_operand:VEC_F 0 "vfloat_operand")
 206         (minus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 207                      (match_operand:VEC_F 2 "vfloat_operand")))]
 208   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 209   "")
 210
 211 (define_expand "mul<mode>3"
 212   [(set (match_operand:VEC_F 0 "vfloat_operand")
 213         (mult:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 214                     (match_operand:VEC_F 2 "vfloat_operand")))]
 215   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 216 {
 217   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 218     {
 219       emit_insn (gen_altivec_mulv4sf3 (operands[0], operands[1], operands[2]));
 220       DONE;
 221     }
 222 })
 223
 224 (define_expand "div<mode>3"
 225   [(set (match_operand:VEC_F 0 "vfloat_operand")
 226         (div:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 227                    (match_operand:VEC_F 2 "vfloat_operand")))]
 228   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 229 {
 230   if (RS6000_RECIP_AUTO_RE_P (<MODE>mode)
 231       && can_create_pseudo_p () && flag_finite_math_only
 232       && !flag_trapping_math && flag_reciprocal_math)
 233     {
 234       rs6000_emit_swdiv (operands[0], operands[1], operands[2], true);
 235       DONE;
 236     }
 237 })
 238
 239 (define_expand "neg<mode>2"
 240   [(set (match_operand:VEC_F 0 "vfloat_operand")
 241         (neg:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 242   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 243 {
 244   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 245     {
 246       emit_insn (gen_altivec_negv4sf2 (operands[0], operands[1]));
 247       DONE;
 248     }
 249 })
 250
 251 (define_expand "abs<mode>2"
 252   [(set (match_operand:VEC_F 0 "vfloat_operand")
 253         (abs:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 254   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 255 {
 256   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 257     {
 258       emit_insn (gen_altivec_absv4sf2 (operands[0], operands[1]));
 259       DONE;
 260     }
 261 })
 262
 263 (define_expand "smin<mode>3"
 264   [(set (match_operand:VEC_F 0 "register_operand")
 265         (smin:VEC_F (match_operand:VEC_F 1 "register_operand")
 266                     (match_operand:VEC_F 2 "register_operand")))]
 267   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 268   "")
 269
 270 (define_expand "smax<mode>3"
 271   [(set (match_operand:VEC_F 0 "register_operand")
 272         (smax:VEC_F (match_operand:VEC_F 1 "register_operand")
 273                     (match_operand:VEC_F 2 "register_operand")))]
 274   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 275   "")
 276
 277
 278 (define_expand "sqrt<mode>2"
 279   [(set (match_operand:VEC_F 0 "vfloat_operand")
 280         (sqrt:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 281   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 282 {
 283   if (<MODE>mode == V4SFmode
 284       && !optimize_function_for_size_p (cfun)
 285       && flag_finite_math_only && !flag_trapping_math
 286       && flag_unsafe_math_optimizations)
 287     {
 288       rs6000_emit_swsqrt (operands[0], operands[1], 0);
 289       DONE;
 290     }
 291 })
 292
 293 (define_expand "rsqrte<mode>2"
 294   [(set (match_operand:VEC_F 0 "vfloat_operand")
 295         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 296                       UNSPEC_RSQRT))]
 297   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 298   "")
 299
 300 (define_expand "re<mode>2"
 301   [(set (match_operand:VEC_F 0 "vfloat_operand")
 302         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 303                       UNSPEC_FRES))]
 304   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 305   "")
 306
 307 (define_expand "ftrunc<mode>2"
 308   [(set (match_operand:VEC_F 0 "vfloat_operand")
 309         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 310   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 311   "")
 312
 313 (define_expand "vector_ceil<mode>2"
 314   [(set (match_operand:VEC_F 0 "vfloat_operand")
 315         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 316                       UNSPEC_FRIP))]
 317   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 318   "")
 319
 320 (define_expand "vector_floor<mode>2"
 321   [(set (match_operand:VEC_F 0 "vfloat_operand")
 322         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 323                       UNSPEC_FRIM))]
 324   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 325   "")
 326
 327 (define_expand "vector_btrunc<mode>2"
 328   [(set (match_operand:VEC_F 0 "vfloat_operand")
 329         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 330   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 331   "")
 332
 333 (define_expand "vector_copysign<mode>3"
 334   [(set (match_operand:VEC_F 0 "vfloat_operand")
 335         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")
 336                        (match_operand:VEC_F 2 "vfloat_operand")] UNSPEC_COPYSIGN))]
 337   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 338 {
 339   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 340     {
 341       emit_insn (gen_altivec_copysign_v4sf3 (operands[0], operands[1],
 342                                              operands[2]));
 343       DONE;
 344     }
 345 })
 346
 347 \f
 348 ;; Vector comparisons
 349 (define_expand "vcond<mode><mode>"
 350   [(set (match_operand:VEC_F 0 "vfloat_operand")
 351         (if_then_else:VEC_F
 352          (match_operator 3 "comparison_operator"
 353                          [(match_operand:VEC_F 4 "vfloat_operand")
 354                           (match_operand:VEC_F 5 "vfloat_operand")])
 355          (match_operand:VEC_F 1 "vfloat_operand")
 356          (match_operand:VEC_F 2 "vfloat_operand")))]
 357   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 358 {
 359   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 360                                     operands[3], operands[4], operands[5]))
 361     DONE;
 362   else
 363     gcc_unreachable ();
 364 })
 365
 366 (define_expand "vcond<mode><mode>"
 367   [(set (match_operand:VEC_I 0 "vint_operand")
 368         (if_then_else:VEC_I
 369          (match_operator 3 "comparison_operator"
 370                          [(match_operand:VEC_I 4 "vint_operand")
 371                           (match_operand:VEC_I 5 "vint_operand")])
 372          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 373          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 374   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 375 {
 376   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 377                                     operands[3], operands[4], operands[5]))
 378     DONE;
 379   else
 380     gcc_unreachable ();
 381 })
 382
 383 (define_expand "vcondv4sfv4si"
 384   [(set (match_operand:V4SF 0 "vfloat_operand")
 385         (if_then_else:V4SF
 386          (match_operator 3 "comparison_operator"
 387                          [(match_operand:V4SI 4 "vint_operand")
 388                           (match_operand:V4SI 5 "vint_operand")])
 389          (match_operand:V4SF 1 "vfloat_operand")
 390          (match_operand:V4SF 2 "vfloat_operand")))]
 391   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 392    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 393 {
 394   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 395                                     operands[3], operands[4], operands[5]))
 396     DONE;
 397   else
 398     gcc_unreachable ();
 399 })
 400
 401 (define_expand "vcondv4siv4sf"
 402   [(set (match_operand:V4SI 0 "vint_operand")
 403         (if_then_else:V4SI
 404          (match_operator 3 "comparison_operator"
 405                          [(match_operand:V4SF 4 "vfloat_operand")
 406                           (match_operand:V4SF 5 "vfloat_operand")])
 407          (match_operand:V4SI 1 "vint_operand")
 408          (match_operand:V4SI 2 "vint_operand")))]
 409   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 410    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 411 {
 412   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 413                                     operands[3], operands[4], operands[5]))
 414     DONE;
 415   else
 416     gcc_unreachable ();
 417 })
 418
 419 (define_expand "vcondv2dfv2di"
 420   [(set (match_operand:V2DF 0 "vfloat_operand")
 421         (if_then_else:V2DF
 422          (match_operator 3 "comparison_operator"
 423                          [(match_operand:V2DI 4 "vint_operand")
 424                           (match_operand:V2DI 5 "vint_operand")])
 425          (match_operand:V2DF 1 "vfloat_operand")
 426          (match_operand:V2DF 2 "vfloat_operand")))]
 427   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 428    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 429 {
 430   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 431                                     operands[3], operands[4], operands[5]))
 432     DONE;
 433   else
 434     gcc_unreachable ();
 435 })
 436
 437 (define_expand "vcondv2div2df"
 438   [(set (match_operand:V2DI 0 "vint_operand")
 439         (if_then_else:V2DI
 440          (match_operator 3 "comparison_operator"
 441                          [(match_operand:V2DF 4 "vfloat_operand")
 442                           (match_operand:V2DF 5 "vfloat_operand")])
 443          (match_operand:V2DI 1 "vint_operand")
 444          (match_operand:V2DI 2 "vint_operand")))]
 445   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 446    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 447 {
 448   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 449                                     operands[3], operands[4], operands[5]))
 450     DONE;
 451   else
 452     gcc_unreachable ();
 453 })
 454
 455 (define_expand "vcondu<mode><mode>"
 456   [(set (match_operand:VEC_I 0 "vint_operand")
 457         (if_then_else:VEC_I
 458          (match_operator 3 "comparison_operator"
 459                          [(match_operand:VEC_I 4 "vint_operand")
 460                           (match_operand:VEC_I 5 "vint_operand")])
 461          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 462          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 463   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 464 {
 465   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 466                                     operands[3], operands[4], operands[5]))
 467     DONE;
 468   else
 469     gcc_unreachable ();
 470 })
 471
 472 (define_expand "vconduv4sfv4si"
 473   [(set (match_operand:V4SF 0 "vfloat_operand")
 474         (if_then_else:V4SF
 475          (match_operator 3 "comparison_operator"
 476                          [(match_operand:V4SI 4 "vint_operand")
 477                           (match_operand:V4SI 5 "vint_operand")])
 478          (match_operand:V4SF 1 "vfloat_operand")
 479          (match_operand:V4SF 2 "vfloat_operand")))]
 480   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 481    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 482 {
 483   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 484                                     operands[3], operands[4], operands[5]))
 485     DONE;
 486   else
 487     gcc_unreachable ();
 488 })
 489
 490 (define_expand "vconduv2dfv2di"
 491   [(set (match_operand:V2DF 0 "vfloat_operand")
 492         (if_then_else:V2DF
 493          (match_operator 3 "comparison_operator"
 494                          [(match_operand:V2DI 4 "vint_operand")
 495                           (match_operand:V2DI 5 "vint_operand")])
 496          (match_operand:V2DF 1 "vfloat_operand")
 497          (match_operand:V2DF 2 "vfloat_operand")))]
 498   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 499    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 500 {
 501   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 502                                     operands[3], operands[4], operands[5]))
 503     DONE;
 504   else
 505     gcc_unreachable ();
 506 })
 507
 508 ;; To support vector condition vectorization, define vcond_mask and vec_cmp.
 509
 510 ;; Same mode for condition true/false values and predicate operand.
 511 (define_expand "vcond_mask_<mode><mode>"
 512   [(match_operand:VEC_I 0 "vint_operand")
 513    (match_operand:VEC_I 1 "vint_operand")
 514    (match_operand:VEC_I 2 "vint_operand")
 515    (match_operand:VEC_I 3 "vint_operand")]
 516   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 517 {
 518   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 519                                   operands[3]));
 520   DONE;
 521 })
 522
 523 ;; Condition true/false values are float but predicate operand is of
 524 ;; type integer vector with same element size.
 525 (define_expand "vcond_mask_<mode><VEC_int>"
 526   [(match_operand:VEC_F 0 "vfloat_operand")
 527    (match_operand:VEC_F 1 "vfloat_operand")
 528    (match_operand:VEC_F 2 "vfloat_operand")
 529    (match_operand:<VEC_INT> 3 "vint_operand")]
 530   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 531 {
 532   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 533                                   gen_lowpart (<MODE>mode, operands[3])));
 534   DONE;
 535 })
 536
 537 ;; For signed integer vectors comparison.
 538 (define_expand "vec_cmp<mode><mode>"
 539   [(set (match_operand:VEC_IC 0 "vint_operand")
 540         (match_operator 1 "signed_or_equality_comparison_operator"
 541           [(match_operand:VEC_IC 2 "vint_operand")
 542            (match_operand:VEC_IC 3 "vint_operand")]))]
 543   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 544 {
 545   enum rtx_code code = GET_CODE (operands[1]);
 546   rtx tmp = gen_reg_rtx (<MODE>mode);
 547   switch (code)
 548     {
 549     case NE:
 550       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 551       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 552       break;
 553     case EQ:
 554       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 555       break;
 556     case GE:
 557       emit_insn (gen_vector_nlt<mode> (operands[0],operands[2], operands[3],
 558                                        tmp));
 559       break;
 560     case GT:
 561       emit_insn (gen_vector_gt<mode> (operands[0], operands[2], operands[3]));
 562       break;
 563     case LE:
 564       emit_insn (gen_vector_ngt<mode> (operands[0], operands[2], operands[3],
 565                                        tmp));
 566       break;
 567     case LT:
 568       emit_insn (gen_vector_gt<mode> (operands[0], operands[3], operands[2]));
 569       break;
 570     default:
 571       gcc_unreachable ();
 572       break;
 573     }
 574   DONE;
 575 })
 576
 577 ;; For unsigned integer vectors comparison.
 578 (define_expand "vec_cmpu<mode><mode>"
 579   [(set (match_operand:VEC_IC 0 "vint_operand")
 580         (match_operator 1 "unsigned_or_equality_comparison_operator"
 581           [(match_operand:VEC_IC 2 "vint_operand")
 582            (match_operand:VEC_IC 3 "vint_operand")]))]
 583   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 584 {
 585   enum rtx_code code = GET_CODE (operands[1]);
 586   rtx tmp = gen_reg_rtx (<MODE>mode);
 587   switch (code)
 588     {
 589     case NE:
 590       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 591       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 592       break;
 593     case EQ:
 594       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 595       break;
 596     case GEU:
 597       emit_insn (gen_vector_nltu<mode> (operands[0], operands[2], operands[3],
 598                                         tmp));
 599       break;
 600     case GTU:
 601       emit_insn (gen_vector_gtu<mode> (operands[0], operands[2], operands[3]));
 602       break;
 603     case LEU:
 604       emit_insn (gen_vector_ngtu<mode> (operands[0], operands[2], operands[3],
 605                                         tmp));
 606       break;
 607     case LTU:
 608       emit_insn (gen_vector_gtu<mode> (operands[0], operands[3], operands[2]));
 609       break;
 610     default:
 611       gcc_unreachable ();
 612       break;
 613     }
 614   DONE;
 615 })
 616
 617 ;; For float point vectors comparison.
 618 (define_expand "vec_cmp<mode><VEC_int>"
 619   [(set (match_operand:<VEC_INT> 0 "vint_operand")
 620          (match_operator 1 "comparison_operator"
 621             [(match_operand:VEC_F 2 "vfloat_operand")
 622             (match_operand:VEC_F 3 "vfloat_operand")]))]
 623   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 624 {
 625   enum rtx_code code = GET_CODE (operands[1]);
 626   rtx res = gen_reg_rtx (<MODE>mode);
 627   switch (code)
 628     {
 629     case NE:
 630       emit_insn (gen_vector_ne<mode> (res, operands[2], operands[3]));
 631       break;
 632     case EQ:
 633       emit_insn (gen_vector_eq<mode> (res, operands[2], operands[3]));
 634       break;
 635     case GE:
 636       emit_insn (gen_vector_ge<mode> (res, operands[2], operands[3]));
 637       break;
 638     case GT:
 639       emit_insn (gen_vector_gt<mode> (res, operands[2], operands[3]));
 640       break;
 641     case LE:
 642       emit_insn (gen_vector_le<mode> (res, operands[2], operands[3]));
 643       break;
 644     case LT:
 645       emit_insn (gen_vector_lt<mode> (res, operands[2], operands[3]));
 646       break;
 647     case LTGT:
 648       emit_insn (gen_vector_ltgt<mode> (res, operands[2], operands[3]));
 649       break;
 650     case UNORDERED:
 651       emit_insn (gen_vector_unordered<mode> (res, operands[2], operands[3]));
 652       break;
 653     case ORDERED:
 654       emit_insn (gen_vector_ordered<mode> (res, operands[2], operands[3]));
 655       break;
 656     case UNEQ:
 657       emit_insn (gen_vector_uneq<mode> (res, operands[2], operands[3]));
 658       break;
 659     case UNGE:
 660       emit_insn (gen_vector_unge<mode> (res, operands[2], operands[3]));
 661       break;
 662     case UNGT:
 663       emit_insn (gen_vector_ungt<mode> (res, operands[2], operands[3]));
 664       break;
 665     case UNLE:
 666       emit_insn (gen_vector_unle<mode> (res, operands[2], operands[3]));
 667       break;
 668     case UNLT:
 669       emit_insn (gen_vector_unlt<mode> (res, operands[2], operands[3]));
 670       break;
 671
 672     default:
 673       gcc_unreachable ();
 674     }
 675
 676   emit_insn (gen_move_insn (operands[0], gen_lowpart (<VEC_INT>mode, res)));
 677   DONE;
 678 })
 679
 680 (define_expand "vector_eq<mode>"
 681   [(set (match_operand:VEC_C 0 "vlogical_operand")
 682         (eq:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 683                   (match_operand:VEC_C 2 "vlogical_operand")))]
 684   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 685   "")
 686
 687 (define_expand "vector_gt<mode>"
 688   [(set (match_operand:VEC_C 0 "vlogical_operand")
 689         (gt:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 690                   (match_operand:VEC_C 2 "vlogical_operand")))]
 691   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 692   "")
 693
 694 ; >= for integer vectors: swap operands and apply not-greater-than
 695 (define_expand "vector_nlt<mode>"
 696   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 697         (gt:VEC_IC (match_operand:VEC_IC 2 "vlogical_operand")
 698                    (match_operand:VEC_IC 1 "vlogical_operand")))
 699    (set (match_operand:VEC_IC 0 "vlogical_operand")
 700         (not:VEC_IC (match_dup 3)))]
 701   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 702 {
 703   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 704 })
 705
 706 (define_expand "vector_gtu<mode>"
 707   [(set (match_operand:VEC_IC 0 "vint_operand")
 708         (gtu:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
 709                     (match_operand:VEC_IC 2 "vint_operand")))]
 710   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 711   "")
 712
 713 ; >= for integer vectors: swap operands and apply not-greater-than
 714 (define_expand "vector_nltu<mode>"
 715   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 716         (gtu:VEC_IC (match_operand:VEC_IC 2 "vlogical_operand")
 717                     (match_operand:VEC_IC 1 "vlogical_operand")))
 718    (set (match_operand:VEC_IC 0 "vlogical_operand")
 719         (not:VEC_IC (match_dup 3)))]
 720   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 721 {
 722   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 723 })
 724
 725 (define_expand "vector_geu<mode>"
 726   [(set (match_operand:VEC_IC 0 "vint_operand")
 727         (geu:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
 728                     (match_operand:VEC_IC 2 "vint_operand")))]
 729   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 730   "")
 731
 732 ; <= for integer vectors: apply not-greater-than
 733 (define_expand "vector_ngt<mode>"
 734   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 735         (gt:VEC_IC (match_operand:VEC_IC 1 "vlogical_operand")
 736                    (match_operand:VEC_IC 2 "vlogical_operand")))
 737    (set (match_operand:VEC_IC 0 "vlogical_operand")
 738         (not:VEC_IC (match_dup 3)))]
 739   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 740 {
 741   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 742 })
 743
 744 (define_expand "vector_ngtu<mode>"
 745   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 746         (gtu:VEC_IC (match_operand:VEC_IC 1 "vlogical_operand")
 747                     (match_operand:VEC_IC 2 "vlogical_operand")))
 748    (set (match_operand:VEC_IC 0 "vlogical_operand")
 749         (not:VEC_IC (match_dup 3)))]
 750   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 751 {
 752   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 753 })
 754
 755 ; There are 14 possible vector FP comparison operators, gt and eq of them have
 756 ; been expanded above, so just support 12 remaining operators here.
 757
 758 ; For ge:
 759 (define_expand "vector_ge<mode>"
 760   [(set (match_operand:VEC_F 0 "vlogical_operand")
 761         (ge:VEC_F (match_operand:VEC_F 1 "vlogical_operand")
 762                   (match_operand:VEC_F 2 "vlogical_operand")))]
 763   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 764   "")
 765
 766 ; For lt/le/ne/ungt/unge/unlt/unle:
 767 ; lt(a,b)   = gt(b,a)
 768 ; le(a,b)   = ge(b,a)
 769 ; unge(a,b) = ~lt(a,b)
 770 ; unle(a,b) = ~gt(a,b)
 771 ; ne(a,b)   = ~eq(a,b)
 772 ; ungt(a,b) = ~le(a,b)
 773 ; unlt(a,b) = ~ge(a,b)
 774 (define_insn_and_split "vector_<code><mode>"
 775   [(set (match_operand:VEC_F 0 "vfloat_operand")
 776         (vector_fp_comparison_simple:VEC_F
 777            (match_operand:VEC_F 1 "vfloat_operand")
 778            (match_operand:VEC_F 2 "vfloat_operand")))]
 779   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 780   "#"
 781   "&& can_create_pseudo_p ()"
 782   [(pc)]
 783 {
 784   enum rtx_code cond = <CODE>;
 785   bool need_invert = false;
 786
 787   if (cond == UNLE || cond == UNLT || cond == NE || cond == UNGE
 788       || cond == UNGT)
 789     {
 790       cond = reverse_condition_maybe_unordered (cond);
 791       need_invert = true;
 792     }
 793
 794   if (cond == LT || cond == LE)
 795     {
 796       cond = swap_condition (cond);
 797       std::swap (operands[1], operands[2]);
 798     }
 799
 800   gcc_assert (cond == EQ || cond == GE || cond == GT);
 801
 802   rtx comp = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 803
 804   if (need_invert)
 805     {
 806       rtx res = gen_reg_rtx (<MODE>mode);
 807       emit_insn (gen_rtx_SET (res, comp));
 808       emit_insn (gen_one_cmpl<mode>2 (operands[0], res));
 809     }
 810   else
 811     emit_insn (gen_rtx_SET (operands[0], comp));
 812
 813   DONE;
 814 })
 815
 816 ; For ltgt/uneq/ordered/unordered:
 817 ; ltgt: gt(a,b) | gt(b,a)
 818 ; uneq: ~(gt(a,b) | gt(b,a))
 819 ; ordered: ge(a,b) | ge(b,a)
 820 ; unordered: ~(ge(a,b) | ge(b,a))
 821 (define_insn_and_split "vector_<code><mode>"
 822   [(set (match_operand:VEC_F 0 "vfloat_operand")
 823         (vector_fp_comparison_complex:VEC_F
 824            (match_operand:VEC_F 1 "vfloat_operand")
 825            (match_operand:VEC_F 2 "vfloat_operand")))]
 826   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 827   "#"
 828   "&& can_create_pseudo_p ()"
 829   [(pc)]
 830 {
 831   enum rtx_code cond = <CODE>;
 832   bool need_invert = false;
 833
 834   if (cond == UNORDERED || cond == UNEQ)
 835     {
 836       cond = reverse_condition_maybe_unordered (cond);
 837       need_invert = true;
 838     }
 839
 840   if (cond == LTGT)
 841     cond = GT;
 842   else if (cond == ORDERED)
 843     cond = GE;
 844   else
 845     gcc_unreachable ();
 846
 847   rtx comp1 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 848   rtx res1 = gen_reg_rtx (<MODE>mode);
 849   emit_insn (gen_rtx_SET (res1, comp1));
 850   rtx comp2 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[2], operands[1]);
 851   rtx res2 = gen_reg_rtx (<MODE>mode);
 852   emit_insn (gen_rtx_SET (res2, comp2));
 853
 854   if (need_invert)
 855     {
 856       rtx not1 = gen_rtx_fmt_e (NOT, <MODE>mode, res1);
 857       rtx not2 = gen_rtx_fmt_e (NOT, <MODE>mode, res2);
 858       rtx comp3 = gen_rtx_fmt_ee (AND, <MODE>mode, not1, not2);
 859       emit_insn (gen_rtx_SET (operands[0], comp3));
 860     }
 861   else
 862     emit_insn (gen_ior<mode>3 (operands[0], res1, res2));
 863
 864   DONE;
 865 })
 866
 867 ;; Note the arguments for __builtin_altivec_vsel are op2, op1, mask
 868 ;; which is in the reverse order that we want
 869 (define_expand "vector_select_<mode>"
 870   [(set (match_operand:VEC_L 0 "vlogical_operand")
 871         (ior:VEC_L
 872           (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
 873                      (match_operand:VEC_L 1 "vlogical_operand"))
 874           (and:VEC_L (match_dup 3)
 875                      (match_operand:VEC_L 2 "vlogical_operand"))))]
 876   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 877
 878 (define_expand "vector_select_<mode>_uns"
 879   [(set (match_operand:VEC_L 0 "vlogical_operand")
 880         (ior:VEC_L
 881           (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
 882                      (match_operand:VEC_L 1 "vlogical_operand"))
 883           (and:VEC_L (match_dup 3)
 884                      (match_operand:VEC_L 2 "vlogical_operand"))))]
 885   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 886
 887 ;; Expansions that compare vectors producing a vector result and a predicate,
 888 ;; setting CR6 to indicate a combined status
 889 (define_expand "vector_eq_<mode>_p"
 890   [(parallel
 891     [(set (reg:CC CR6_REGNO)
 892           (unspec:CC [(eq:CC (match_operand:VEC_A 1 "vlogical_operand")
 893                              (match_operand:VEC_A 2 "vlogical_operand"))]
 894                      UNSPEC_PREDICATE))
 895      (set (match_operand:VEC_A 0 "vlogical_operand")
 896           (eq:VEC_A (match_dup 1)
 897                     (match_dup 2)))])]
 898   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 899   "")
 900
 901 (define_expand "vector_eq_v1ti_p"
 902   [(parallel
 903     [(set (reg:CC CR6_REGNO)
 904           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
 905                              (match_operand:V1TI 2 "altivec_register_operand"))]
 906                      UNSPEC_PREDICATE))
 907      (set (match_operand:V1TI 0 "vlogical_operand")
 908           (eq:V1TI (match_dup 1)
 909                    (match_dup 2)))])]
 910   "TARGET_POWER10"
 911   "")
 912
 913 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 914 ;; implementation of the vec_all_ne built-in functions on Power9.
 915 (define_expand "vector_ne_<mode>_p"
 916   [(parallel
 917     [(set (reg:CC CR6_REGNO)
 918           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 919                              (match_operand:VI 2 "vlogical_operand"))]
 920            UNSPEC_PREDICATE))
 921      (set (match_dup 3)
 922           (ne:VI (match_dup 1)
 923                  (match_dup 2)))])
 924    (set (match_operand:SI 0 "register_operand" "=r")
 925         (lt:SI (reg:CC CR6_REGNO)
 926                (const_int 0)))]
 927   "TARGET_P9_VECTOR"
 928 {
 929   operands[3] = gen_reg_rtx (<MODE>mode);
 930 })
 931
 932 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 933 ;; implementation of the vec_any_eq built-in functions on Power9.
 934 (define_expand "vector_ae_<mode>_p"
 935   [(parallel
 936     [(set (reg:CC CR6_REGNO)
 937           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 938                              (match_operand:VI 2 "vlogical_operand"))]
 939            UNSPEC_PREDICATE))
 940      (set (match_dup 3)
 941           (ne:VI (match_dup 1)
 942                  (match_dup 2)))])
 943    (set (match_operand:SI 0 "register_operand" "=r")
 944         (lt:SI (reg:CC CR6_REGNO)
 945                (const_int 0)))
 946    (set (match_dup 0)
 947         (xor:SI (match_dup 0)
 948                 (const_int 1)))]
 949   "TARGET_P9_VECTOR"
 950 {
 951   operands[3] = gen_reg_rtx (<MODE>mode);
 952 })
 953
 954 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 955 ;; implementation of the vec_all_nez and vec_any_eqz built-in
 956 ;; functions on Power9.
 957 (define_expand "vector_nez_<mode>_p"
 958   [(parallel
 959     [(set (reg:CC CR6_REGNO)
 960           (unspec:CC [(unspec:VI
 961                        [(match_operand:VI 1 "vlogical_operand")
 962                         (match_operand:VI 2 "vlogical_operand")]
 963                        UNSPEC_NEZ_P)]
 964            UNSPEC_PREDICATE))
 965      (set (match_operand:VI 0 "vlogical_operand")
 966           (unspec:VI [(match_dup 1)
 967                       (match_dup 2)]
 968            UNSPEC_NEZ_P))])]
 969   "TARGET_P9_VECTOR"
 970   "")
 971
 972 ;; This expansion handles the V2DI mode in the implementation of the
 973 ;; vec_all_ne built-in function on Power9.
 974 ;;
 975 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
 976 ;; this expands into the same rtl that would be used for the Power8
 977 ;; architecture.
 978 (define_expand "vector_ne_v2di_p"
 979   [(parallel
 980     [(set (reg:CC CR6_REGNO)
 981           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
 982                              (match_operand:V2DI 2 "vlogical_operand"))]
 983                      UNSPEC_PREDICATE))
 984      (set (match_dup 3)
 985           (eq:V2DI (match_dup 1)
 986                    (match_dup 2)))])
 987    (set (match_operand:SI 0 "register_operand" "=r")
 988         (eq:SI (reg:CC CR6_REGNO)
 989                (const_int 0)))]
 990   "TARGET_P9_VECTOR"
 991 {
 992   operands[3] = gen_reg_rtx (V2DImode);
 993 })
 994
 995 (define_expand "vector_ne_v1ti_p"
 996   [(parallel
 997     [(set (reg:CC CR6_REGNO)
 998           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
 999                              (match_operand:V1TI 2 "altivec_register_operand"))]
1000                      UNSPEC_PREDICATE))
1001      (set (match_dup 3)
1002           (eq:V1TI (match_dup 1)
1003                    (match_dup 2)))])
1004    (set (match_operand:SI 0 "register_operand" "=r")
1005         (eq:SI (reg:CC CR6_REGNO)
1006                (const_int 0)))]
1007   "TARGET_POWER10"
1008 {
1009   operands[3] = gen_reg_rtx (V1TImode);
1010 })
1011
1012 ;; This expansion handles the V2DI mode in the implementation of the
1013 ;; vec_any_eq built-in function on Power9.
1014 ;;
1015 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
1016 ;; this expands into the same rtl that would be used for the Power8
1017 ;; architecture.
1018 (define_expand "vector_ae_v2di_p"
1019   [(parallel
1020     [(set (reg:CC CR6_REGNO)
1021           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
1022                              (match_operand:V2DI 2 "vlogical_operand"))]
1023                      UNSPEC_PREDICATE))
1024      (set (match_dup 3)
1025           (eq:V2DI (match_dup 1)
1026                    (match_dup 2)))])
1027    (set (match_operand:SI 0 "register_operand" "=r")
1028         (eq:SI (reg:CC CR6_REGNO)
1029                (const_int 0)))
1030    (set (match_dup 0)
1031         (xor:SI (match_dup 0)
1032                 (const_int 1)))]
1033   "TARGET_P9_VECTOR"
1034 {
1035   operands[3] = gen_reg_rtx (V2DImode);
1036 })
1037
1038 (define_expand "vector_ae_v1ti_p"
1039   [(parallel
1040     [(set (reg:CC CR6_REGNO)
1041           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
1042                              (match_operand:V1TI 2 "altivec_register_operand"))]
1043                      UNSPEC_PREDICATE))
1044      (set (match_dup 3)
1045           (eq:V1TI (match_dup 1)
1046                    (match_dup 2)))])
1047    (set (match_operand:SI 0 "register_operand" "=r")
1048         (eq:SI (reg:CC CR6_REGNO)
1049                (const_int 0)))
1050    (set (match_dup 0)
1051         (xor:SI (match_dup 0)
1052                 (const_int 1)))]
1053   "TARGET_POWER10"
1054 {
1055   operands[3] = gen_reg_rtx (V1TImode);
1056 })
1057
1058 ;; This expansion handles the V4SF and V2DF modes in the Power9
1059 ;; implementation of the vec_all_ne built-in functions.  Note that the
1060 ;; expansions for this pattern with these modes makes no use of power9-
1061 ;; specific instructions since there are no new power9 instructions
1062 ;; for vector compare not equal with floating point arguments.
1063 (define_expand "vector_ne_<mode>_p"
1064   [(parallel
1065     [(set (reg:CC CR6_REGNO)
1066           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1067                              (match_operand:VEC_F 2 "vlogical_operand"))]
1068                      UNSPEC_PREDICATE))
1069      (set (match_dup 3)
1070           (eq:VEC_F (match_dup 1)
1071                     (match_dup 2)))])
1072    (set (match_operand:SI 0 "register_operand" "=r")
1073         (eq:SI (reg:CC CR6_REGNO)
1074                (const_int 0)))]
1075   "TARGET_P9_VECTOR"
1076 {
1077   operands[3] = gen_reg_rtx (<MODE>mode);
1078 })
1079
1080 ;; This expansion handles the V4SF and V2DF modes in the Power9
1081 ;; implementation of the vec_any_eq built-in functions.  Note that the
1082 ;; expansions for this pattern with these modes makes no use of power9-
1083 ;; specific instructions since there are no new power9 instructions
1084 ;; for vector compare not equal with floating point arguments.
1085 (define_expand "vector_ae_<mode>_p"
1086   [(parallel
1087     [(set (reg:CC CR6_REGNO)
1088           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1089                              (match_operand:VEC_F 2 "vlogical_operand"))]
1090                      UNSPEC_PREDICATE))
1091      (set (match_dup 3)
1092           (eq:VEC_F (match_dup 1)
1093                     (match_dup 2)))])
1094    (set (match_operand:SI 0 "register_operand" "=r")
1095         (eq:SI (reg:CC CR6_REGNO)
1096                (const_int 0)))
1097    (set (match_dup 0)
1098         (xor:SI (match_dup 0)
1099                 (const_int 1)))]
1100   "TARGET_P9_VECTOR"
1101 {
1102   operands[3] = gen_reg_rtx (<MODE>mode);
1103 })
1104
1105 (define_expand "vector_gt_<mode>_p"
1106   [(parallel
1107     [(set (reg:CC CR6_REGNO)
1108           (unspec:CC [(gt:CC (match_operand:VEC_A 1 "vlogical_operand")
1109                              (match_operand:VEC_A 2 "vlogical_operand"))]
1110                      UNSPEC_PREDICATE))
1111      (set (match_operand:VEC_A 0 "vlogical_operand")
1112           (gt:VEC_A (match_dup 1)
1113                     (match_dup 2)))])]
1114   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1115   "")
1116
1117 (define_expand "vector_gt_v1ti_p"
1118   [(parallel
1119     [(set (reg:CC CR6_REGNO)
1120           (unspec:CC [(gt:CC (match_operand:V1TI 1 "vlogical_operand")
1121                              (match_operand:V1TI 2 "vlogical_operand"))]
1122                      UNSPEC_PREDICATE))
1123      (set (match_operand:V1TI 0 "vlogical_operand")
1124           (gt:V1TI (match_dup 1)
1125                    (match_dup 2)))])]
1126   "TARGET_POWER10"
1127   "")
1128
1129 (define_expand "vector_ge_<mode>_p"
1130   [(parallel
1131     [(set (reg:CC CR6_REGNO)
1132           (unspec:CC [(ge:CC (match_operand:VEC_F 1 "vfloat_operand")
1133                              (match_operand:VEC_F 2 "vfloat_operand"))]
1134                      UNSPEC_PREDICATE))
1135      (set (match_operand:VEC_F 0 "vfloat_operand")
1136           (ge:VEC_F (match_dup 1)
1137                     (match_dup 2)))])]
1138   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1139   "")
1140
1141 (define_expand "vector_gtu_<mode>_p"
1142   [(parallel
1143     [(set (reg:CC CR6_REGNO)
1144           (unspec:CC [(gtu:CC (match_operand:VEC_IC 1 "vint_operand")
1145                               (match_operand:VEC_IC 2 "vint_operand"))]
1146                      UNSPEC_PREDICATE))
1147      (set (match_operand:VEC_IC 0 "vlogical_operand")
1148           (gtu:VEC_IC (match_dup 1)
1149                       (match_dup 2)))])]
1150   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1151   "")
1152
1153 ;; AltiVec/VSX predicates.
1154
1155 ;; This expansion is triggered during expansion of predicate built-in
1156 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1157 ;; altivec_expand_predicate_builtin() function when the value of the
1158 ;; integer constant first argument equals zero (aka __CR6_EQ in altivec.h).
1159 (define_expand "cr6_test_for_zero"
1160   [(set (match_operand:SI 0 "register_operand" "=r")
1161         (eq:SI (reg:CC CR6_REGNO)
1162                (const_int 0)))]
1163   "TARGET_ALTIVEC || TARGET_VSX"
1164   "")
1165
1166 ;; This expansion is triggered during expansion of predicate built-in
1167 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1168 ;; altivec_expand_predicate_builtin() function when the value of the
1169 ;; integer constant first argument equals one (aka __CR6_EQ_REV in altivec.h).
1170 (define_expand "cr6_test_for_zero_reverse"
1171   [(set (match_operand:SI 0 "register_operand" "=r")
1172         (eq:SI (reg:CC CR6_REGNO)
1173                (const_int 0)))
1174    (set (match_dup 0)
1175         (xor:SI (match_dup 0)
1176                 (const_int 1)))]
1177   "TARGET_ALTIVEC || TARGET_VSX"
1178   "")
1179
1180 ;; This expansion is triggered during expansion of predicate built-in
1181 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1182 ;; altivec_expand_predicate_builtin() function when the value of the
1183 ;; integer constant first argument equals two (aka __CR6_LT in altivec.h).
1184 (define_expand "cr6_test_for_lt"
1185   [(set (match_operand:SI 0 "register_operand" "=r")
1186         (lt:SI (reg:CC CR6_REGNO)
1187                (const_int 0)))]
1188   "TARGET_ALTIVEC || TARGET_VSX"
1189   "")
1190
1191 ;; This expansion is triggered during expansion of predicate built-in
1192 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1193 ;; altivec_expand_predicate_builtin() function when the value of the
1194 ;; integer constant first argument equals three
1195 ;; (aka __CR6_LT_REV in altivec.h).
1196 (define_expand "cr6_test_for_lt_reverse"
1197   [(set (match_operand:SI 0 "register_operand" "=r")
1198         (lt:SI (reg:CC CR6_REGNO)
1199                (const_int 0)))
1200    (set (match_dup 0)
1201         (xor:SI (match_dup 0)
1202                 (const_int 1)))]
1203   "TARGET_ALTIVEC || TARGET_VSX"
1204   "")
1205
1206 \f
1207 ;; Vector count leading zeros
1208 (define_expand "clz<mode>2"
1209   [(set (match_operand:VEC_I 0 "register_operand")
1210         (clz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1211   "TARGET_P8_VECTOR")
1212
1213 ;; Vector count trailing zeros
1214 (define_expand "ctz<mode>2"
1215   [(set (match_operand:VEC_I 0 "register_operand")
1216         (ctz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1217   "TARGET_P9_VECTOR")
1218
1219 ;; Vector population count
1220 (define_expand "popcount<mode>2"
1221   [(set (match_operand:VEC_I 0 "register_operand")
1222         (popcount:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1223   "TARGET_P8_VECTOR")
1224
1225 ;; Vector parity
1226 (define_expand "parity<mode>2"
1227   [(set (match_operand:VEC_IP 0 "register_operand")
1228         (parity:VEC_IP (match_operand:VEC_IP 1 "register_operand")))]
1229   "TARGET_P9_VECTOR")
1230
1231 \f
1232 ;; Same size conversions
1233 (define_expand "float<VEC_int><mode>2"
1234   [(set (match_operand:VEC_F 0 "vfloat_operand")
1235         (float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1236   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1237 {
1238   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1239     {
1240       emit_insn (gen_altivec_vcfsx (operands[0], operands[1], const0_rtx));
1241       DONE;
1242     }
1243 })
1244
1245 (define_expand "floatuns<VEC_int><mode>2"
1246   [(set (match_operand:VEC_F 0 "vfloat_operand")
1247         (unsigned_float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1248   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1249 {
1250   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1251     {
1252       emit_insn (gen_altivec_vcfux (operands[0], operands[1], const0_rtx));
1253       DONE;
1254     }
1255 })
1256
1257 (define_expand "fix_trunc<mode><VEC_int>2"
1258   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1259         (fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1260   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1261 {
1262   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1263     {
1264       emit_insn (gen_altivec_vctsxs (operands[0], operands[1], const0_rtx));
1265       DONE;
1266     }
1267 })
1268
1269 (define_expand "fixuns_trunc<mode><VEC_int>2"
1270   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1271         (unsigned_fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1272   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1273 {
1274   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1275     {
1276       emit_insn (gen_altivec_vctuxs (operands[0], operands[1], const0_rtx));
1277       DONE;
1278     }
1279 })
1280
1281 \f
1282 ;; Vector initialization, set, extract
1283 (define_expand "vec_init<mode><VEC_base_l>"
1284   [(match_operand:VEC_E 0 "vlogical_operand")
1285    (match_operand:VEC_E 1 "")]
1286   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1287 {
1288   rs6000_expand_vector_init (operands[0], operands[1]);
1289   DONE;
1290 })
1291
1292 (define_expand "vec_set<mode>"
1293   [(match_operand:VEC_E 0 "vlogical_operand")
1294    (match_operand:<VEC_base> 1 "register_operand")
1295    (match_operand 2 "vec_set_index_operand")]
1296   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1297 {
1298   rs6000_expand_vector_set (operands[0], operands[1], operands[2]);
1299   DONE;
1300 })
1301
1302 (define_expand "vec_extract<mode><VEC_base_l>"
1303   [(match_operand:<VEC_base> 0 "register_operand")
1304    (match_operand:VEC_E 1 "vlogical_operand")
1305    (match_operand 2 "const_int_operand")]
1306   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1307 {
1308   rs6000_expand_vector_extract (operands[0], operands[1], operands[2]);
1309   DONE;
1310 })
1311 \f
1312 ;; Convert double word types to single word types
1313 (define_expand "vec_pack_trunc_v2df"
1314   [(match_operand:V4SF 0 "vfloat_operand")
1315    (match_operand:V2DF 1 "vfloat_operand")
1316    (match_operand:V2DF 2 "vfloat_operand")]
1317   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1318 {
1319   rtx r1 = gen_reg_rtx (V4SFmode);
1320   rtx r2 = gen_reg_rtx (V4SFmode);
1321
1322   emit_insn (gen_vsx_xvcvdpsp (r1, operands[1]));
1323   emit_insn (gen_vsx_xvcvdpsp (r2, operands[2]));
1324   rs6000_expand_extract_even (operands[0], r1, r2);
1325   DONE;
1326 })
1327
1328 (define_expand "vec_pack_sfix_trunc_v2df"
1329   [(match_operand:V4SI 0 "vint_operand")
1330    (match_operand:V2DF 1 "vfloat_operand")
1331    (match_operand:V2DF 2 "vfloat_operand")]
1332   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1333 {
1334   rtx r1 = gen_reg_rtx (V4SImode);
1335   rtx r2 = gen_reg_rtx (V4SImode);
1336
1337   emit_insn (gen_vsx_xvcvdpsxws (r1, operands[1]));
1338   emit_insn (gen_vsx_xvcvdpsxws (r2, operands[2]));
1339   rs6000_expand_extract_even (operands[0], r1, r2);
1340   DONE;
1341 })
1342
1343 (define_expand "vec_pack_ufix_trunc_v2df"
1344   [(match_operand:V4SI 0 "vint_operand")
1345    (match_operand:V2DF 1 "vfloat_operand")
1346    (match_operand:V2DF 2 "vfloat_operand")]
1347   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1348 {
1349   rtx r1 = gen_reg_rtx (V4SImode);
1350   rtx r2 = gen_reg_rtx (V4SImode);
1351
1352   emit_insn (gen_vsx_xvcvdpuxws (r1, operands[1]));
1353   emit_insn (gen_vsx_xvcvdpuxws (r2, operands[2]));
1354   rs6000_expand_extract_even (operands[0], r1, r2);
1355   DONE;
1356 })
1357
1358 ;; Convert single word types to double word
1359 (define_expand "vec_unpacks_hi_v4sf"
1360   [(match_operand:V2DF 0 "vfloat_operand")
1361    (match_operand:V4SF 1 "vfloat_operand")]
1362   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1363 {
1364   rtx reg = gen_reg_rtx (V4SFmode);
1365
1366   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1367   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1368   DONE;
1369 })
1370
1371 (define_expand "vec_unpacks_lo_v4sf"
1372   [(match_operand:V2DF 0 "vfloat_operand")
1373    (match_operand:V4SF 1 "vfloat_operand")]
1374   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1375 {
1376   rtx reg = gen_reg_rtx (V4SFmode);
1377
1378   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1379   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1380   DONE;
1381 })
1382
1383 (define_expand "vec_unpacks_float_hi_v4si"
1384   [(match_operand:V2DF 0 "vfloat_operand")
1385    (match_operand:V4SI 1 "vint_operand")]
1386   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1387 {
1388   rtx reg = gen_reg_rtx (V4SImode);
1389
1390   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1391   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1392   DONE;
1393 })
1394
1395 (define_expand "vec_unpacks_float_lo_v4si"
1396   [(match_operand:V2DF 0 "vfloat_operand")
1397    (match_operand:V4SI 1 "vint_operand")]
1398   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1399 {
1400   rtx reg = gen_reg_rtx (V4SImode);
1401
1402   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1403   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1404   DONE;
1405 })
1406
1407 (define_expand "vec_unpacku_float_hi_v4si"
1408   [(match_operand:V2DF 0 "vfloat_operand")
1409    (match_operand:V4SI 1 "vint_operand")]
1410   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1411 {
1412   rtx reg = gen_reg_rtx (V4SImode);
1413
1414   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1415   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1416   DONE;
1417 })
1418
1419 (define_expand "vec_unpacku_float_lo_v4si"
1420   [(match_operand:V2DF 0 "vfloat_operand")
1421    (match_operand:V4SI 1 "vint_operand")]
1422   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1423 {
1424   rtx reg = gen_reg_rtx (V4SImode);
1425
1426   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1427   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1428   DONE;
1429 })
1430
1431 \f
1432 ;; Align vector loads with a permute.
1433 (define_expand "vec_realign_load_<mode>"
1434   [(match_operand:VEC_K 0 "vlogical_operand")
1435    (match_operand:VEC_K 1 "vlogical_operand")
1436    (match_operand:VEC_K 2 "vlogical_operand")
1437    (match_operand:V16QI 3 "vlogical_operand")]
1438   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1439 {
1440   if (BYTES_BIG_ENDIAN)
1441     emit_insn (gen_altivec_vperm_<mode> (operands[0], operands[1],
1442                                          operands[2], operands[3]));
1443   else
1444     {
1445       /* We have changed lvsr to lvsl, so to complete the transformation
1446          of vperm for LE, we must swap the inputs.  */
1447       rtx unspec = gen_rtx_UNSPEC (<MODE>mode,
1448                                    gen_rtvec (3, operands[2],
1449                                               operands[1], operands[3]),
1450                                    UNSPEC_VPERM);
1451       emit_move_insn (operands[0], unspec);
1452     }
1453   DONE;
1454 })
1455
1456 ;; Under VSX, vectors of 4/8 byte alignments do not need to be aligned
1457 ;; since the load already handles it.
1458 (define_expand "movmisalign<mode>"
1459  [(set (match_operand:VEC_N 0 "nonimmediate_operand")
1460        (match_operand:VEC_N 1 "any_operand"))]
1461  "VECTOR_MEM_VSX_P (<MODE>mode) && TARGET_ALLOW_MOVMISALIGN"
1462 {
1463   rs6000_emit_move (operands[0], operands[1], <MODE>mode);
1464   DONE;
1465 })
1466
1467 ;; Vector shift right in bits. Currently supported ony for shift
1468 ;; amounts that can be expressed as byte shifts (divisible by 8).
1469 ;; General shift amounts can be supported using vsro + vsr. We're
1470 ;; not expecting to see these yet (the vectorizer currently
1471 ;; generates only shifts by a whole number of vector elements).
1472 ;; Note that the vec_shr operation is actually defined as
1473 ;; 'shift toward element 0' so is a shr for LE and shl for BE.
1474 (define_expand "vec_shr_<mode>"
1475   [(match_operand:VEC_L 0 "vlogical_operand")
1476    (match_operand:VEC_L 1 "vlogical_operand")
1477    (match_operand:QI 2 "reg_or_short_operand")]
1478   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1479 {
1480   rtx bitshift = operands[2];
1481   rtx shift;
1482   rtx insn;
1483   rtx zero_reg, op1, op2;
1484   HOST_WIDE_INT bitshift_val;
1485   HOST_WIDE_INT byteshift_val;
1486
1487   if (! CONSTANT_P (bitshift))
1488     FAIL;
1489   bitshift_val = INTVAL (bitshift);
1490   if (bitshift_val & 0x7)
1491     FAIL;
1492   byteshift_val = (bitshift_val >> 3);
1493   zero_reg = gen_reg_rtx (<MODE>mode);
1494   emit_move_insn (zero_reg, CONST0_RTX (<MODE>mode));
1495   if (!BYTES_BIG_ENDIAN)
1496     {
1497       /* Note, byteshift_val can be 0!  */
1498       byteshift_val = -byteshift_val & 15;
1499       op1 = zero_reg;
1500       op2 = operands[1];
1501     }
1502   else
1503     {
1504       op1 = operands[1];
1505       op2 = zero_reg;
1506     }
1507
1508   if (TARGET_VSX && (byteshift_val & 0x3) == 0)
1509     {
1510       shift = gen_rtx_CONST_INT (QImode, byteshift_val >> 2);
1511       insn = gen_vsx_xxsldwi_<mode> (operands[0], op1, op2, shift);
1512     }
1513   else
1514     {
1515       shift = gen_rtx_CONST_INT (QImode, byteshift_val);
1516       insn = gen_altivec_vsldoi_<mode> (operands[0], op1, op2, shift);
1517     }
1518
1519   emit_insn (insn);
1520   DONE;
1521 })
1522
1523 ;; Expanders for rotate each element in a vector
1524 (define_expand "vrotl<mode>3"
1525   [(set (match_operand:VEC_IC 0 "vint_operand")
1526         (rotate:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
1527                        (match_operand:VEC_IC 2 "vint_operand")))]
1528   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1529 {
1530   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2.  */
1531   if (<MODE>mode == V1TImode)
1532     {
1533       rtx tmp = gen_reg_rtx (V1TImode);
1534
1535       emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1536       emit_insn (gen_altivec_vrlq (operands[0], operands[1], tmp));
1537       DONE;
1538     }
1539  })
1540
1541 ;; Expanders for rotatert to make use of vrotl
1542 (define_expand "vrotr<mode>3"
1543   [(set (match_operand:VEC_I 0 "vint_operand")
1544         (rotatert:VEC_I (match_operand:VEC_I 1 "vint_operand")
1545                 (match_operand:VEC_I 2 "vint_operand")))]
1546   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1547 {
1548   rtx rot_count = gen_reg_rtx (<MODE>mode);
1549   emit_insn (gen_neg<mode>2 (rot_count, operands[2]));
1550   emit_insn (gen_vrotl<mode>3 (operands[0], operands[1], rot_count));
1551   DONE;
1552 })
1553
1554 ;; Expanders for arithmetic shift left on each vector element
1555 (define_expand "vashl<mode>3"
1556   [(set (match_operand:VEC_I 0 "vint_operand")
1557         (ashift:VEC_I (match_operand:VEC_I 1 "vint_operand")
1558                       (match_operand:VEC_I 2 "vint_operand")))]
1559   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1560   "")
1561
1562 ;; No immediate version of this 128-bit instruction
1563 (define_expand "vashl<mode>3"
1564   [(set (match_operand:VEC_TI 0 "vsx_register_operand" "=v")
1565         (ashift:VEC_TI (match_operand:VEC_TI 1 "vsx_register_operand")
1566                          (match_operand:VEC_TI 2 "vsx_register_operand")))]
1567   "TARGET_POWER10"
1568 {
1569   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2. */
1570   rtx tmp = gen_reg_rtx (<MODE>mode);
1571
1572   emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1573   emit_insn(gen_altivec_vslq_<mode> (operands[0], operands[1], tmp));
1574   DONE;
1575 })
1576
1577 ;; Expanders for logical shift right on each vector element
1578 (define_expand "vlshr<mode>3"
1579   [(set (match_operand:VEC_I 0 "vint_operand")
1580         (lshiftrt:VEC_I (match_operand:VEC_I 1 "vint_operand")
1581                         (match_operand:VEC_I 2 "vint_operand")))]
1582   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1583   "")
1584
1585 ;; No immediate version of this 128-bit instruction
1586 (define_expand "vlshr<mode>3"
1587   [(set (match_operand:VEC_TI 0 "vsx_register_operand" "=v")
1588         (lshiftrt:VEC_TI (match_operand:VEC_TI 1 "vsx_register_operand")
1589                            (match_operand:VEC_TI 2 "vsx_register_operand")))]
1590   "TARGET_POWER10"
1591 {
1592   /* Shift amount in needs to be put into bits[57:63] of 128-bit operand2. */
1593   rtx tmp = gen_reg_rtx (<MODE>mode);
1594
1595   emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1596   emit_insn(gen_altivec_vsrq_<mode> (operands[0], operands[1], tmp));
1597   DONE;
1598 })
1599
1600 ;; Expanders for arithmetic shift right on each vector element
1601 (define_expand "vashr<mode>3"
1602   [(set (match_operand:VEC_IC 0 "vint_operand")
1603         (ashiftrt:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
1604                          (match_operand:VEC_IC 2 "vint_operand")))]
1605   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1606 {
1607   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2.  */
1608   if (<MODE>mode == V1TImode)
1609     {
1610       rtx tmp = gen_reg_rtx (V1TImode);
1611
1612       emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1613       emit_insn (gen_altivec_vsraq (operands[0], operands[1], tmp));
1614       DONE;
1615     }
1616 })
1617
1618 \f
1619 ;; Vector reduction expanders for VSX
1620 ; The (VEC_reduc:...
1621 ;       (op1)
1622 ;       (unspec:... [(const_int 0)] UNSPEC_REDUC))
1623 ;
1624 ; is to allow us to use a code iterator, but not completely list all of the
1625 ; vector rotates, etc. to prevent canonicalization
1626
1627
1628 (define_expand "reduc_<VEC_reduc:VEC_reduc_name>_scal_<VEC_F:mode>"
1629   [(match_operand:<VEC_base> 0 "register_operand")
1630    (VEC_reduc:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
1631                     (unspec:VEC_F [(const_int 0)] UNSPEC_REDUC))]
1632   "VECTOR_UNIT_VSX_P (<VEC_F:MODE>mode)"
1633   {
1634     rtx vec = gen_reg_rtx (<VEC_F:MODE>mode);
1635     rtx elt = BYTES_BIG_ENDIAN
1636                 ? gen_int_mode (GET_MODE_NUNITS (<VEC_F:MODE>mode) - 1, QImode)
1637                 : const0_rtx;
1638     emit_insn (gen_vsx_reduc_<VEC_reduc:VEC_reduc_name>_<VEC_F:mode> (vec,
1639         operand1));
1640     emit_insn (gen_vsx_extract_<VEC_F:mode> (operand0, vec, elt));
1641     DONE;
1642   })