gcc/config/rs6000/vector.md

   1 ;; Expander definitions for vector support between altivec & vsx.  No
   2 ;; instructions are in this file, this file provides the generic vector
   3 ;; expander, and the actual vector instructions will be in altivec.md and
   4 ;; vsx.md
   5
   6 ;; Copyright (C) 2009-2020 Free Software Foundation, Inc.
   7 ;; Contributed by Michael Meissner <meissner@linux.vnet.ibm.com>
   8
   9 ;; This file is part of GCC.
  10
  11 ;; GCC is free software; you can redistribute it and/or modify it
  12 ;; under the terms of the GNU General Public License as published
  13 ;; by the Free Software Foundation; either version 3, or (at your
  14 ;; option) any later version.
  15
  16 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  17 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  18 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  19 ;; License for more details.
  20
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GCC; see the file COPYING3.  If not see
  23 ;; <http://www.gnu.org/licenses/>.
  24
  25
  26 ;; Vector int modes
  27 (define_mode_iterator VEC_I [V16QI V8HI V4SI V2DI])
  28
  29 ;; Vector int modes for parity
  30 (define_mode_iterator VEC_IP [V8HI
  31                               V4SI
  32                               V2DI
  33                               V1TI
  34                               TI])
  35
  36 ;; Vector float modes
  37 (define_mode_iterator VEC_F [V4SF V2DF])
  38
  39 ;; Vector arithmetic modes
  40 (define_mode_iterator VEC_A [V16QI V8HI V4SI V2DI V4SF V2DF])
  41
  42 ;; Vector modes that need alginment via permutes
  43 (define_mode_iterator VEC_K [V16QI V8HI V4SI V4SF])
  44
  45 ;; Vector logical modes
  46 (define_mode_iterator VEC_L [V16QI V8HI V4SI V2DI V4SF V2DF V1TI TI KF TF])
  47
  48 ;; Vector modes for moves.  Don't do TImode or TFmode here, since their
  49 ;; moves are handled elsewhere.
  50 (define_mode_iterator VEC_M [V16QI V8HI V4SI V2DI V4SF V2DF V1TI KF])
  51
  52 ;; Vector modes for types that don't need a realignment under VSX
  53 (define_mode_iterator VEC_N [V4SI V4SF V2DI V2DF V1TI KF TF])
  54
  55 ;; Vector comparison modes
  56 (define_mode_iterator VEC_C [V16QI V8HI V4SI V2DI V4SF V2DF])
  57
  58 ;; Vector init/extract modes
  59 (define_mode_iterator VEC_E [V16QI V8HI V4SI V2DI V4SF V2DF])
  60
  61 ;; Vector modes for 64-bit base types
  62 (define_mode_iterator VEC_64 [V2DI V2DF])
  63
  64 ;; Vector integer modes
  65 (define_mode_iterator VI [V4SI V8HI V16QI])
  66
  67 ;; Base type from vector mode
  68 (define_mode_attr VEC_base [(V16QI "QI")
  69                             (V8HI  "HI")
  70                             (V4SI  "SI")
  71                             (V2DI  "DI")
  72                             (V4SF  "SF")
  73                             (V2DF  "DF")
  74                             (V1TI  "TI")
  75                             (TI    "TI")])
  76
  77 ;; As above, but in lower case
  78 (define_mode_attr VEC_base_l [(V16QI "qi")
  79                               (V8HI  "hi")
  80                               (V4SI  "si")
  81                               (V2DI  "di")
  82                               (V4SF  "sf")
  83                               (V2DF  "df")
  84                               (V1TI  "ti")
  85                               (TI    "ti")])
  86
  87 ;; Same size integer type for floating point data
  88 (define_mode_attr VEC_int [(V4SF  "v4si")
  89                            (V2DF  "v2di")])
  90
  91 (define_mode_attr VEC_INT [(V4SF  "V4SI")
  92                            (V2DF  "V2DI")])
  93
  94 ;; constants for unspec
  95 (define_c_enum "unspec" [UNSPEC_PREDICATE
  96                          UNSPEC_REDUC
  97                          UNSPEC_NEZ_P])
  98
  99 ;; Vector reduction code iterators
 100 (define_code_iterator VEC_reduc [plus smin smax])
 101
 102 (define_code_attr VEC_reduc_name [(plus "plus")
 103                                   (smin "smin")
 104                                   (smax "smax")])
 105
 106 (define_code_attr VEC_reduc_rtx [(plus "add")
 107                                  (smin "smin")
 108                                  (smax "smax")])
 109
 110 ;; code iterators and attributes for vector FP comparison operators:
 111 (define_code_iterator
 112   vector_fp_comparison_simple [lt le ne ungt unge unlt unle])
 113 (define_code_iterator
 114   vector_fp_comparison_complex [ltgt uneq unordered ordered])
 115
 116 \f
 117 ;; Vector move instructions.  Little-endian VSX loads and stores require
 118 ;; special handling to circumvent "element endianness."
 119 (define_expand "mov<mode>"
 120   [(set (match_operand:VEC_M 0 "nonimmediate_operand")
 121         (match_operand:VEC_M 1 "any_operand"))]
 122   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 123 {
 124   if (can_create_pseudo_p ())
 125     {
 126       if (CONSTANT_P (operands[1]))
 127         {
 128           if (FLOAT128_VECTOR_P (<MODE>mode))
 129             {
 130               if (!easy_fp_constant (operands[1], <MODE>mode))
 131                 operands[1] = force_const_mem (<MODE>mode, operands[1]);
 132             }
 133           else if (!easy_vector_constant (operands[1], <MODE>mode))
 134             operands[1] = force_const_mem (<MODE>mode, operands[1]);
 135         }
 136
 137       if (!vlogical_operand (operands[0], <MODE>mode)
 138           && !vlogical_operand (operands[1], <MODE>mode))
 139         operands[1] = force_reg (<MODE>mode, operands[1]);
 140     }
 141   /* When generating load/store instructions to/from VSX registers on
 142      pre-power9 hardware in little endian mode, we need to emit register
 143      permute instructions to byte swap the contents, since the VSX load/store
 144      instructions do not include a byte swap as part of their operation.
 145      Altivec loads and stores have no such problem, so we skip them below.  */
 146   if (!BYTES_BIG_ENDIAN
 147       && VECTOR_MEM_VSX_P (<MODE>mode)
 148       && !TARGET_P9_VECTOR
 149       && !gpr_or_gpr_p (operands[0], operands[1])
 150       && ((memory_operand (operands[0], <MODE>mode)
 151            && !altivec_indexed_or_indirect_operand(operands[0], <MODE>mode))
 152           ^ (memory_operand (operands[1], <MODE>mode)
 153              && !altivec_indexed_or_indirect_operand(operands[1], <MODE>mode))))
 154     {
 155       rs6000_emit_le_vsx_move (operands[0], operands[1], <MODE>mode);
 156       DONE;
 157     }
 158 })
 159
 160 ;; Generic vector floating point load/store instructions.  These will match
 161 ;; insns defined in vsx.md or altivec.md depending on the switches.
 162 (define_expand "vector_load_<mode>"
 163   [(set (match_operand:VEC_M 0 "vfloat_operand")
 164         (match_operand:VEC_M 1 "memory_operand"))]
 165   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 166   "")
 167
 168 (define_expand "vector_store_<mode>"
 169   [(set (match_operand:VEC_M 0 "memory_operand")
 170         (match_operand:VEC_M 1 "vfloat_operand"))]
 171   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 172   "")
 173
 174 ;; Splits if a GPR register was chosen for the move
 175 (define_split
 176   [(set (match_operand:VEC_L 0 "nonimmediate_operand")
 177         (match_operand:VEC_L 1 "input_operand"))]
 178   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)
 179    && reload_completed
 180    && gpr_or_gpr_p (operands[0], operands[1])
 181    && !direct_move_p (operands[0], operands[1])
 182    && !quad_load_store_p (operands[0], operands[1])"
 183   [(pc)]
 184 {
 185   rs6000_split_multireg_move (operands[0], operands[1]);
 186   DONE;
 187 })
 188
 189 \f
 190 ;; Generic floating point vector arithmetic support
 191 (define_expand "add<mode>3"
 192   [(set (match_operand:VEC_F 0 "vfloat_operand")
 193         (plus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 194                     (match_operand:VEC_F 2 "vfloat_operand")))]
 195   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 196   "")
 197
 198 (define_expand "sub<mode>3"
 199   [(set (match_operand:VEC_F 0 "vfloat_operand")
 200         (minus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 201                      (match_operand:VEC_F 2 "vfloat_operand")))]
 202   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 203   "")
 204
 205 (define_expand "mul<mode>3"
 206   [(set (match_operand:VEC_F 0 "vfloat_operand")
 207         (mult:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 208                     (match_operand:VEC_F 2 "vfloat_operand")))]
 209   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 210 {
 211   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 212     {
 213       emit_insn (gen_altivec_mulv4sf3 (operands[0], operands[1], operands[2]));
 214       DONE;
 215     }
 216 })
 217
 218 (define_expand "div<mode>3"
 219   [(set (match_operand:VEC_F 0 "vfloat_operand")
 220         (div:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 221                    (match_operand:VEC_F 2 "vfloat_operand")))]
 222   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 223 {
 224   if (RS6000_RECIP_AUTO_RE_P (<MODE>mode)
 225       && can_create_pseudo_p () && flag_finite_math_only
 226       && !flag_trapping_math && flag_reciprocal_math)
 227     {
 228       rs6000_emit_swdiv (operands[0], operands[1], operands[2], true);
 229       DONE;
 230     }
 231 })
 232
 233 (define_expand "neg<mode>2"
 234   [(set (match_operand:VEC_F 0 "vfloat_operand")
 235         (neg:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 236   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 237 {
 238   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 239     {
 240       emit_insn (gen_altivec_negv4sf2 (operands[0], operands[1]));
 241       DONE;
 242     }
 243 })
 244
 245 (define_expand "abs<mode>2"
 246   [(set (match_operand:VEC_F 0 "vfloat_operand")
 247         (abs:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 248   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 249 {
 250   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 251     {
 252       emit_insn (gen_altivec_absv4sf2 (operands[0], operands[1]));
 253       DONE;
 254     }
 255 })
 256
 257 (define_expand "smin<mode>3"
 258   [(set (match_operand:VEC_F 0 "register_operand")
 259         (smin:VEC_F (match_operand:VEC_F 1 "register_operand")
 260                     (match_operand:VEC_F 2 "register_operand")))]
 261   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 262   "")
 263
 264 (define_expand "smax<mode>3"
 265   [(set (match_operand:VEC_F 0 "register_operand")
 266         (smax:VEC_F (match_operand:VEC_F 1 "register_operand")
 267                     (match_operand:VEC_F 2 "register_operand")))]
 268   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 269   "")
 270
 271
 272 (define_expand "sqrt<mode>2"
 273   [(set (match_operand:VEC_F 0 "vfloat_operand")
 274         (sqrt:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 275   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 276 {
 277   if (<MODE>mode == V4SFmode
 278       && !optimize_function_for_size_p (cfun)
 279       && flag_finite_math_only && !flag_trapping_math
 280       && flag_unsafe_math_optimizations)
 281     {
 282       rs6000_emit_swsqrt (operands[0], operands[1], 0);
 283       DONE;
 284     }
 285 })
 286
 287 (define_expand "rsqrte<mode>2"
 288   [(set (match_operand:VEC_F 0 "vfloat_operand")
 289         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 290                       UNSPEC_RSQRT))]
 291   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 292   "")
 293
 294 (define_expand "re<mode>2"
 295   [(set (match_operand:VEC_F 0 "vfloat_operand")
 296         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 297                       UNSPEC_FRES))]
 298   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 299   "")
 300
 301 (define_expand "ftrunc<mode>2"
 302   [(set (match_operand:VEC_F 0 "vfloat_operand")
 303         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 304   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 305   "")
 306
 307 (define_expand "vector_ceil<mode>2"
 308   [(set (match_operand:VEC_F 0 "vfloat_operand")
 309         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 310                       UNSPEC_FRIP))]
 311   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 312   "")
 313
 314 (define_expand "vector_floor<mode>2"
 315   [(set (match_operand:VEC_F 0 "vfloat_operand")
 316         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 317                       UNSPEC_FRIM))]
 318   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 319   "")
 320
 321 (define_expand "vector_btrunc<mode>2"
 322   [(set (match_operand:VEC_F 0 "vfloat_operand")
 323         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 324   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 325   "")
 326
 327 (define_expand "vector_copysign<mode>3"
 328   [(set (match_operand:VEC_F 0 "vfloat_operand")
 329         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")
 330                        (match_operand:VEC_F 2 "vfloat_operand")] UNSPEC_COPYSIGN))]
 331   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 332 {
 333   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 334     {
 335       emit_insn (gen_altivec_copysign_v4sf3 (operands[0], operands[1],
 336                                              operands[2]));
 337       DONE;
 338     }
 339 })
 340
 341 \f
 342 ;; Vector comparisons
 343 (define_expand "vcond<mode><mode>"
 344   [(set (match_operand:VEC_F 0 "vfloat_operand")
 345         (if_then_else:VEC_F
 346          (match_operator 3 "comparison_operator"
 347                          [(match_operand:VEC_F 4 "vfloat_operand")
 348                           (match_operand:VEC_F 5 "vfloat_operand")])
 349          (match_operand:VEC_F 1 "vfloat_operand")
 350          (match_operand:VEC_F 2 "vfloat_operand")))]
 351   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 352 {
 353   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 354                                     operands[3], operands[4], operands[5]))
 355     DONE;
 356   else
 357     FAIL;
 358 })
 359
 360 (define_expand "vcond<mode><mode>"
 361   [(set (match_operand:VEC_I 0 "vint_operand")
 362         (if_then_else:VEC_I
 363          (match_operator 3 "comparison_operator"
 364                          [(match_operand:VEC_I 4 "vint_operand")
 365                           (match_operand:VEC_I 5 "vint_operand")])
 366          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 367          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 368   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 369 {
 370   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 371                                     operands[3], operands[4], operands[5]))
 372     DONE;
 373   else
 374     FAIL;
 375 })
 376
 377 (define_expand "vcondv4sfv4si"
 378   [(set (match_operand:V4SF 0 "vfloat_operand")
 379         (if_then_else:V4SF
 380          (match_operator 3 "comparison_operator"
 381                          [(match_operand:V4SI 4 "vint_operand")
 382                           (match_operand:V4SI 5 "vint_operand")])
 383          (match_operand:V4SF 1 "vfloat_operand")
 384          (match_operand:V4SF 2 "vfloat_operand")))]
 385   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 386    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 387 {
 388   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 389                                     operands[3], operands[4], operands[5]))
 390     DONE;
 391   else
 392     FAIL;
 393 })
 394
 395 (define_expand "vcondv4siv4sf"
 396   [(set (match_operand:V4SI 0 "vint_operand")
 397         (if_then_else:V4SI
 398          (match_operator 3 "comparison_operator"
 399                          [(match_operand:V4SF 4 "vfloat_operand")
 400                           (match_operand:V4SF 5 "vfloat_operand")])
 401          (match_operand:V4SI 1 "vint_operand")
 402          (match_operand:V4SI 2 "vint_operand")))]
 403   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 404    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 405 {
 406   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 407                                     operands[3], operands[4], operands[5]))
 408     DONE;
 409   else
 410     FAIL;
 411 })
 412
 413 (define_expand "vcondv2dfv2di"
 414   [(set (match_operand:V2DF 0 "vfloat_operand")
 415         (if_then_else:V2DF
 416          (match_operator 3 "comparison_operator"
 417                          [(match_operand:V2DI 4 "vint_operand")
 418                           (match_operand:V2DI 5 "vint_operand")])
 419          (match_operand:V2DF 1 "vfloat_operand")
 420          (match_operand:V2DF 2 "vfloat_operand")))]
 421   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 422    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 423 {
 424   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 425                                     operands[3], operands[4], operands[5]))
 426     DONE;
 427   else
 428     FAIL;
 429 })
 430
 431 (define_expand "vcondv2div2df"
 432   [(set (match_operand:V2DI 0 "vint_operand")
 433         (if_then_else:V2DI
 434          (match_operator 3 "comparison_operator"
 435                          [(match_operand:V2DF 4 "vfloat_operand")
 436                           (match_operand:V2DF 5 "vfloat_operand")])
 437          (match_operand:V2DI 1 "vint_operand")
 438          (match_operand:V2DI 2 "vint_operand")))]
 439   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 440    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 441 {
 442   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 443                                     operands[3], operands[4], operands[5]))
 444     DONE;
 445   else
 446     FAIL;
 447 })
 448
 449 (define_expand "vcondu<mode><mode>"
 450   [(set (match_operand:VEC_I 0 "vint_operand")
 451         (if_then_else:VEC_I
 452          (match_operator 3 "comparison_operator"
 453                          [(match_operand:VEC_I 4 "vint_operand")
 454                           (match_operand:VEC_I 5 "vint_operand")])
 455          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 456          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 457   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 458 {
 459   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 460                                     operands[3], operands[4], operands[5]))
 461     DONE;
 462   else
 463     FAIL;
 464 })
 465
 466 (define_expand "vconduv4sfv4si"
 467   [(set (match_operand:V4SF 0 "vfloat_operand")
 468         (if_then_else:V4SF
 469          (match_operator 3 "comparison_operator"
 470                          [(match_operand:V4SI 4 "vint_operand")
 471                           (match_operand:V4SI 5 "vint_operand")])
 472          (match_operand:V4SF 1 "vfloat_operand")
 473          (match_operand:V4SF 2 "vfloat_operand")))]
 474   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 475    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 476 {
 477   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 478                                     operands[3], operands[4], operands[5]))
 479     DONE;
 480   else
 481     FAIL;
 482 })
 483
 484 (define_expand "vconduv2dfv2di"
 485   [(set (match_operand:V2DF 0 "vfloat_operand")
 486         (if_then_else:V2DF
 487          (match_operator 3 "comparison_operator"
 488                          [(match_operand:V2DI 4 "vint_operand")
 489                           (match_operand:V2DI 5 "vint_operand")])
 490          (match_operand:V2DF 1 "vfloat_operand")
 491          (match_operand:V2DF 2 "vfloat_operand")))]
 492   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 493    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 494 {
 495   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 496                                     operands[3], operands[4], operands[5]))
 497     DONE;
 498   else
 499     FAIL;
 500 })
 501
 502 ;; To support vector condition vectorization, define vcond_mask and vec_cmp.
 503
 504 ;; Same mode for condition true/false values and predicate operand.
 505 (define_expand "vcond_mask_<mode><mode>"
 506   [(match_operand:VEC_I 0 "vint_operand")
 507    (match_operand:VEC_I 1 "vint_operand")
 508    (match_operand:VEC_I 2 "vint_operand")
 509    (match_operand:VEC_I 3 "vint_operand")]
 510   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 511 {
 512   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 513                                   operands[3]));
 514   DONE;
 515 })
 516
 517 ;; Condition true/false values are float but predicate operand is of
 518 ;; type integer vector with same element size.
 519 (define_expand "vcond_mask_<mode><VEC_int>"
 520   [(match_operand:VEC_F 0 "vfloat_operand")
 521    (match_operand:VEC_F 1 "vfloat_operand")
 522    (match_operand:VEC_F 2 "vfloat_operand")
 523    (match_operand:<VEC_INT> 3 "vint_operand")]
 524   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 525 {
 526   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 527                                   gen_lowpart (<MODE>mode, operands[3])));
 528   DONE;
 529 })
 530
 531 ;; For signed integer vectors comparison.
 532 (define_expand "vec_cmp<mode><mode>"
 533   [(set (match_operand:VEC_I 0 "vint_operand")
 534         (match_operator 1 "signed_or_equality_comparison_operator"
 535           [(match_operand:VEC_I 2 "vint_operand")
 536            (match_operand:VEC_I 3 "vint_operand")]))]
 537   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 538 {
 539   enum rtx_code code = GET_CODE (operands[1]);
 540   rtx tmp = gen_reg_rtx (<MODE>mode);
 541   switch (code)
 542     {
 543     case NE:
 544       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 545       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 546       break;
 547     case EQ:
 548       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 549       break;
 550     case GE:
 551       emit_insn (gen_vector_nlt<mode> (operands[0],operands[2], operands[3],
 552                                        tmp));
 553       break;
 554     case GT:
 555       emit_insn (gen_vector_gt<mode> (operands[0], operands[2], operands[3]));
 556       break;
 557     case LE:
 558       emit_insn (gen_vector_ngt<mode> (operands[0], operands[2], operands[3],
 559                                        tmp));
 560       break;
 561     case LT:
 562       emit_insn (gen_vector_gt<mode> (operands[0], operands[3], operands[2]));
 563       break;
 564     default:
 565       gcc_unreachable ();
 566       break;
 567     }
 568   DONE;
 569 })
 570
 571 ;; For unsigned integer vectors comparison.
 572 (define_expand "vec_cmpu<mode><mode>"
 573   [(set (match_operand:VEC_I 0 "vint_operand")
 574         (match_operator 1 "unsigned_or_equality_comparison_operator"
 575           [(match_operand:VEC_I 2 "vint_operand")
 576            (match_operand:VEC_I 3 "vint_operand")]))]
 577   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 578 {
 579   enum rtx_code code = GET_CODE (operands[1]);
 580   rtx tmp = gen_reg_rtx (<MODE>mode);
 581   switch (code)
 582     {
 583     case NE:
 584       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 585       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 586       break;
 587     case EQ:
 588       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 589       break;
 590     case GEU:
 591       emit_insn (gen_vector_nltu<mode> (operands[0], operands[2], operands[3],
 592                                         tmp));
 593       break;
 594     case GTU:
 595       emit_insn (gen_vector_gtu<mode> (operands[0], operands[2], operands[3]));
 596       break;
 597     case LEU:
 598       emit_insn (gen_vector_ngtu<mode> (operands[0], operands[2], operands[3],
 599                                         tmp));
 600       break;
 601     case LTU:
 602       emit_insn (gen_vector_gtu<mode> (operands[0], operands[3], operands[2]));
 603       break;
 604     default:
 605       gcc_unreachable ();
 606       break;
 607     }
 608   DONE;
 609 })
 610
 611 ;; For float point vectors comparison.
 612 (define_expand "vec_cmp<mode><VEC_int>"
 613   [(set (match_operand:<VEC_INT> 0 "vint_operand")
 614          (match_operator 1 "comparison_operator"
 615             [(match_operand:VEC_F 2 "vfloat_operand")
 616             (match_operand:VEC_F 3 "vfloat_operand")]))]
 617   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 618 {
 619   enum rtx_code code = GET_CODE (operands[1]);
 620   rtx res = gen_reg_rtx (<MODE>mode);
 621   switch (code)
 622     {
 623     case NE:
 624       emit_insn (gen_vector_ne<mode> (res, operands[2], operands[3]));
 625       break;
 626     case EQ:
 627       emit_insn (gen_vector_eq<mode> (res, operands[2], operands[3]));
 628       break;
 629     case GE:
 630       emit_insn (gen_vector_ge<mode> (res, operands[2], operands[3]));
 631       break;
 632     case GT:
 633       emit_insn (gen_vector_gt<mode> (res, operands[2], operands[3]));
 634       break;
 635     case LE:
 636       emit_insn (gen_vector_le<mode> (res, operands[2], operands[3]));
 637       break;
 638     case LT:
 639       emit_insn (gen_vector_lt<mode> (res, operands[2], operands[3]));
 640       break;
 641     case LTGT:
 642       emit_insn (gen_vector_ltgt<mode> (res, operands[2], operands[3]));
 643       break;
 644     case UNORDERED:
 645       emit_insn (gen_vector_unordered<mode> (res, operands[2], operands[3]));
 646       break;
 647     case ORDERED:
 648       emit_insn (gen_vector_ordered<mode> (res, operands[2], operands[3]));
 649       break;
 650     case UNEQ:
 651       emit_insn (gen_vector_uneq<mode> (res, operands[2], operands[3]));
 652       break;
 653     case UNGE:
 654       emit_insn (gen_vector_unge<mode> (res, operands[2], operands[3]));
 655       break;
 656     case UNGT:
 657       emit_insn (gen_vector_ungt<mode> (res, operands[2], operands[3]));
 658       break;
 659     case UNLE:
 660       emit_insn (gen_vector_unle<mode> (res, operands[2], operands[3]));
 661       break;
 662     case UNLT:
 663       emit_insn (gen_vector_unlt<mode> (res, operands[2], operands[3]));
 664       break;
 665
 666     default:
 667       gcc_unreachable ();
 668     }
 669
 670   emit_insn (gen_move_insn (operands[0], gen_lowpart (<VEC_INT>mode, res)));
 671   DONE;
 672 })
 673
 674 (define_expand "vector_eq<mode>"
 675   [(set (match_operand:VEC_C 0 "vlogical_operand")
 676         (eq:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 677                   (match_operand:VEC_C 2 "vlogical_operand")))]
 678   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 679   "")
 680
 681 (define_expand "vector_gt<mode>"
 682   [(set (match_operand:VEC_C 0 "vlogical_operand")
 683         (gt:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 684                   (match_operand:VEC_C 2 "vlogical_operand")))]
 685   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 686   "")
 687
 688 ; >= for integer vectors: swap operands and apply not-greater-than
 689 (define_expand "vector_nlt<mode>"
 690   [(set (match_operand:VEC_I 3 "vlogical_operand")
 691         (gt:VEC_I (match_operand:VEC_I 2 "vlogical_operand")
 692                   (match_operand:VEC_I 1 "vlogical_operand")))
 693    (set (match_operand:VEC_I 0 "vlogical_operand")
 694         (not:VEC_I (match_dup 3)))]
 695   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 696 {
 697   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 698 })
 699
 700 (define_expand "vector_gtu<mode>"
 701   [(set (match_operand:VEC_I 0 "vint_operand")
 702         (gtu:VEC_I (match_operand:VEC_I 1 "vint_operand")
 703                    (match_operand:VEC_I 2 "vint_operand")))]
 704   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 705   "")
 706
 707 ; >= for integer vectors: swap operands and apply not-greater-than
 708 (define_expand "vector_nltu<mode>"
 709   [(set (match_operand:VEC_I 3 "vlogical_operand")
 710         (gtu:VEC_I (match_operand:VEC_I 2 "vlogical_operand")
 711                    (match_operand:VEC_I 1 "vlogical_operand")))
 712    (set (match_operand:VEC_I 0 "vlogical_operand")
 713         (not:VEC_I (match_dup 3)))]
 714   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 715 {
 716   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 717 })
 718
 719 (define_expand "vector_geu<mode>"
 720   [(set (match_operand:VEC_I 0 "vint_operand")
 721         (geu:VEC_I (match_operand:VEC_I 1 "vint_operand")
 722                    (match_operand:VEC_I 2 "vint_operand")))]
 723   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 724   "")
 725
 726 ; <= for integer vectors: apply not-greater-than
 727 (define_expand "vector_ngt<mode>"
 728   [(set (match_operand:VEC_I 3 "vlogical_operand")
 729         (gt:VEC_I (match_operand:VEC_I 1 "vlogical_operand")
 730                   (match_operand:VEC_I 2 "vlogical_operand")))
 731    (set (match_operand:VEC_I 0 "vlogical_operand")
 732         (not:VEC_I (match_dup 3)))]
 733   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 734 {
 735   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 736 })
 737
 738 (define_expand "vector_ngtu<mode>"
 739   [(set (match_operand:VEC_I 3 "vlogical_operand")
 740         (gtu:VEC_I (match_operand:VEC_I 1 "vlogical_operand")
 741                    (match_operand:VEC_I 2 "vlogical_operand")))
 742    (set (match_operand:VEC_I 0 "vlogical_operand")
 743         (not:VEC_I (match_dup 3)))]
 744   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 745 {
 746   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 747 })
 748
 749 ; There are 14 possible vector FP comparison operators, gt and eq of them have
 750 ; been expanded above, so just support 12 remaining operators here.
 751
 752 ; For ge:
 753 (define_expand "vector_ge<mode>"
 754   [(set (match_operand:VEC_F 0 "vlogical_operand")
 755         (ge:VEC_F (match_operand:VEC_F 1 "vlogical_operand")
 756                   (match_operand:VEC_F 2 "vlogical_operand")))]
 757   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 758   "")
 759
 760 ; For lt/le/ne/ungt/unge/unlt/unle:
 761 ; lt(a,b)   = gt(b,a)
 762 ; le(a,b)   = ge(b,a)
 763 ; unge(a,b) = ~lt(a,b)
 764 ; unle(a,b) = ~gt(a,b)
 765 ; ne(a,b)   = ~eq(a,b)
 766 ; ungt(a,b) = ~le(a,b)
 767 ; unlt(a,b) = ~ge(a,b)
 768 (define_insn_and_split "vector_<code><mode>"
 769   [(set (match_operand:VEC_F 0 "vfloat_operand")
 770         (vector_fp_comparison_simple:VEC_F
 771            (match_operand:VEC_F 1 "vfloat_operand")
 772            (match_operand:VEC_F 2 "vfloat_operand")))]
 773   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 774   "#"
 775   "&& can_create_pseudo_p ()"
 776   [(pc)]
 777 {
 778   enum rtx_code cond = <CODE>;
 779   bool need_invert = false;
 780
 781   if (cond == UNLE || cond == UNLT || cond == NE || cond == UNGE
 782       || cond == UNGT)
 783     {
 784       cond = reverse_condition_maybe_unordered (cond);
 785       need_invert = true;
 786     }
 787
 788   if (cond == LT || cond == LE)
 789     {
 790       cond = swap_condition (cond);
 791       std::swap (operands[1], operands[2]);
 792     }
 793
 794   gcc_assert (cond == EQ || cond == GE || cond == GT);
 795
 796   rtx comp = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 797
 798   if (need_invert)
 799     {
 800       rtx res = gen_reg_rtx (<MODE>mode);
 801       emit_insn (gen_rtx_SET (res, comp));
 802       emit_insn (gen_one_cmpl<mode>2 (operands[0], res));
 803     }
 804   else
 805     emit_insn (gen_rtx_SET (operands[0], comp));
 806
 807   DONE;
 808 })
 809
 810 ; For ltgt/uneq/ordered/unordered:
 811 ; ltgt: gt(a,b) | gt(b,a)
 812 ; uneq: ~(gt(a,b) | gt(b,a))
 813 ; ordered: ge(a,b) | ge(b,a)
 814 ; unordered: ~(ge(a,b) | ge(b,a))
 815 (define_insn_and_split "vector_<code><mode>"
 816   [(set (match_operand:VEC_F 0 "vfloat_operand")
 817         (vector_fp_comparison_complex:VEC_F
 818            (match_operand:VEC_F 1 "vfloat_operand")
 819            (match_operand:VEC_F 2 "vfloat_operand")))]
 820   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 821   "#"
 822   "&& can_create_pseudo_p ()"
 823   [(pc)]
 824 {
 825   enum rtx_code cond = <CODE>;
 826   bool need_invert = false;
 827
 828   if (cond == UNORDERED || cond == UNEQ)
 829     {
 830       cond = reverse_condition_maybe_unordered (cond);
 831       need_invert = true;
 832     }
 833
 834   if (cond == LTGT)
 835     cond = GT;
 836   else if (cond == ORDERED)
 837     cond = GE;
 838   else
 839     gcc_unreachable ();
 840
 841   rtx comp1 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 842   rtx res1 = gen_reg_rtx (<MODE>mode);
 843   emit_insn (gen_rtx_SET (res1, comp1));
 844   rtx comp2 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[2], operands[1]);
 845   rtx res2 = gen_reg_rtx (<MODE>mode);
 846   emit_insn (gen_rtx_SET (res2, comp2));
 847
 848   if (need_invert)
 849     {
 850       rtx not1 = gen_rtx_fmt_e (NOT, <MODE>mode, res1);
 851       rtx not2 = gen_rtx_fmt_e (NOT, <MODE>mode, res2);
 852       rtx comp3 = gen_rtx_fmt_ee (AND, <MODE>mode, not1, not2);
 853       emit_insn (gen_rtx_SET (operands[0], comp3));
 854     }
 855   else
 856     emit_insn (gen_ior<mode>3 (operands[0], res1, res2));
 857
 858   DONE;
 859 })
 860
 861 ;; Note the arguments for __builtin_altivec_vsel are op2, op1, mask
 862 ;; which is in the reverse order that we want
 863 (define_expand "vector_select_<mode>"
 864   [(set (match_operand:VEC_L 0 "vlogical_operand")
 865         (if_then_else:VEC_L
 866          (ne:CC (match_operand:VEC_L 3 "vlogical_operand")
 867                 (match_dup 4))
 868          (match_operand:VEC_L 2 "vlogical_operand")
 869          (match_operand:VEC_L 1 "vlogical_operand")))]
 870   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 871   "operands[4] = CONST0_RTX (<MODE>mode);")
 872
 873 (define_expand "vector_select_<mode>_uns"
 874   [(set (match_operand:VEC_L 0 "vlogical_operand")
 875         (if_then_else:VEC_L
 876          (ne:CCUNS (match_operand:VEC_L 3 "vlogical_operand")
 877                    (match_dup 4))
 878          (match_operand:VEC_L 2 "vlogical_operand")
 879          (match_operand:VEC_L 1 "vlogical_operand")))]
 880   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 881   "operands[4] = CONST0_RTX (<MODE>mode);")
 882
 883 ;; Expansions that compare vectors producing a vector result and a predicate,
 884 ;; setting CR6 to indicate a combined status
 885 (define_expand "vector_eq_<mode>_p"
 886   [(parallel
 887     [(set (reg:CC CR6_REGNO)
 888           (unspec:CC [(eq:CC (match_operand:VEC_A 1 "vlogical_operand")
 889                              (match_operand:VEC_A 2 "vlogical_operand"))]
 890                      UNSPEC_PREDICATE))
 891      (set (match_operand:VEC_A 0 "vlogical_operand")
 892           (eq:VEC_A (match_dup 1)
 893                     (match_dup 2)))])]
 894   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 895   "")
 896
 897 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 898 ;; implementation of the vec_all_ne built-in functions on Power9.
 899 (define_expand "vector_ne_<mode>_p"
 900   [(parallel
 901     [(set (reg:CC CR6_REGNO)
 902           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 903                              (match_operand:VI 2 "vlogical_operand"))]
 904            UNSPEC_PREDICATE))
 905      (set (match_dup 3)
 906           (ne:VI (match_dup 1)
 907                  (match_dup 2)))])
 908    (set (match_operand:SI 0 "register_operand" "=r")
 909         (lt:SI (reg:CC CR6_REGNO)
 910                (const_int 0)))]
 911   "TARGET_P9_VECTOR"
 912 {
 913   operands[3] = gen_reg_rtx (<MODE>mode);
 914 })
 915
 916 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 917 ;; implementation of the vec_any_eq built-in functions on Power9.
 918 (define_expand "vector_ae_<mode>_p"
 919   [(parallel
 920     [(set (reg:CC CR6_REGNO)
 921           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 922                              (match_operand:VI 2 "vlogical_operand"))]
 923            UNSPEC_PREDICATE))
 924      (set (match_dup 3)
 925           (ne:VI (match_dup 1)
 926                  (match_dup 2)))])
 927    (set (match_operand:SI 0 "register_operand" "=r")
 928         (lt:SI (reg:CC CR6_REGNO)
 929                (const_int 0)))
 930    (set (match_dup 0)
 931         (xor:SI (match_dup 0)
 932                 (const_int 1)))]
 933   "TARGET_P9_VECTOR"
 934 {
 935   operands[3] = gen_reg_rtx (<MODE>mode);
 936 })
 937
 938 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 939 ;; implementation of the vec_all_nez and vec_any_eqz built-in
 940 ;; functions on Power9.
 941 (define_expand "vector_nez_<mode>_p"
 942   [(parallel
 943     [(set (reg:CC CR6_REGNO)
 944           (unspec:CC [(unspec:VI
 945                        [(match_operand:VI 1 "vlogical_operand")
 946                         (match_operand:VI 2 "vlogical_operand")]
 947                        UNSPEC_NEZ_P)]
 948            UNSPEC_PREDICATE))
 949      (set (match_operand:VI 0 "vlogical_operand")
 950           (unspec:VI [(match_dup 1)
 951                       (match_dup 2)]
 952            UNSPEC_NEZ_P))])]
 953   "TARGET_P9_VECTOR"
 954   "")
 955
 956 ;; This expansion handles the V2DI mode in the implementation of the
 957 ;; vec_all_ne built-in function on Power9.
 958 ;;
 959 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
 960 ;; this expands into the same rtl that would be used for the Power8
 961 ;; architecture.
 962 (define_expand "vector_ne_v2di_p"
 963   [(parallel
 964     [(set (reg:CC CR6_REGNO)
 965           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
 966                              (match_operand:V2DI 2 "vlogical_operand"))]
 967                      UNSPEC_PREDICATE))
 968      (set (match_dup 3)
 969           (eq:V2DI (match_dup 1)
 970                    (match_dup 2)))])
 971    (set (match_operand:SI 0 "register_operand" "=r")
 972         (eq:SI (reg:CC CR6_REGNO)
 973                (const_int 0)))]
 974   "TARGET_P9_VECTOR"
 975 {
 976   operands[3] = gen_reg_rtx (V2DImode);
 977 })
 978
 979 ;; This expansion handles the V2DI mode in the implementation of the
 980 ;; vec_any_eq built-in function on Power9.
 981 ;;
 982 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
 983 ;; this expands into the same rtl that would be used for the Power8
 984 ;; architecture.
 985 (define_expand "vector_ae_v2di_p"
 986   [(parallel
 987     [(set (reg:CC CR6_REGNO)
 988           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
 989                              (match_operand:V2DI 2 "vlogical_operand"))]
 990                      UNSPEC_PREDICATE))
 991      (set (match_dup 3)
 992           (eq:V2DI (match_dup 1)
 993                    (match_dup 2)))])
 994    (set (match_operand:SI 0 "register_operand" "=r")
 995         (eq:SI (reg:CC CR6_REGNO)
 996                (const_int 0)))
 997    (set (match_dup 0)
 998         (xor:SI (match_dup 0)
 999                 (const_int 1)))]
1000   "TARGET_P9_VECTOR"
1001 {
1002   operands[3] = gen_reg_rtx (V2DImode);
1003 })
1004
1005 ;; This expansion handles the V4SF and V2DF modes in the Power9
1006 ;; implementation of the vec_all_ne built-in functions.  Note that the
1007 ;; expansions for this pattern with these modes makes no use of power9-
1008 ;; specific instructions since there are no new power9 instructions
1009 ;; for vector compare not equal with floating point arguments.
1010 (define_expand "vector_ne_<mode>_p"
1011   [(parallel
1012     [(set (reg:CC CR6_REGNO)
1013           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1014                              (match_operand:VEC_F 2 "vlogical_operand"))]
1015                      UNSPEC_PREDICATE))
1016      (set (match_dup 3)
1017           (eq:VEC_F (match_dup 1)
1018                     (match_dup 2)))])
1019    (set (match_operand:SI 0 "register_operand" "=r")
1020         (eq:SI (reg:CC CR6_REGNO)
1021                (const_int 0)))]
1022   "TARGET_P9_VECTOR"
1023 {
1024   operands[3] = gen_reg_rtx (<MODE>mode);
1025 })
1026
1027 ;; This expansion handles the V4SF and V2DF modes in the Power9
1028 ;; implementation of the vec_any_eq built-in functions.  Note that the
1029 ;; expansions for this pattern with these modes makes no use of power9-
1030 ;; specific instructions since there are no new power9 instructions
1031 ;; for vector compare not equal with floating point arguments.
1032 (define_expand "vector_ae_<mode>_p"
1033   [(parallel
1034     [(set (reg:CC CR6_REGNO)
1035           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1036                              (match_operand:VEC_F 2 "vlogical_operand"))]
1037                      UNSPEC_PREDICATE))
1038      (set (match_dup 3)
1039           (eq:VEC_F (match_dup 1)
1040                     (match_dup 2)))])
1041    (set (match_operand:SI 0 "register_operand" "=r")
1042         (eq:SI (reg:CC CR6_REGNO)
1043                (const_int 0)))
1044    (set (match_dup 0)
1045         (xor:SI (match_dup 0)
1046                 (const_int 1)))]
1047   "TARGET_P9_VECTOR"
1048 {
1049   operands[3] = gen_reg_rtx (<MODE>mode);
1050 })
1051
1052 (define_expand "vector_gt_<mode>_p"
1053   [(parallel
1054     [(set (reg:CC CR6_REGNO)
1055           (unspec:CC [(gt:CC (match_operand:VEC_A 1 "vlogical_operand")
1056                              (match_operand:VEC_A 2 "vlogical_operand"))]
1057                      UNSPEC_PREDICATE))
1058      (set (match_operand:VEC_A 0 "vlogical_operand")
1059           (gt:VEC_A (match_dup 1)
1060                     (match_dup 2)))])]
1061   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1062   "")
1063
1064 (define_expand "vector_ge_<mode>_p"
1065   [(parallel
1066     [(set (reg:CC CR6_REGNO)
1067           (unspec:CC [(ge:CC (match_operand:VEC_F 1 "vfloat_operand")
1068                              (match_operand:VEC_F 2 "vfloat_operand"))]
1069                      UNSPEC_PREDICATE))
1070      (set (match_operand:VEC_F 0 "vfloat_operand")
1071           (ge:VEC_F (match_dup 1)
1072                     (match_dup 2)))])]
1073   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1074   "")
1075
1076 (define_expand "vector_gtu_<mode>_p"
1077   [(parallel
1078     [(set (reg:CC CR6_REGNO)
1079           (unspec:CC [(gtu:CC (match_operand:VEC_I 1 "vint_operand")
1080                               (match_operand:VEC_I 2 "vint_operand"))]
1081                      UNSPEC_PREDICATE))
1082      (set (match_operand:VEC_I 0 "vlogical_operand")
1083           (gtu:VEC_I (match_dup 1)
1084                      (match_dup 2)))])]
1085   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1086   "")
1087
1088 ;; AltiVec/VSX predicates.
1089
1090 ;; This expansion is triggered during expansion of predicate built-in
1091 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1092 ;; altivec_expand_predicate_builtin() function when the value of the
1093 ;; integer constant first argument equals zero (aka __CR6_EQ in altivec.h).
1094 (define_expand "cr6_test_for_zero"
1095   [(set (match_operand:SI 0 "register_operand" "=r")
1096         (eq:SI (reg:CC CR6_REGNO)
1097                (const_int 0)))]
1098   "TARGET_ALTIVEC || TARGET_VSX"
1099   "")
1100
1101 ;; This expansion is triggered during expansion of predicate built-in
1102 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1103 ;; altivec_expand_predicate_builtin() function when the value of the
1104 ;; integer constant first argument equals one (aka __CR6_EQ_REV in altivec.h).
1105 (define_expand "cr6_test_for_zero_reverse"
1106   [(set (match_operand:SI 0 "register_operand" "=r")
1107         (eq:SI (reg:CC CR6_REGNO)
1108                (const_int 0)))
1109    (set (match_dup 0)
1110         (xor:SI (match_dup 0)
1111                 (const_int 1)))]
1112   "TARGET_ALTIVEC || TARGET_VSX"
1113   "")
1114
1115 ;; This expansion is triggered during expansion of predicate built-in
1116 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1117 ;; altivec_expand_predicate_builtin() function when the value of the
1118 ;; integer constant first argument equals two (aka __CR6_LT in altivec.h).
1119 (define_expand "cr6_test_for_lt"
1120   [(set (match_operand:SI 0 "register_operand" "=r")
1121         (lt:SI (reg:CC CR6_REGNO)
1122                (const_int 0)))]
1123   "TARGET_ALTIVEC || TARGET_VSX"
1124   "")
1125
1126 ;; This expansion is triggered during expansion of predicate built-in
1127 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1128 ;; altivec_expand_predicate_builtin() function when the value of the
1129 ;; integer constant first argument equals three
1130 ;; (aka __CR6_LT_REV in altivec.h).
1131 (define_expand "cr6_test_for_lt_reverse"
1132   [(set (match_operand:SI 0 "register_operand" "=r")
1133         (lt:SI (reg:CC CR6_REGNO)
1134                (const_int 0)))
1135    (set (match_dup 0)
1136         (xor:SI (match_dup 0)
1137                 (const_int 1)))]
1138   "TARGET_ALTIVEC || TARGET_VSX"
1139   "")
1140
1141 \f
1142 ;; Vector count leading zeros
1143 (define_expand "clz<mode>2"
1144   [(set (match_operand:VEC_I 0 "register_operand")
1145         (clz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1146   "TARGET_P8_VECTOR")
1147
1148 ;; Vector count trailing zeros
1149 (define_expand "ctz<mode>2"
1150   [(set (match_operand:VEC_I 0 "register_operand")
1151         (ctz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1152   "TARGET_P9_VECTOR")
1153
1154 ;; Vector population count
1155 (define_expand "popcount<mode>2"
1156   [(set (match_operand:VEC_I 0 "register_operand")
1157         (popcount:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1158   "TARGET_P8_VECTOR")
1159
1160 ;; Vector parity
1161 (define_expand "parity<mode>2"
1162   [(set (match_operand:VEC_IP 0 "register_operand")
1163         (parity:VEC_IP (match_operand:VEC_IP 1 "register_operand")))]
1164   "TARGET_P9_VECTOR")
1165
1166 \f
1167 ;; Same size conversions
1168 (define_expand "float<VEC_int><mode>2"
1169   [(set (match_operand:VEC_F 0 "vfloat_operand")
1170         (float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1171   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1172 {
1173   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1174     {
1175       emit_insn (gen_altivec_vcfsx (operands[0], operands[1], const0_rtx));
1176       DONE;
1177     }
1178 })
1179
1180 (define_expand "floatuns<VEC_int><mode>2"
1181   [(set (match_operand:VEC_F 0 "vfloat_operand")
1182         (unsigned_float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1183   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1184 {
1185   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1186     {
1187       emit_insn (gen_altivec_vcfux (operands[0], operands[1], const0_rtx));
1188       DONE;
1189     }
1190 })
1191
1192 (define_expand "fix_trunc<mode><VEC_int>2"
1193   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1194         (fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1195   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1196 {
1197   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1198     {
1199       emit_insn (gen_altivec_vctsxs (operands[0], operands[1], const0_rtx));
1200       DONE;
1201     }
1202 })
1203
1204 (define_expand "fixuns_trunc<mode><VEC_int>2"
1205   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1206         (unsigned_fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1207   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1208 {
1209   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1210     {
1211       emit_insn (gen_altivec_vctuxs (operands[0], operands[1], const0_rtx));
1212       DONE;
1213     }
1214 })
1215
1216 \f
1217 ;; Vector initialization, set, extract
1218 (define_expand "vec_init<mode><VEC_base_l>"
1219   [(match_operand:VEC_E 0 "vlogical_operand")
1220    (match_operand:VEC_E 1 "")]
1221   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1222 {
1223   rs6000_expand_vector_init (operands[0], operands[1]);
1224   DONE;
1225 })
1226
1227 (define_expand "vec_set<mode>"
1228   [(match_operand:VEC_E 0 "vlogical_operand")
1229    (match_operand:<VEC_base> 1 "register_operand")
1230    (match_operand 2 "const_int_operand")]
1231   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1232 {
1233   rs6000_expand_vector_set (operands[0], operands[1], INTVAL (operands[2]));
1234   DONE;
1235 })
1236
1237 (define_expand "vec_extract<mode><VEC_base_l>"
1238   [(match_operand:<VEC_base> 0 "register_operand")
1239    (match_operand:VEC_E 1 "vlogical_operand")
1240    (match_operand 2 "const_int_operand")]
1241   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1242 {
1243   rs6000_expand_vector_extract (operands[0], operands[1], operands[2]);
1244   DONE;
1245 })
1246 \f
1247 ;; Convert double word types to single word types
1248 (define_expand "vec_pack_trunc_v2df"
1249   [(match_operand:V4SF 0 "vfloat_operand")
1250    (match_operand:V2DF 1 "vfloat_operand")
1251    (match_operand:V2DF 2 "vfloat_operand")]
1252   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1253 {
1254   rtx r1 = gen_reg_rtx (V4SFmode);
1255   rtx r2 = gen_reg_rtx (V4SFmode);
1256
1257   emit_insn (gen_vsx_xvcvdpsp (r1, operands[1]));
1258   emit_insn (gen_vsx_xvcvdpsp (r2, operands[2]));
1259   rs6000_expand_extract_even (operands[0], r1, r2);
1260   DONE;
1261 })
1262
1263 (define_expand "vec_pack_sfix_trunc_v2df"
1264   [(match_operand:V4SI 0 "vint_operand")
1265    (match_operand:V2DF 1 "vfloat_operand")
1266    (match_operand:V2DF 2 "vfloat_operand")]
1267   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1268 {
1269   rtx r1 = gen_reg_rtx (V4SImode);
1270   rtx r2 = gen_reg_rtx (V4SImode);
1271
1272   emit_insn (gen_vsx_xvcvdpsxws (r1, operands[1]));
1273   emit_insn (gen_vsx_xvcvdpsxws (r2, operands[2]));
1274   rs6000_expand_extract_even (operands[0], r1, r2);
1275   DONE;
1276 })
1277
1278 (define_expand "vec_pack_ufix_trunc_v2df"
1279   [(match_operand:V4SI 0 "vint_operand")
1280    (match_operand:V2DF 1 "vfloat_operand")
1281    (match_operand:V2DF 2 "vfloat_operand")]
1282   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1283 {
1284   rtx r1 = gen_reg_rtx (V4SImode);
1285   rtx r2 = gen_reg_rtx (V4SImode);
1286
1287   emit_insn (gen_vsx_xvcvdpuxws (r1, operands[1]));
1288   emit_insn (gen_vsx_xvcvdpuxws (r2, operands[2]));
1289   rs6000_expand_extract_even (operands[0], r1, r2);
1290   DONE;
1291 })
1292
1293 ;; Convert single word types to double word
1294 (define_expand "vec_unpacks_hi_v4sf"
1295   [(match_operand:V2DF 0 "vfloat_operand")
1296    (match_operand:V4SF 1 "vfloat_operand")]
1297   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1298 {
1299   rtx reg = gen_reg_rtx (V4SFmode);
1300
1301   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1302   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1303   DONE;
1304 })
1305
1306 (define_expand "vec_unpacks_lo_v4sf"
1307   [(match_operand:V2DF 0 "vfloat_operand")
1308    (match_operand:V4SF 1 "vfloat_operand")]
1309   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1310 {
1311   rtx reg = gen_reg_rtx (V4SFmode);
1312
1313   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1314   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1315   DONE;
1316 })
1317
1318 (define_expand "vec_unpacks_float_hi_v4si"
1319   [(match_operand:V2DF 0 "vfloat_operand")
1320    (match_operand:V4SI 1 "vint_operand")]
1321   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1322 {
1323   rtx reg = gen_reg_rtx (V4SImode);
1324
1325   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1326   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1327   DONE;
1328 })
1329
1330 (define_expand "vec_unpacks_float_lo_v4si"
1331   [(match_operand:V2DF 0 "vfloat_operand")
1332    (match_operand:V4SI 1 "vint_operand")]
1333   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1334 {
1335   rtx reg = gen_reg_rtx (V4SImode);
1336
1337   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1338   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1339   DONE;
1340 })
1341
1342 (define_expand "vec_unpacku_float_hi_v4si"
1343   [(match_operand:V2DF 0 "vfloat_operand")
1344    (match_operand:V4SI 1 "vint_operand")]
1345   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1346 {
1347   rtx reg = gen_reg_rtx (V4SImode);
1348
1349   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1350   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1351   DONE;
1352 })
1353
1354 (define_expand "vec_unpacku_float_lo_v4si"
1355   [(match_operand:V2DF 0 "vfloat_operand")
1356    (match_operand:V4SI 1 "vint_operand")]
1357   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1358 {
1359   rtx reg = gen_reg_rtx (V4SImode);
1360
1361   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1362   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1363   DONE;
1364 })
1365
1366 \f
1367 ;; Align vector loads with a permute.
1368 (define_expand "vec_realign_load_<mode>"
1369   [(match_operand:VEC_K 0 "vlogical_operand")
1370    (match_operand:VEC_K 1 "vlogical_operand")
1371    (match_operand:VEC_K 2 "vlogical_operand")
1372    (match_operand:V16QI 3 "vlogical_operand")]
1373   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1374 {
1375   if (BYTES_BIG_ENDIAN)
1376     emit_insn (gen_altivec_vperm_<mode> (operands[0], operands[1],
1377                                          operands[2], operands[3]));
1378   else
1379     {
1380       /* We have changed lvsr to lvsl, so to complete the transformation
1381          of vperm for LE, we must swap the inputs.  */
1382       rtx unspec = gen_rtx_UNSPEC (<MODE>mode,
1383                                    gen_rtvec (3, operands[2],
1384                                               operands[1], operands[3]),
1385                                    UNSPEC_VPERM);
1386       emit_move_insn (operands[0], unspec);
1387     }
1388   DONE;
1389 })
1390
1391 ;; Under VSX, vectors of 4/8 byte alignments do not need to be aligned
1392 ;; since the load already handles it.
1393 (define_expand "movmisalign<mode>"
1394  [(set (match_operand:VEC_N 0 "nonimmediate_operand")
1395        (match_operand:VEC_N 1 "any_operand"))]
1396  "VECTOR_MEM_VSX_P (<MODE>mode) && TARGET_ALLOW_MOVMISALIGN"
1397  "")
1398
1399 ;; Vector shift right in bits. Currently supported ony for shift
1400 ;; amounts that can be expressed as byte shifts (divisible by 8).
1401 ;; General shift amounts can be supported using vsro + vsr. We're
1402 ;; not expecting to see these yet (the vectorizer currently
1403 ;; generates only shifts by a whole number of vector elements).
1404 ;; Note that the vec_shr operation is actually defined as
1405 ;; 'shift toward element 0' so is a shr for LE and shl for BE.
1406 (define_expand "vec_shr_<mode>"
1407   [(match_operand:VEC_L 0 "vlogical_operand")
1408    (match_operand:VEC_L 1 "vlogical_operand")
1409    (match_operand:QI 2 "reg_or_short_operand")]
1410   "TARGET_ALTIVEC"
1411 {
1412   rtx bitshift = operands[2];
1413   rtx shift;
1414   rtx insn;
1415   rtx zero_reg, op1, op2;
1416   HOST_WIDE_INT bitshift_val;
1417   HOST_WIDE_INT byteshift_val;
1418
1419   if (! CONSTANT_P (bitshift))
1420     FAIL;
1421   bitshift_val = INTVAL (bitshift);
1422   if (bitshift_val & 0x7)
1423     FAIL;
1424   byteshift_val = (bitshift_val >> 3);
1425   zero_reg = gen_reg_rtx (<MODE>mode);
1426   emit_move_insn (zero_reg, CONST0_RTX (<MODE>mode));
1427   if (!BYTES_BIG_ENDIAN)
1428     {
1429       byteshift_val = 16 - byteshift_val;
1430       op1 = zero_reg;
1431       op2 = operands[1];
1432     }
1433   else
1434     {
1435       op1 = operands[1];
1436       op2 = zero_reg;
1437     }
1438
1439   if (TARGET_VSX && (byteshift_val & 0x3) == 0)
1440     {
1441       shift = gen_rtx_CONST_INT (QImode, byteshift_val >> 2);
1442       insn = gen_vsx_xxsldwi_<mode> (operands[0], op1, op2, shift);
1443     }
1444   else
1445     {
1446       shift = gen_rtx_CONST_INT (QImode, byteshift_val);
1447       insn = gen_altivec_vsldoi_<mode> (operands[0], op1, op2, shift);
1448     }
1449
1450   emit_insn (insn);
1451   DONE;
1452 })
1453
1454 ;; Expanders for rotate each element in a vector
1455 (define_expand "vrotl<mode>3"
1456   [(set (match_operand:VEC_I 0 "vint_operand")
1457         (rotate:VEC_I (match_operand:VEC_I 1 "vint_operand")
1458                       (match_operand:VEC_I 2 "vint_operand")))]
1459   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1460   "")
1461
1462 ;; Expanders for rotatert to make use of vrotl
1463 (define_expand "vrotr<mode>3"
1464   [(set (match_operand:VEC_I 0 "vint_operand")
1465         (rotatert:VEC_I (match_operand:VEC_I 1 "vint_operand")
1466                 (match_operand:VEC_I 2 "vint_operand")))]
1467   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1468 {
1469   rtx rot_count = gen_reg_rtx (<MODE>mode);
1470   emit_insn (gen_neg<mode>2 (rot_count, operands[2]));
1471   emit_insn (gen_vrotl<mode>3 (operands[0], operands[1], rot_count));
1472   DONE;
1473 })
1474
1475 ;; Expanders for arithmetic shift left on each vector element
1476 (define_expand "vashl<mode>3"
1477   [(set (match_operand:VEC_I 0 "vint_operand")
1478         (ashift:VEC_I (match_operand:VEC_I 1 "vint_operand")
1479                       (match_operand:VEC_I 2 "vint_operand")))]
1480   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1481   "")
1482
1483 ;; Expanders for logical shift right on each vector element
1484 (define_expand "vlshr<mode>3"
1485   [(set (match_operand:VEC_I 0 "vint_operand")
1486         (lshiftrt:VEC_I (match_operand:VEC_I 1 "vint_operand")
1487                         (match_operand:VEC_I 2 "vint_operand")))]
1488   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1489   "")
1490
1491 ;; Expanders for arithmetic shift right on each vector element
1492 (define_expand "vashr<mode>3"
1493   [(set (match_operand:VEC_I 0 "vint_operand")
1494         (ashiftrt:VEC_I (match_operand:VEC_I 1 "vint_operand")
1495                         (match_operand:VEC_I 2 "vint_operand")))]
1496   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1497   "")
1498 \f
1499 ;; Vector reduction expanders for VSX
1500 ; The (VEC_reduc:...
1501 ;       (op1)
1502 ;       (unspec:... [(const_int 0)] UNSPEC_REDUC))
1503 ;
1504 ; is to allow us to use a code iterator, but not completely list all of the
1505 ; vector rotates, etc. to prevent canonicalization
1506
1507
1508 (define_expand "reduc_<VEC_reduc:VEC_reduc_name>_scal_<VEC_F:mode>"
1509   [(match_operand:<VEC_base> 0 "register_operand")
1510    (VEC_reduc:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
1511                     (unspec:VEC_F [(const_int 0)] UNSPEC_REDUC))]
1512   "VECTOR_UNIT_VSX_P (<VEC_F:MODE>mode)"
1513   {
1514     rtx vec = gen_reg_rtx (<VEC_F:MODE>mode);
1515     rtx elt = BYTES_BIG_ENDIAN
1516                 ? gen_int_mode (GET_MODE_NUNITS (<VEC_F:MODE>mode) - 1, QImode)
1517                 : const0_rtx;
1518     emit_insn (gen_vsx_reduc_<VEC_reduc:VEC_reduc_name>_<VEC_F:mode> (vec,
1519         operand1));
1520     emit_insn (gen_vsx_extract_<VEC_F:mode> (operand0, vec, elt));
1521     DONE;
1522   })