gcc/config/i386/mmx.md

   1 ;; GCC machine description for MMX and 3dNOW! instructions
   2 ;; Copyright (C) 2005-2023 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify
   7 ;; it under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful,
  12 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;; GNU General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 ;; The MMX and 3dNOW! patterns are in the same file because they use
  21 ;; the same register file, and 3dNOW! adds a number of extensions to
  22 ;; the base integer MMX isa.
  23
  24 ;; Note!  Except for the basic move instructions, *all* of these
  25 ;; patterns are outside the normal optabs namespace.  This is because
  26 ;; use of these registers requires the insertion of emms or femms
  27 ;; instructions to return to normal fpu mode.  The compiler doesn't
  28 ;; know how to do that itself, which means it's up to the user.  Which
  29 ;; means that we should never use any of these patterns except at the
  30 ;; direction of the user via a builtin.
  31
  32 (define_c_enum "unspec" [
  33   UNSPEC_MOVNTQ
  34   UNSPEC_PFRCP
  35   UNSPEC_PFRCPIT1
  36   UNSPEC_PFRCPIT2
  37   UNSPEC_PFRSQRT
  38   UNSPEC_PFRSQIT1
  39 ])
  40
  41 (define_c_enum "unspecv" [
  42   UNSPECV_EMMS
  43   UNSPECV_FEMMS
  44 ])
  45
  46 ;; 8 byte integral modes handled by MMX (and by extension, SSE)
  47 (define_mode_iterator MMXMODEI [V8QI V4HI V2SI])
  48 (define_mode_iterator MMXMODEI8 [V8QI V4HI V2SI (V1DI "TARGET_SSE2")])
  49
  50 ;; All 8-byte vector modes handled by MMX
  51 (define_mode_iterator MMXMODE [V8QI V4HI V2SI V1DI V2SF V4HF V4BF])
  52 (define_mode_iterator MMXMODE124 [V8QI V4HI V2SI V2SF])
  53
  54 ;; Mix-n-match
  55 (define_mode_iterator MMXMODE12 [V8QI V4HI])
  56 (define_mode_iterator MMXMODE14 [V8QI V2SI])
  57 (define_mode_iterator MMXMODE24 [V4HI V2SI])
  58 (define_mode_iterator MMXMODE248 [V4HI V2SI V1DI])
  59
  60 ;; All 4-byte integer/float16 vector modes
  61 (define_mode_iterator V_32 [V4QI V2HI V1SI V2HF V2BF])
  62
  63 ;; 4-byte integer vector modes
  64 (define_mode_iterator VI_32 [V4QI V2HI])
  65
  66 ;; 4-byte and 2-byte integer vector modes
  67 (define_mode_iterator VI_16_32 [V4QI V2QI V2HI])
  68
  69 ;; 4-byte and 2-byte QImode vector modes
  70 (define_mode_iterator VI1_16_32 [V4QI V2QI])
  71
  72 ;; All 2-byte, 4-byte and 8-byte vector modes with more than 1 element
  73 (define_mode_iterator V_16_32_64
  74    [V2QI V4QI V2HI V2HF
  75     (V8QI "TARGET_64BIT") (V4HI "TARGET_64BIT")
  76     (V4HF "TARGET_64BIT") (V4BF "TARGET_64BIT")
  77     (V2SI "TARGET_64BIT") (V2SF "TARGET_64BIT")])
  78
  79 ;; V2S* modes
  80 (define_mode_iterator V2FI [V2SF V2SI])
  81
  82 (define_mode_iterator V2FI_V4HF [V2SF V2SI V4HF])
  83 ;; Mapping from integer vector mode to mnemonic suffix
  84 (define_mode_attr mmxvecsize
  85   [(V8QI "b") (V4QI "b") (V2QI "b")
  86    (V4HI "w") (V2HI "w") (V2SI "d") (V1DI "q")])
  87
  88 ;; Mapping to same size integral mode.
  89 (define_mode_attr mmxinsnmode
  90   [(V8QI "DI") (V4QI "SI") (V2QI "HI")
  91    (V4HI "DI") (V2HI "SI")
  92    (V2SI "DI")
  93    (V4HF "DI") (V2HF "SI")
  94    (V4BF "DI") (V2BF "SI")
  95    (V2SF "DI")])
  96
  97 (define_mode_attr mmxdoublemode
  98   [(V8QI "V8HI") (V4HI "V4SI")])
  99
 100 ;; Mapping of vector float modes to an integer mode of the same size
 101 (define_mode_attr mmxintvecmode
 102   [(V2SF "V2SI") (V2SI "V2SI") (V4HI "V4HI") (V8QI "V8QI")])
 103
 104 (define_mode_attr mmxintvecmodelower
 105   [(V2SF "v2si") (V2SI "v2si") (V4HI "v4hi") (V8QI "v8qi")])
 106
 107 ;; Mapping of vector modes to a vector mode of double size
 108 (define_mode_attr mmxdoublevecmode
 109   [(V2SF "V4SF") (V2SI "V4SI") (V4HF "V8HF")])
 110
 111 ;; Mapping of vector modes back to the scalar modes
 112 (define_mode_attr mmxscalarmode
 113   [(V2SI "SI") (V2SF "SF")])
 114
 115 (define_mode_attr Yv_Yw
 116   [(V8QI "Yw") (V4HI "Yw") (V2SI "Yv") (V1DI "Yv") (V2SF "Yv")])
 117
 118 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 119 ;;
 120 ;; Move patterns
 121 ;;
 122 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 123
 124 ;; All of these patterns are enabled for MMX as well as 3dNOW.
 125 ;; This is essential for maintaining stable calling conventions.
 126
 127 (define_expand "mov<mode>"
 128   [(set (match_operand:MMXMODE 0 "nonimmediate_operand")
 129         (match_operand:MMXMODE 1 "nonimmediate_operand"))]
 130   "TARGET_MMX || TARGET_MMX_WITH_SSE"
 131 {
 132   ix86_expand_vector_move (<MODE>mode, operands);
 133   DONE;
 134 })
 135
 136 (define_insn "*mov<mode>_internal"
 137   [(set (match_operand:MMXMODE 0 "nonimmediate_operand"
 138     "=r ,o ,r,r ,m ,?!y,!y,?!y,m  ,r  ,?!y,v,v,v,m,r,v,!y,*x")
 139         (match_operand:MMXMODE 1 "nonimm_or_0_operand"
 140     "rCo,rC,C,rm,rC,C  ,!y,m  ,?!y,?!y,r  ,C,v,m,v,v,r,*x,!y"))]
 141   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
 142    && !(MEM_P (operands[0]) && MEM_P (operands[1]))
 143    && ix86_hardreg_mov_ok (operands[0], operands[1])"
 144 {
 145   switch (get_attr_type (insn))
 146     {
 147     case TYPE_MULTI:
 148       return "#";
 149
 150     case TYPE_IMOV:
 151       if (get_attr_mode (insn) == MODE_SI)
 152         return "mov{l}\t{%1, %k0|%k0, %1}";
 153       else
 154         return "mov{q}\t{%1, %0|%0, %1}";
 155
 156     case TYPE_MMX:
 157       return "pxor\t%0, %0";
 158
 159     case TYPE_MMXMOV:
 160       /* Handle broken assemblers that require movd instead of movq.  */
 161       if (!HAVE_AS_IX86_INTERUNIT_MOVQ
 162           && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
 163         return "movd\t{%1, %0|%0, %1}";
 164       return "movq\t{%1, %0|%0, %1}";
 165
 166     case TYPE_SSECVT:
 167       if (SSE_REG_P (operands[0]))
 168         return "movq2dq\t{%1, %0|%0, %1}";
 169       else
 170         return "movdq2q\t{%1, %0|%0, %1}";
 171
 172     case TYPE_SSELOG1:
 173       return standard_sse_constant_opcode (insn, operands);
 174
 175     case TYPE_SSEMOV:
 176       return ix86_output_ssemov (insn, operands);
 177
 178     default:
 179       gcc_unreachable ();
 180     }
 181 }
 182   [(set (attr "isa")
 183      (cond [(eq_attr "alternative" "0,1")
 184               (const_string "nox64")
 185             (eq_attr "alternative" "2,3,4,9,10")
 186               (const_string "x64")
 187             (eq_attr "alternative" "15,16")
 188               (const_string "x64_sse2")
 189             (eq_attr "alternative" "17,18")
 190               (const_string "sse2")
 191            ]
 192            (const_string "*")))
 193    (set (attr "type")
 194      (cond [(eq_attr "alternative" "0,1")
 195               (const_string "multi")
 196             (eq_attr "alternative" "2,3,4")
 197               (const_string "imov")
 198             (eq_attr "alternative" "5")
 199               (const_string "mmx")
 200             (eq_attr "alternative" "6,7,8,9,10")
 201               (const_string "mmxmov")
 202             (eq_attr "alternative" "11")
 203               (const_string "sselog1")
 204             (eq_attr "alternative" "17,18")
 205               (const_string "ssecvt")
 206            ]
 207            (const_string "ssemov")))
 208    (set (attr "prefix_rex")
 209      (if_then_else (eq_attr "alternative" "9,10,15,16")
 210        (const_string "1")
 211        (const_string "*")))
 212    (set (attr "prefix")
 213      (if_then_else (eq_attr "type" "sselog1,ssemov")
 214        (const_string "maybe_vex")
 215        (const_string "orig")))
 216    (set (attr "prefix_data16")
 217      (if_then_else
 218        (and (eq_attr "type" "ssemov") (eq_attr "mode" "DI"))
 219        (const_string "1")
 220        (const_string "*")))
 221    (set (attr "mode")
 222      (cond [(eq_attr "alternative" "2")
 223               (const_string "SI")
 224             (eq_attr "alternative" "11,12")
 225               (cond [(match_test "<MODE>mode == V2SFmode
 226                                   || <MODE>mode == V4HFmode
 227                                   || <MODE>mode == V4BFmode")
 228                        (const_string "V4SF")
 229                      (ior (not (match_test "TARGET_SSE2"))
 230                           (match_test "optimize_function_for_size_p (cfun)"))
 231                        (const_string "V4SF")
 232                     ]
 233                     (const_string "TI"))
 234
 235             (and (eq_attr "alternative" "13")
 236                  (ior (ior (and (match_test "<MODE>mode == V2SFmode")
 237                                 (not (match_test "TARGET_MMX_WITH_SSE")))
 238                            (not (match_test "TARGET_SSE2")))
 239                       (match_test "<MODE>mode == V4HFmode
 240                                   || <MODE>mode == V4BFmode")))
 241               (const_string "V2SF")
 242
 243             (and (eq_attr "alternative" "14")
 244                  (ior (ior (match_test "<MODE>mode == V2SFmode")
 245                            (not (match_test "TARGET_SSE2")))
 246                       (match_test "<MODE>mode == V4HFmode
 247                                   || <MODE>mode == V4BFmode")))
 248               (const_string "V2SF")
 249            ]
 250            (const_string "DI")))
 251    (set (attr "preferred_for_speed")
 252      (cond [(eq_attr "alternative" "9,15")
 253               (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
 254             (eq_attr "alternative" "10,16")
 255               (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC")
 256            ]
 257            (symbol_ref "true")))])
 258
 259 (define_split
 260   [(set (match_operand:MMXMODE 0 "nonimmediate_gr_operand")
 261         (match_operand:MMXMODE 1 "nonimmediate_gr_operand"))]
 262   "!TARGET_64BIT && reload_completed"
 263   [(const_int 0)]
 264   "ix86_split_long_move (operands); DONE;")
 265
 266 (define_split
 267   [(set (match_operand:MMXMODE 0 "nonimmediate_gr_operand")
 268         (match_operand:MMXMODE 1 "const0_operand"))]
 269   "!TARGET_64BIT && reload_completed"
 270   [(const_int 0)]
 271   "ix86_split_long_move (operands); DONE;")
 272
 273 (define_expand "movmisalign<mode>"
 274   [(set (match_operand:MMXMODE 0 "nonimmediate_operand")
 275         (match_operand:MMXMODE 1 "nonimmediate_operand"))]
 276   "TARGET_MMX || TARGET_MMX_WITH_SSE"
 277 {
 278   ix86_expand_vector_move (<MODE>mode, operands);
 279   DONE;
 280 })
 281
 282 (define_expand "mov<mode>"
 283   [(set (match_operand:V_32 0 "nonimmediate_operand")
 284         (match_operand:V_32 1 "nonimmediate_operand"))]
 285   ""
 286 {
 287   ix86_expand_vector_move (<MODE>mode, operands);
 288   DONE;
 289 })
 290
 291 (define_insn "*mov<mode>_internal"
 292   [(set (match_operand:V_32 0 "nonimmediate_operand"
 293     "=r ,m ,v,v,v,m,r,v")
 294         (match_operand:V_32 1 "general_operand"
 295     "rmC,rC,C,v,m,v,v,r"))]
 296   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
 297    && ix86_hardreg_mov_ok (operands[0], operands[1])"
 298 {
 299   switch (get_attr_type (insn))
 300     {
 301     case TYPE_IMOV:
 302       return "mov{l}\t{%1, %0|%0, %1}";
 303
 304     case TYPE_SSELOG1:
 305       return standard_sse_constant_opcode (insn, operands);
 306
 307     case TYPE_SSEMOV:
 308       return ix86_output_ssemov (insn, operands);
 309
 310     default:
 311       gcc_unreachable ();
 312     }
 313 }
 314   [(set (attr "isa")
 315      (cond [(eq_attr "alternative" "6,7")
 316               (const_string "sse2")
 317            ]
 318            (const_string "*")))
 319    (set (attr "type")
 320      (cond [(eq_attr "alternative" "2")
 321               (const_string "sselog1")
 322             (eq_attr "alternative" "3,4,5,6,7")
 323               (const_string "ssemov")
 324            ]
 325            (const_string "imov")))
 326    (set (attr "prefix")
 327      (if_then_else (eq_attr "type" "sselog1,ssemov")
 328        (const_string "maybe_vex")
 329        (const_string "orig")))
 330    (set (attr "prefix_data16")
 331      (if_then_else (and (eq_attr "type" "ssemov") (eq_attr "mode" "SI"))
 332        (const_string "1")
 333        (const_string "*")))
 334    (set (attr "mode")
 335      (cond [(eq_attr "alternative" "2,3")
 336               (cond [(match_test "<MODE>mode == V2HFmode
 337                                  || <MODE>mode == V2BFmode")
 338                        (const_string "V4SF")
 339                      (match_test "TARGET_AVX")
 340                        (const_string "TI")
 341                      (ior (not (match_test "TARGET_SSE2"))
 342                           (match_test "optimize_function_for_size_p (cfun)"))
 343                        (const_string "V4SF")
 344                     ]
 345                     (const_string "TI"))
 346
 347             (and (eq_attr "alternative" "4,5")
 348                  (ior (match_test "<MODE>mode == V2HFmode
 349                                  || <MODE>mode == V2BFmode")
 350                       (not (match_test "TARGET_SSE2"))))
 351               (const_string "SF")
 352            ]
 353            (const_string "SI")))
 354    (set (attr "preferred_for_speed")
 355      (cond [(eq_attr "alternative" "6")
 356               (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
 357             (eq_attr "alternative" "7")
 358               (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC")
 359            ]
 360            (symbol_ref "true")))])
 361
 362 ;; 16-bit, 32-bit and 64-bit constant vector stores.  After reload,
 363 ;; convert them to immediate integer stores.
 364 (define_insn_and_split "*mov<mode>_imm"
 365   [(set (match_operand:V_16_32_64 0 "memory_operand" "=m")
 366         (match_operand:V_16_32_64 1 "x86_64_const_vector_operand" "i"))]
 367   ""
 368   "#"
 369   "&& reload_completed"
 370   [(set (match_dup 0) (match_dup 1))]
 371 {
 372   HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (operands[1],
 373                                                             <MODE>mode);
 374   operands[1] = GEN_INT (val);
 375   operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
 376 })
 377
 378 ;; For TARGET_64BIT we always round up to 8 bytes.
 379 (define_insn "*push<mode>2_rex64"
 380   [(set (match_operand:V_32 0 "push_operand" "=X,X")
 381         (match_operand:V_32 1 "nonmemory_no_elim_operand" "rC,*v"))]
 382   "TARGET_64BIT"
 383   "@
 384    push{q}\t%q1
 385    #"
 386   [(set_attr "type" "push,multi")
 387    (set_attr "mode" "DI")])
 388
 389 (define_split
 390   [(set (match_operand:V_32 0 "push_operand")
 391         (match_operand:V_32 1 "sse_reg_operand"))]
 392   "TARGET_64BIT && TARGET_SSE && reload_completed"
 393   [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (match_dup 2)))
 394    (set (match_dup 0) (match_dup 1))]
 395 {
 396   operands[2] = GEN_INT (-PUSH_ROUNDING (GET_MODE_SIZE (<V_32:MODE>mode)));
 397   /* Preserve memory attributes. */
 398   operands[0] = replace_equiv_address (operands[0], stack_pointer_rtx);
 399 })
 400
 401 (define_expand "movmisalign<mode>"
 402   [(set (match_operand:V_32 0 "nonimmediate_operand")
 403         (match_operand:V_32 1 "nonimmediate_operand"))]
 404   ""
 405 {
 406   ix86_expand_vector_move (<MODE>mode, operands);
 407   DONE;
 408 })
 409
 410 (define_expand "movv2qi"
 411   [(set (match_operand:V2QI 0 "nonimmediate_operand")
 412         (match_operand:V2QI 1 "nonimmediate_operand"))]
 413   ""
 414 {
 415   ix86_expand_vector_move (V2QImode, operands);
 416   DONE;
 417 })
 418
 419 (define_insn "*movv2qi_internal"
 420   [(set (match_operand:V2QI 0 "nonimmediate_operand"
 421     "=r,r,r,m ,v,v,v,m,r,v")
 422         (match_operand:V2QI 1 "general_operand"
 423     "r ,C,m,rC,C,v,m,v,v,r"))]
 424   "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 425 {
 426   switch (get_attr_type (insn))
 427     {
 428     case TYPE_IMOV:
 429       if (get_attr_mode (insn) == MODE_SI)
 430         return "mov{l}\t{%k1, %k0|%k0, %k1}";
 431       else
 432         return "mov{w}\t{%1, %0|%0, %1}";
 433
 434     case TYPE_IMOVX:
 435       /* movzwl is faster than movw on p2 due to partial word stalls,
 436          though not as fast as an aligned movl.  */
 437       return "movz{wl|x}\t{%1, %k0|%k0, %1}";
 438
 439     case TYPE_SSELOG1:
 440       if (satisfies_constraint_C (operands[1]))
 441         return standard_sse_constant_opcode (insn, operands);
 442
 443       if (SSE_REG_P (operands[0]))
 444         return "%vpinsrw\t{$0, %1, %d0|%d0, %1, 0}";
 445       else
 446         return "%vpextrw\t{$0, %1, %0|%0, %1, 0}";
 447
 448     case TYPE_SSEMOV:
 449       return ix86_output_ssemov (insn, operands);
 450
 451     default:
 452       gcc_unreachable ();
 453     }
 454 }
 455   [(set (attr "isa")
 456         (cond [(eq_attr "alternative" "6,8,9")
 457                   (const_string "sse2")
 458                (eq_attr "alternative" "7")
 459                   (const_string "sse4")
 460                ]
 461                (const_string "*")))
 462    (set (attr "type")
 463      (cond [(eq_attr "alternative" "6,7")
 464               (if_then_else (match_test "TARGET_AVX512FP16")
 465                 (const_string "ssemov")
 466                 (const_string "sselog1"))
 467             (eq_attr "alternative" "4")
 468               (const_string "sselog1")
 469             (eq_attr "alternative" "5,8,9")
 470               (const_string "ssemov")
 471             (match_test "optimize_function_for_size_p (cfun)")
 472               (const_string "imov")
 473             (and (eq_attr "alternative" "0")
 474                  (ior (not (match_test "TARGET_PARTIAL_REG_STALL"))
 475                       (not (match_test "TARGET_HIMODE_MATH"))))
 476               (const_string "imov")
 477             (and (eq_attr "alternative" "1,2")
 478                  (match_operand:V2QI 1 "aligned_operand"))
 479               (const_string "imov")
 480             (and (match_test "TARGET_MOVX")
 481                  (eq_attr "alternative" "0,2"))
 482               (const_string "imovx")
 483            ]
 484            (const_string "imov")))
 485    (set (attr "prefix")
 486         (cond [(eq_attr "alternative" "4,5,6,7,8,9")
 487                  (const_string "maybe_evex")
 488               ]
 489               (const_string "orig")))
 490    (set (attr "mode")
 491      (cond [(eq_attr "alternative" "6,7")
 492               (if_then_else (match_test "TARGET_AVX512FP16")
 493                 (const_string "HI")
 494                 (const_string "TI"))
 495             (eq_attr "alternative" "8,9")
 496               (if_then_else (match_test "TARGET_AVX512FP16")
 497                 (const_string "HI")
 498                 (const_string "SI"))
 499             (eq_attr "alternative" "4")
 500               (cond [(match_test "TARGET_AVX")
 501                        (const_string "TI")
 502                      (ior (not (match_test "TARGET_SSE2"))
 503                           (match_test "optimize_function_for_size_p (cfun)"))
 504                        (const_string "V4SF")
 505                     ]
 506                     (const_string "TI"))
 507             (eq_attr "alternative" "5")
 508               (cond [(match_test "TARGET_AVX512FP16")
 509                        (const_string "HF")
 510                      (match_test "TARGET_AVX")
 511                        (const_string "TI")
 512                      (ior (not (match_test "TARGET_SSE2"))
 513                           (match_test "optimize_function_for_size_p (cfun)"))
 514                        (const_string "V4SF")
 515                     ]
 516                     (const_string "TI"))
 517             (eq_attr "type" "imovx")
 518               (const_string "SI")
 519             (and (eq_attr "alternative" "1,2")
 520                  (match_operand:V2QI 1 "aligned_operand"))
 521               (const_string "SI")
 522             (and (eq_attr "alternative" "0")
 523                  (ior (not (match_test "TARGET_PARTIAL_REG_STALL"))
 524                       (not (match_test "TARGET_HIMODE_MATH"))))
 525               (const_string "SI")
 526             ]
 527             (const_string "HI")))
 528    (set (attr "preferred_for_speed")
 529      (cond [(eq_attr "alternative" "8")
 530               (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
 531             (eq_attr "alternative" "9")
 532               (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC")
 533            ]
 534            (symbol_ref "true")))])
 535
 536 ;; We always round up to UNITS_PER_WORD bytes.
 537 (define_insn "*pushv2qi2"
 538   [(set (match_operand:V2QI 0 "push_operand" "=X,X")
 539         (match_operand:V2QI 1 "nonmemory_no_elim_operand" "rC,v"))]
 540   ""
 541   "* return TARGET_64BIT ? \"push{q}\t%q1\" : \"push{l}\t%k1\";
 542    #"
 543   [(set_attr "isa" "*,sse4")
 544    (set_attr "type" "push,multi")
 545    (set (attr "mode")
 546      (cond [(eq_attr "alternative" "0")
 547               (if_then_else (match_test "TARGET_64BIT")
 548                 (const_string "DI")
 549                 (const_string "SI"))
 550             (eq_attr "alternative" "1")
 551               (if_then_else (match_test "TARGET_AVX512FP16")
 552                 (const_string "HI")
 553                 (const_string "TI"))
 554            ]
 555            (const_string "HI")))])
 556
 557 (define_split
 558   [(set (match_operand:V2QI 0 "push_operand")
 559         (match_operand:V2QI 1 "sse_reg_operand"))]
 560   "TARGET_SSE4_1 && reload_completed"
 561   [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (match_dup 2)))
 562    (set (match_dup 0) (match_dup 1))]
 563 {
 564   operands[2] = GEN_INT (-PUSH_ROUNDING (GET_MODE_SIZE (V2QImode)));
 565   /* Preserve memory attributes. */
 566   operands[0] = replace_equiv_address (operands[0], stack_pointer_rtx);
 567 })
 568
 569 (define_expand "movmisalignv2qi"
 570   [(set (match_operand:V2QI 0 "nonimmediate_operand")
 571         (match_operand:V2QI 1 "nonimmediate_operand"))]
 572   ""
 573 {
 574   ix86_expand_vector_move (V2QImode, operands);
 575   DONE;
 576 })
 577
 578 (define_insn "sse_movntq"
 579   [(set (match_operand:DI 0 "memory_operand" "=m,m")
 580         (unspec:DI [(match_operand:DI 1 "register_operand" "y,r")]
 581                    UNSPEC_MOVNTQ))]
 582   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
 583    && (TARGET_SSE || TARGET_3DNOW_A)"
 584   "@
 585    movntq\t{%1, %0|%0, %1}
 586    movnti\t{%1, %0|%0, %1}"
 587   [(set_attr "isa" "*,x64")
 588    (set_attr "mmx_isa" "native,*")
 589    (set_attr "type" "mmxmov,ssemov")
 590    (set_attr "mode" "DI")])
 591
 592 (define_expand "movq_<mode>_to_sse"
 593   [(set (match_operand:<mmxdoublevecmode> 0 "register_operand")
 594         (vec_concat:<mmxdoublevecmode>
 595           (match_operand:V2FI_V4HF 1 "nonimmediate_operand")
 596           (match_dup 2)))]
 597   "TARGET_SSE2"
 598   "operands[2] = CONST0_RTX (<MODE>mode);")
 599
 600 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 601 ;;
 602 ;; Parallel single-precision floating point arithmetic
 603 ;;
 604 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 605
 606 (define_expand "<code>v2sf2"
 607   [(set (match_operand:V2SF 0 "register_operand")
 608         (absneg:V2SF
 609           (match_operand:V2SF 1 "register_operand")))]
 610   "TARGET_MMX_WITH_SSE"
 611   "ix86_expand_fp_absneg_operator (<CODE>, V2SFmode, operands); DONE;")
 612
 613 (define_insn_and_split "*mmx_<code>v2sf2"
 614   [(set (match_operand:V2SF 0 "register_operand" "=x,x,x")
 615         (absneg:V2SF
 616           (match_operand:V2SF 1 "register_operand" "0,x,x")))
 617    (use (match_operand:V2SF 2 "nonimmediate_operand" "x,0,x"))]
 618   "TARGET_MMX_WITH_SSE"
 619   "#"
 620   "&& reload_completed"
 621   [(set (match_dup 0)
 622         (<absneg_op>:V2SF (match_dup 1) (match_dup 2)))]
 623 {
 624   if (!TARGET_AVX && operands_match_p (operands[0], operands[2]))
 625     std::swap (operands[1], operands[2]);
 626 }
 627   [(set_attr "isa" "noavx,noavx,avx")])
 628
 629 (define_insn_and_split "*mmx_nabsv2sf2"
 630   [(set (match_operand:V2SF 0 "register_operand" "=x,x,x")
 631         (neg:V2SF
 632           (abs:V2SF
 633             (match_operand:V2SF 1 "register_operand" "0,x,x"))))
 634    (use (match_operand:V2SF 2 "nonimmediate_operand" "x,0,x"))]
 635   "TARGET_MMX_WITH_SSE"
 636   "#"
 637   "&& reload_completed"
 638   [(set (match_dup 0)
 639         (ior:V2SF (match_dup 1) (match_dup 2)))]
 640 {
 641   if (!TARGET_AVX && operands_match_p (operands[0], operands[2]))
 642     std::swap (operands[1], operands[2]);
 643 }
 644   [(set_attr "isa" "noavx,noavx,avx")])
 645
 646 (define_expand "<insn>v2sf3"
 647   [(set (match_operand:V2SF 0 "register_operand")
 648         (plusminusmult:V2SF
 649           (match_operand:V2SF 1 "nonimmediate_operand")
 650           (match_operand:V2SF 2 "nonimmediate_operand")))]
 651   "TARGET_MMX_WITH_SSE"
 652 {
 653   rtx op2 = gen_reg_rtx (V4SFmode);
 654   rtx op1 = gen_reg_rtx (V4SFmode);
 655   rtx op0 = gen_reg_rtx (V4SFmode);
 656
 657   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
 658   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 659
 660   emit_insn (gen_<insn>v4sf3 (op0, op1, op2));
 661
 662   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
 663   DONE;
 664 })
 665
 666 (define_expand "mmx_addv2sf3"
 667   [(set (match_operand:V2SF 0 "register_operand")
 668         (plus:V2SF
 669           (match_operand:V2SF 1 "nonimmediate_operand")
 670           (match_operand:V2SF 2 "nonimmediate_operand")))]
 671   "TARGET_3DNOW"
 672   "ix86_fixup_binary_operands_no_copy (PLUS, V2SFmode, operands);")
 673
 674 (define_insn "*mmx_addv2sf3"
 675   [(set (match_operand:V2SF 0 "register_operand" "=y")
 676         (plus:V2SF (match_operand:V2SF 1 "nonimmediate_operand" "%0")
 677                    (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
 678   "TARGET_3DNOW && ix86_binary_operator_ok (PLUS, V2SFmode, operands)"
 679   "pfadd\t{%2, %0|%0, %2}"
 680   [(set_attr "type" "mmxadd")
 681    (set_attr "prefix_extra" "1")
 682    (set_attr "mode" "V2SF")])
 683
 684 (define_expand "mmx_subv2sf3"
 685   [(set (match_operand:V2SF 0 "register_operand")
 686         (minus:V2SF (match_operand:V2SF 1 "register_operand")
 687                     (match_operand:V2SF 2 "nonimmediate_operand")))]
 688   "TARGET_3DNOW")
 689
 690 (define_expand "mmx_subrv2sf3"
 691   [(set (match_operand:V2SF 0 "register_operand")
 692         (minus:V2SF (match_operand:V2SF 2 "register_operand")
 693                     (match_operand:V2SF 1 "nonimmediate_operand")))]
 694   "TARGET_3DNOW")
 695
 696 (define_insn "*mmx_subv2sf3"
 697   [(set (match_operand:V2SF 0 "register_operand" "=y,y")
 698         (minus:V2SF (match_operand:V2SF 1 "nonimmediate_operand" "0,ym")
 699                     (match_operand:V2SF 2 "nonimmediate_operand" "ym,0")))]
 700   "TARGET_3DNOW && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 701   "@
 702    pfsub\t{%2, %0|%0, %2}
 703    pfsubr\t{%1, %0|%0, %1}"
 704   [(set_attr "type" "mmxadd")
 705    (set_attr "prefix_extra" "1")
 706    (set_attr "mode" "V2SF")])
 707
 708 (define_expand "mmx_mulv2sf3"
 709   [(set (match_operand:V2SF 0 "register_operand")
 710         (mult:V2SF (match_operand:V2SF 1 "nonimmediate_operand")
 711                    (match_operand:V2SF 2 "nonimmediate_operand")))]
 712   "TARGET_3DNOW"
 713   "ix86_fixup_binary_operands_no_copy (MULT, V2SFmode, operands);")
 714
 715 (define_insn "*mmx_mulv2sf3"
 716   [(set (match_operand:V2SF 0 "register_operand" "=y")
 717         (mult:V2SF (match_operand:V2SF 1 "nonimmediate_operand" "%0")
 718                    (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
 719   "TARGET_3DNOW && ix86_binary_operator_ok (MULT, V2SFmode, operands)"
 720   "pfmul\t{%2, %0|%0, %2}"
 721   [(set_attr "type" "mmxmul")
 722    (set_attr "prefix_extra" "1")
 723    (set_attr "mode" "V2SF")])
 724
 725 (define_expand "divv2sf3"
 726   [(set (match_operand:V2SF 0 "register_operand")
 727         (div:V2SF (match_operand:V2SF 1 "register_operand")
 728                   (match_operand:V2SF 2 "register_operand")))]
 729   "TARGET_MMX_WITH_SSE"
 730 {
 731   rtx op2 = gen_reg_rtx (V4SFmode);
 732   rtx op1 = gen_reg_rtx (V4SFmode);
 733   rtx op0 = gen_reg_rtx (V4SFmode);
 734
 735   rtx tmp = gen_rtx_VEC_CONCAT (V4SFmode, operands[2],
 736                                 force_reg (V2SFmode, CONST1_RTX (V2SFmode)));
 737   emit_insn (gen_rtx_SET (op2, tmp));
 738   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 739
 740   emit_insn (gen_divv4sf3 (op0, op1, op2));
 741
 742   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
 743   DONE;
 744 })
 745
 746 (define_expand "<code>v2sf3"
 747   [(set (match_operand:V2SF 0 "register_operand")
 748         (smaxmin:V2SF
 749           (match_operand:V2SF 1 "register_operand")
 750           (match_operand:V2SF 2 "register_operand")))]
 751   "TARGET_MMX_WITH_SSE"
 752 {
 753   rtx op2 = gen_reg_rtx (V4SFmode);
 754   rtx op1 = gen_reg_rtx (V4SFmode);
 755   rtx op0 = gen_reg_rtx (V4SFmode);
 756
 757   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
 758   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 759
 760   emit_insn (gen_<code>v4sf3 (op0, op1, op2));
 761
 762   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
 763   DONE;
 764 })
 765
 766 (define_expand "mmx_<code>v2sf3"
 767   [(set (match_operand:V2SF 0 "register_operand")
 768         (smaxmin:V2SF
 769           (match_operand:V2SF 1 "nonimmediate_operand")
 770           (match_operand:V2SF 2 "nonimmediate_operand")))]
 771   "TARGET_3DNOW"
 772 {
 773   if (!flag_finite_math_only || flag_signed_zeros)
 774     {
 775       operands[1] = force_reg (V2SFmode, operands[1]);
 776       emit_insn (gen_mmx_ieee_<maxmin_float>v2sf3
 777                  (operands[0], operands[1], operands[2]));
 778       DONE;
 779     }
 780   else
 781     ix86_fixup_binary_operands_no_copy (<CODE>, V2SFmode, operands);
 782 })
 783
 784 ;; These versions of the min/max patterns are intentionally ignorant of
 785 ;; their behavior wrt -0.0 and NaN (via the commutative operand mark).
 786 ;; Since both the tree-level MAX_EXPR and the rtl-level SMAX operator
 787 ;; are undefined in this condition, we're certain this is correct.
 788
 789 (define_insn "*mmx_<code>v2sf3"
 790   [(set (match_operand:V2SF 0 "register_operand" "=y")
 791         (smaxmin:V2SF
 792           (match_operand:V2SF 1 "nonimmediate_operand" "%0")
 793           (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
 794   "TARGET_3DNOW && ix86_binary_operator_ok (<CODE>, V2SFmode, operands)"
 795   "pf<maxmin_float>\t{%2, %0|%0, %2}"
 796   [(set_attr "type" "mmxadd")
 797    (set_attr "prefix_extra" "1")
 798    (set_attr "mode" "V2SF")])
 799
 800 ;; These versions of the min/max patterns implement exactly the operations
 801 ;;   min = (op1 < op2 ? op1 : op2)
 802 ;;   max = (!(op1 < op2) ? op1 : op2)
 803 ;; Their operands are not commutative, and thus they may be used in the
 804 ;; presence of -0.0 and NaN.
 805
 806 (define_insn "mmx_ieee_<ieee_maxmin>v2sf3"
 807   [(set (match_operand:V2SF 0 "register_operand" "=y")
 808         (unspec:V2SF
 809           [(match_operand:V2SF 1 "register_operand" "0")
 810            (match_operand:V2SF 2 "nonimmediate_operand" "ym")]
 811           IEEE_MAXMIN))]
 812   "TARGET_3DNOW"
 813   "pf<ieee_maxmin>\t{%2, %0|%0, %2}"
 814   [(set_attr "type" "mmxadd")
 815    (set_attr "prefix_extra" "1")
 816    (set_attr "mode" "V2SF")])
 817
 818 (define_insn "mmx_rcpv2sf2"
 819   [(set (match_operand:V2SF 0 "register_operand" "=y")
 820         (unspec:V2SF [(match_operand:V2SF 1 "nonimmediate_operand" "ym")]
 821                      UNSPEC_PFRCP))]
 822   "TARGET_3DNOW"
 823   "pfrcp\t{%1, %0|%0, %1}"
 824   [(set_attr "type" "mmx")
 825    (set_attr "prefix_extra" "1")
 826    (set_attr "mode" "V2SF")])
 827
 828 (define_insn "mmx_rcpit1v2sf3"
 829   [(set (match_operand:V2SF 0 "register_operand" "=y")
 830         (unspec:V2SF [(match_operand:V2SF 1 "register_operand" "0")
 831                       (match_operand:V2SF 2 "nonimmediate_operand" "ym")]
 832                      UNSPEC_PFRCPIT1))]
 833   "TARGET_3DNOW"
 834   "pfrcpit1\t{%2, %0|%0, %2}"
 835   [(set_attr "type" "mmx")
 836    (set_attr "prefix_extra" "1")
 837    (set_attr "mode" "V2SF")])
 838
 839 (define_insn "mmx_rcpit2v2sf3"
 840   [(set (match_operand:V2SF 0 "register_operand" "=y")
 841         (unspec:V2SF [(match_operand:V2SF 1 "register_operand" "0")
 842                       (match_operand:V2SF 2 "nonimmediate_operand" "ym")]
 843                      UNSPEC_PFRCPIT2))]
 844   "TARGET_3DNOW"
 845   "pfrcpit2\t{%2, %0|%0, %2}"
 846   [(set_attr "type" "mmx")
 847    (set_attr "prefix_extra" "1")
 848    (set_attr "mode" "V2SF")])
 849
 850 (define_expand "sqrtv2sf2"
 851   [(set (match_operand:V2SF 0 "register_operand")
 852         (sqrt:V2SF (match_operand:V2SF 1 "nonimmediate_operand")))]
 853   "TARGET_MMX_WITH_SSE"
 854 {
 855   rtx op1 = gen_reg_rtx (V4SFmode);
 856   rtx op0 = gen_reg_rtx (V4SFmode);
 857
 858   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 859
 860   emit_insn (gen_sqrtv4sf2 (op0, op1));
 861
 862   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
 863   DONE;
 864 })
 865
 866 (define_insn "mmx_rsqrtv2sf2"
 867   [(set (match_operand:V2SF 0 "register_operand" "=y")
 868         (unspec:V2SF [(match_operand:V2SF 1 "nonimmediate_operand" "ym")]
 869                      UNSPEC_PFRSQRT))]
 870   "TARGET_3DNOW"
 871   "pfrsqrt\t{%1, %0|%0, %1}"
 872   [(set_attr "type" "mmx")
 873    (set_attr "prefix_extra" "1")
 874    (set_attr "mode" "V2SF")])
 875
 876 (define_insn "mmx_rsqit1v2sf3"
 877   [(set (match_operand:V2SF 0 "register_operand" "=y")
 878         (unspec:V2SF [(match_operand:V2SF 1 "register_operand" "0")
 879                       (match_operand:V2SF 2 "nonimmediate_operand" "ym")]
 880                      UNSPEC_PFRSQIT1))]
 881   "TARGET_3DNOW"
 882   "pfrsqit1\t{%2, %0|%0, %2}"
 883   [(set_attr "type" "mmx")
 884    (set_attr "prefix_extra" "1")
 885    (set_attr "mode" "V2SF")])
 886
 887 (define_expand "mmx_haddv2sf3"
 888   [(set (match_operand:V2SF 0 "register_operand")
 889         (vec_concat:V2SF
 890           (plus:SF
 891             (vec_select:SF
 892               (match_operand:V2SF 1 "register_operand")
 893               (parallel [(const_int 0)]))
 894             (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
 895           (plus:SF
 896             (vec_select:SF
 897               (match_operand:V2SF 2 "nonimmediate_operand")
 898               (parallel [(const_int 0)]))
 899             (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))))]
 900   "TARGET_3DNOW")
 901
 902 (define_insn "*mmx_haddv2sf3"
 903   [(set (match_operand:V2SF 0 "register_operand" "=y")
 904         (vec_concat:V2SF
 905           (plus:SF
 906             (vec_select:SF
 907               (match_operand:V2SF 1 "register_operand" "0")
 908               (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))
 909             (vec_select:SF (match_dup 1)
 910             (parallel [(match_operand:SI 4 "const_0_to_1_operand")])))
 911           (plus:SF
 912             (vec_select:SF
 913               (match_operand:V2SF 2 "nonimmediate_operand" "ym")
 914               (parallel [(match_operand:SI 5 "const_0_to_1_operand")]))
 915             (vec_select:SF (match_dup 2)
 916             (parallel [(match_operand:SI 6 "const_0_to_1_operand")])))))]
 917   "TARGET_3DNOW
 918    && INTVAL (operands[3]) != INTVAL (operands[4])
 919    && INTVAL (operands[5]) != INTVAL (operands[6])"
 920   "pfacc\t{%2, %0|%0, %2}"
 921   [(set_attr "type" "mmxadd")
 922    (set_attr "prefix_extra" "1")
 923    (set_attr "mode" "V2SF")])
 924
 925 (define_insn_and_split "*mmx_haddv2sf3_low"
 926   [(set (match_operand:SF 0 "register_operand")
 927         (plus:SF
 928           (vec_select:SF
 929             (match_operand:V2SF 1 "nonimmediate_operand")
 930             (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))
 931           (vec_select:SF
 932             (match_dup 1)
 933             (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))))]
 934   "TARGET_SSE3 && TARGET_MMX_WITH_SSE
 935    && INTVAL (operands[2]) != INTVAL (operands[3])
 936    && ix86_pre_reload_split ()"
 937   "#"
 938   "&& 1"
 939   [(const_int 0)]
 940 {
 941   rtx op1 = gen_reg_rtx (V4SFmode);
 942   rtx op0 = gen_reg_rtx (V4SFmode);
 943
 944   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 945
 946   emit_insn (gen_sse3_haddv4sf3 (op0, op1, op1));
 947
 948   emit_move_insn (operands[0], lowpart_subreg (SFmode, op0, V4SFmode));
 949   DONE;
 950 })
 951
 952 (define_insn "mmx_hsubv2sf3"
 953   [(set (match_operand:V2SF 0 "register_operand" "=y")
 954         (vec_concat:V2SF
 955           (minus:SF
 956             (vec_select:SF
 957               (match_operand:V2SF 1 "register_operand" "0")
 958               (parallel [(const_int  0)]))
 959             (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
 960           (minus:SF
 961             (vec_select:SF
 962               (match_operand:V2SF 2 "nonimmediate_operand" "ym")
 963               (parallel [(const_int  0)]))
 964             (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))))]
 965   "TARGET_3DNOW_A"
 966   "pfnacc\t{%2, %0|%0, %2}"
 967   [(set_attr "type" "mmxadd")
 968    (set_attr "prefix_extra" "1")
 969    (set_attr "mode" "V2SF")])
 970
 971 (define_insn_and_split "*mmx_hsubv2sf3_low"
 972   [(set (match_operand:SF 0 "register_operand")
 973         (minus:SF
 974           (vec_select:SF
 975             (match_operand:V2SF 1 "register_operand")
 976             (parallel [(const_int 0)]))
 977           (vec_select:SF
 978             (match_dup 1)
 979             (parallel [(const_int 1)]))))]
 980   "TARGET_SSE3 && TARGET_MMX_WITH_SSE
 981    && ix86_pre_reload_split ()"
 982   "#"
 983   "&& 1"
 984   [(const_int 0)]
 985 {
 986   rtx op1 = gen_reg_rtx (V4SFmode);
 987   rtx op0 = gen_reg_rtx (V4SFmode);
 988
 989   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
 990
 991   emit_insn (gen_sse3_hsubv4sf3 (op0, op1, op1));
 992
 993   emit_move_insn (operands[0], lowpart_subreg (SFmode, op0, V4SFmode));
 994   DONE;
 995 })
 996
 997 (define_expand "mmx_haddsubv2sf3"
 998   [(set (match_operand:V2SF 0 "register_operand")
 999         (vec_concat:V2SF
1000           (minus:SF
1001             (vec_select:SF
1002               (match_operand:V2SF 1 "register_operand")
1003               (parallel [(const_int 0)]))
1004             (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
1005           (plus:SF
1006             (vec_select:SF
1007               (match_operand:V2SF 2 "nonimmediate_operand")
1008               (parallel [(const_int 0)]))
1009             (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))))]
1010   "TARGET_3DNOW_A")
1011
1012 (define_insn "*mmx_haddsubv2sf3"
1013   [(set (match_operand:V2SF 0 "register_operand" "=y")
1014         (vec_concat:V2SF
1015           (minus:SF
1016             (vec_select:SF
1017               (match_operand:V2SF 1 "register_operand" "0")
1018               (parallel [(const_int  0)]))
1019             (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
1020           (plus:SF
1021             (vec_select:SF
1022               (match_operand:V2SF 2 "nonimmediate_operand" "ym")
1023               (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))
1024             (vec_select:SF
1025               (match_dup 2)
1026               (parallel [(match_operand:SI 4 "const_0_to_1_operand")])))))]
1027   "TARGET_3DNOW_A
1028    && INTVAL (operands[3]) != INTVAL (operands[4])"
1029   "pfpnacc\t{%2, %0|%0, %2}"
1030   [(set_attr "type" "mmxadd")
1031    (set_attr "prefix_extra" "1")
1032    (set_attr "mode" "V2SF")])
1033
1034 (define_expand "vec_addsubv2sf3"
1035   [(set (match_operand:V2SF 0 "register_operand")
1036         (vec_merge:V2SF
1037           (minus:V2SF
1038             (match_operand:V2SF 1 "nonimmediate_operand")
1039             (match_operand:V2SF 2 "nonimmediate_operand"))
1040           (plus:V2SF (match_dup 1) (match_dup 2))
1041           (const_int 1)))]
1042   "TARGET_SSE3 && TARGET_MMX_WITH_SSE"
1043 {
1044   rtx op2 = gen_reg_rtx (V4SFmode);
1045   rtx op1 = gen_reg_rtx (V4SFmode);
1046   rtx op0 = gen_reg_rtx (V4SFmode);
1047
1048   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
1049   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1050
1051   emit_insn (gen_vec_addsubv4sf3 (op0, op1, op2));
1052
1053   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1054   DONE;
1055 })
1056
1057 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1058 ;;
1059 ;; Parallel single-precision floating point comparisons
1060 ;;
1061 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1062
1063 (define_expand "mmx_eqv2sf3"
1064   [(set (match_operand:V2SI 0 "register_operand")
1065         (eq:V2SI (match_operand:V2SF 1 "nonimmediate_operand")
1066                  (match_operand:V2SF 2 "nonimmediate_operand")))]
1067   "TARGET_3DNOW"
1068   "ix86_fixup_binary_operands_no_copy (EQ, V2SFmode, operands);")
1069
1070 (define_insn "*mmx_eqv2sf3"
1071   [(set (match_operand:V2SI 0 "register_operand" "=y")
1072         (eq:V2SI (match_operand:V2SF 1 "nonimmediate_operand" "%0")
1073                  (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
1074   "TARGET_3DNOW && ix86_binary_operator_ok (EQ, V2SFmode, operands)"
1075   "pfcmpeq\t{%2, %0|%0, %2}"
1076   [(set_attr "type" "mmxcmp")
1077    (set_attr "prefix_extra" "1")
1078    (set_attr "mode" "V2SF")])
1079
1080 (define_insn "mmx_gtv2sf3"
1081   [(set (match_operand:V2SI 0 "register_operand" "=y")
1082         (gt:V2SI (match_operand:V2SF 1 "register_operand" "0")
1083                  (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
1084   "TARGET_3DNOW"
1085   "pfcmpgt\t{%2, %0|%0, %2}"
1086   [(set_attr "type" "mmxcmp")
1087    (set_attr "prefix_extra" "1")
1088    (set_attr "mode" "V2SF")])
1089
1090 (define_insn "mmx_gev2sf3"
1091   [(set (match_operand:V2SI 0 "register_operand" "=y")
1092         (ge:V2SI (match_operand:V2SF 1 "register_operand" "0")
1093                  (match_operand:V2SF 2 "nonimmediate_operand" "ym")))]
1094   "TARGET_3DNOW"
1095   "pfcmpge\t{%2, %0|%0, %2}"
1096   [(set_attr "type" "mmxcmp")
1097    (set_attr "prefix_extra" "1")
1098    (set_attr "mode" "V2SF")])
1099
1100 (define_expand "vec_cmpv2sfv2si"
1101   [(set (match_operand:V2SI 0 "register_operand")
1102         (match_operator:V2SI 1 ""
1103           [(match_operand:V2SF 2 "nonimmediate_operand")
1104            (match_operand:V2SF 3 "nonimmediate_operand")]))]
1105   "TARGET_MMX_WITH_SSE"
1106 {
1107   rtx ops[4];
1108   ops[3] = gen_reg_rtx (V4SFmode);
1109   ops[2] = gen_reg_rtx (V4SFmode);
1110   ops[1] = gen_rtx_fmt_ee (GET_CODE (operands[1]), V4SImode, ops[2], ops[3]);
1111   ops[0] = gen_reg_rtx (V4SImode);
1112
1113   emit_insn (gen_movq_v2sf_to_sse (ops[3], operands[3]));
1114   emit_insn (gen_movq_v2sf_to_sse (ops[2], operands[2]));
1115
1116   bool ok = ix86_expand_fp_vec_cmp (ops);
1117   gcc_assert (ok);
1118
1119   emit_move_insn (operands[0], lowpart_subreg (V2SImode, ops[0], V4SImode));
1120   DONE;
1121 })
1122
1123 (define_expand "vcond<mode>v2sf"
1124   [(set (match_operand:V2FI 0 "register_operand")
1125         (if_then_else:V2FI
1126           (match_operator 3 ""
1127             [(match_operand:V2SF 4 "nonimmediate_operand")
1128              (match_operand:V2SF 5 "nonimmediate_operand")])
1129           (match_operand:V2FI 1 "general_operand")
1130           (match_operand:V2FI 2 "general_operand")))]
1131   "TARGET_MMX_WITH_SSE"
1132 {
1133   rtx ops[6];
1134   ops[5] = gen_reg_rtx (V4SFmode);
1135   ops[4] = gen_reg_rtx (V4SFmode);
1136   ops[3] = gen_rtx_fmt_ee (GET_CODE (operands[3]), VOIDmode, ops[4], ops[5]);
1137   ops[2] = lowpart_subreg (<mmxdoublevecmode>mode,
1138                            force_reg (<MODE>mode, operands[2]),
1139                            <MODE>mode);
1140   ops[1] = lowpart_subreg (<mmxdoublevecmode>mode,
1141                            force_reg (<MODE>mode, operands[1]),
1142                            <MODE>mode);
1143   ops[0] = gen_reg_rtx (<mmxdoublevecmode>mode);
1144
1145   emit_insn (gen_movq_v2sf_to_sse (ops[5], operands[5]));
1146   emit_insn (gen_movq_v2sf_to_sse (ops[4], operands[4]));
1147
1148   bool ok = ix86_expand_fp_vcond (ops);
1149   gcc_assert (ok);
1150
1151   emit_move_insn (operands[0], lowpart_subreg (<MODE>mode, ops[0],
1152                                                <mmxdoublevecmode>mode));
1153   DONE;
1154 })
1155
1156 (define_insn "@sse4_1_insertps_<mode>"
1157   [(set (match_operand:V2FI 0 "register_operand" "=Yr,*x,v")
1158         (unspec:V2FI
1159           [(match_operand:V2FI 2 "nonimmediate_operand" "Yrm,*xm,vm")
1160            (match_operand:V2FI 1 "register_operand" "0,0,v")
1161            (match_operand:SI 3 "const_0_to_255_operand")]
1162           UNSPEC_INSERTPS))]
1163   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
1164 {
1165   if (MEM_P (operands[2]))
1166     {
1167       unsigned count_s = INTVAL (operands[3]) >> 6;
1168       if (count_s)
1169         operands[3] = GEN_INT (INTVAL (operands[3]) & 0x3f);
1170       operands[2] = adjust_address_nv (operands[2],
1171                                        <mmxscalarmode>mode, count_s * 4);
1172     }
1173   switch (which_alternative)
1174     {
1175     case 0:
1176     case 1:
1177       return "insertps\t{%3, %2, %0|%0, %2, %3}";
1178     case 2:
1179       return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
1180     default:
1181       gcc_unreachable ();
1182     }
1183 }
1184   [(set_attr "isa" "noavx,noavx,avx")
1185    (set_attr "type" "sselog")
1186    (set_attr "prefix_data16" "1,1,*")
1187    (set_attr "prefix_extra" "1")
1188    (set_attr "length_immediate" "1")
1189    (set_attr "prefix" "orig,orig,maybe_evex")
1190    (set_attr "mode" "V4SF")])
1191
1192 (define_insn "*mmx_blendps"
1193   [(set (match_operand:V2SF 0 "register_operand" "=Yr,*x,x")
1194         (vec_merge:V2SF
1195           (match_operand:V2SF 2 "register_operand" "Yr,*x,x")
1196           (match_operand:V2SF 1 "register_operand" "0,0,x")
1197           (match_operand:SI 3 "const_0_to_3_operand")))]
1198   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
1199   "@
1200    blendps\t{%3, %2, %0|%0, %2, %3}
1201    blendps\t{%3, %2, %0|%0, %2, %3}
1202    vblendps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
1203   [(set_attr "isa" "noavx,noavx,avx")
1204    (set_attr "type" "ssemov")
1205    (set_attr "length_immediate" "1")
1206    (set_attr "prefix_data16" "1,1,*")
1207    (set_attr "prefix_extra" "1")
1208    (set_attr "prefix" "orig,orig,vex")
1209    (set_attr "mode" "V4SF")])
1210
1211 (define_insn "mmx_blendvps"
1212   [(set (match_operand:V2SF 0 "register_operand" "=Yr,*x,x")
1213         (unspec:V2SF
1214           [(match_operand:V2SF 1 "register_operand" "0,0,x")
1215            (match_operand:V2SF 2 "register_operand" "Yr,*x,x")
1216            (match_operand:V2SF 3 "register_operand" "Yz,Yz,x")]
1217           UNSPEC_BLENDV))]
1218   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
1219   "@
1220    blendvps\t{%3, %2, %0|%0, %2, %3}
1221    blendvps\t{%3, %2, %0|%0, %2, %3}
1222    vblendvps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
1223   [(set_attr "isa" "noavx,noavx,avx")
1224    (set_attr "type" "ssemov")
1225    (set_attr "length_immediate" "1")
1226    (set_attr "prefix_data16" "1,1,*")
1227    (set_attr "prefix_extra" "1")
1228    (set_attr "prefix" "orig,orig,vex")
1229    (set_attr "btver2_decode" "vector")
1230    (set_attr "mode" "V4SF")])
1231
1232 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1233 ;;
1234 ;; Parallel single-precision floating point logical operations
1235 ;;
1236 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1237
1238 (define_insn "*mmx_andnotv2sf3"
1239   [(set (match_operand:V2SF 0 "register_operand" "=x,x")
1240         (and:V2SF
1241           (not:V2SF
1242             (match_operand:V2SF 1 "register_operand" "0,x"))
1243           (match_operand:V2SF 2 "register_operand" "x,x")))]
1244   "TARGET_MMX_WITH_SSE"
1245   "@
1246    andnps\t{%2, %0|%0, %2}
1247    vandnps\t{%2, %1, %0|%0, %1, %2}"
1248   [(set_attr "isa" "noavx,avx")
1249    (set_attr "type" "sselog")
1250    (set_attr "prefix" "orig,vex")
1251    (set_attr "mode" "V4SF")])
1252
1253 (define_insn "<code>v2sf3"
1254   [(set (match_operand:V2SF 0 "register_operand" "=x,x")
1255         (any_logic:V2SF
1256           (match_operand:V2SF 1 "register_operand" "%0,x")
1257           (match_operand:V2SF 2 "register_operand" "x,x")))]
1258   "TARGET_MMX_WITH_SSE"
1259   "@
1260    <logic>ps\t{%2, %0|%0, %2}
1261    v<logic>ps\t{%2, %1, %0|%0, %1, %2}"
1262   [(set_attr "isa" "noavx,avx")
1263    (set_attr "type" "sselog")
1264    (set_attr "prefix" "orig,vex")
1265    (set_attr "mode" "V4SF")])
1266
1267 (define_expand "copysignv2sf3"
1268   [(set (match_dup 4)
1269         (and:V2SF
1270           (not:V2SF (match_dup 3))
1271           (match_operand:V2SF 1 "register_operand")))
1272    (set (match_dup 5)
1273         (and:V2SF (match_dup 3)
1274                   (match_operand:V2SF 2 "register_operand")))
1275    (set (match_operand:V2SF 0 "register_operand")
1276         (ior:V2SF (match_dup 4) (match_dup 5)))]
1277   "TARGET_MMX_WITH_SSE"
1278 {
1279   operands[3] = ix86_build_signbit_mask (V2SFmode, true, false);
1280
1281   operands[4] = gen_reg_rtx (V2SFmode);
1282   operands[5] = gen_reg_rtx (V2SFmode);
1283 })
1284
1285 (define_expand "xorsignv2sf3"
1286   [(set (match_dup 4)
1287         (and:V2SF (match_dup 3)
1288                   (match_operand:V2SF 2 "register_operand")))
1289    (set (match_operand:V2SF 0 "register_operand")
1290         (xor:V2SF (match_dup 4)
1291                   (match_operand:V2SF 1 "register_operand")))]
1292   "TARGET_MMX_WITH_SSE"
1293 {
1294   operands[3] = ix86_build_signbit_mask (V2SFmode, true, false);
1295
1296   operands[4] = gen_reg_rtx (V2SFmode);
1297 })
1298
1299 (define_expand "signbitv2sf2"
1300   [(set (match_operand:V2SI 0 "register_operand")
1301         (lshiftrt:V2SI
1302           (subreg:V2SI
1303             (match_operand:V2SF 1 "register_operand") 0)
1304           (match_dup 2)))]
1305   "TARGET_MMX_WITH_SSE"
1306   "operands[2] = GEN_INT (GET_MODE_UNIT_BITSIZE (V2SFmode)-1);")
1307
1308 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1309 ;;
1310 ;; Parallel single-precision FMA multiply/accumulate instructions.
1311 ;;
1312 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1313
1314 (define_expand "fmav2sf4"
1315   [(set (match_operand:V2SF 0 "register_operand")
1316         (fma:V2SF
1317           (match_operand:V2SF 1 "nonimmediate_operand")
1318           (match_operand:V2SF 2 "nonimmediate_operand")
1319           (match_operand:V2SF 3 "nonimmediate_operand")))]
1320   "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL)
1321    && TARGET_MMX_WITH_SSE"
1322 {
1323   rtx op3 = gen_reg_rtx (V4SFmode);
1324   rtx op2 = gen_reg_rtx (V4SFmode);
1325   rtx op1 = gen_reg_rtx (V4SFmode);
1326   rtx op0 = gen_reg_rtx (V4SFmode);
1327
1328   emit_insn (gen_movq_v2sf_to_sse (op3, operands[3]));
1329   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
1330   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1331
1332   emit_insn (gen_fmav4sf4 (op0, op1, op2, op3));
1333
1334   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1335   DONE;
1336 })
1337
1338 (define_expand "fmsv2sf4"
1339   [(set (match_operand:V2SF 0 "register_operand")
1340         (fma:V2SF
1341           (match_operand:V2SF   1 "nonimmediate_operand")
1342           (match_operand:V2SF   2 "nonimmediate_operand")
1343           (neg:V2SF
1344             (match_operand:V2SF 3 "nonimmediate_operand"))))]
1345   "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL)
1346    && TARGET_MMX_WITH_SSE"
1347 {
1348   rtx op3 = gen_reg_rtx (V4SFmode);
1349   rtx op2 = gen_reg_rtx (V4SFmode);
1350   rtx op1 = gen_reg_rtx (V4SFmode);
1351   rtx op0 = gen_reg_rtx (V4SFmode);
1352
1353   emit_insn (gen_movq_v2sf_to_sse (op3, operands[3]));
1354   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
1355   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1356
1357   emit_insn (gen_fmsv4sf4 (op0, op1, op2, op3));
1358
1359   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1360   DONE;
1361 })
1362
1363 (define_expand "fnmav2sf4"
1364   [(set (match_operand:V2SF 0 "register_operand")
1365         (fma:V2SF
1366           (neg:V2SF
1367             (match_operand:V2SF 1 "nonimmediate_operand"))
1368           (match_operand:V2SF   2 "nonimmediate_operand")
1369           (match_operand:V2SF   3 "nonimmediate_operand")))]
1370   "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL)
1371    && TARGET_MMX_WITH_SSE"
1372 {
1373   rtx op3 = gen_reg_rtx (V4SFmode);
1374   rtx op2 = gen_reg_rtx (V4SFmode);
1375   rtx op1 = gen_reg_rtx (V4SFmode);
1376   rtx op0 = gen_reg_rtx (V4SFmode);
1377
1378   emit_insn (gen_movq_v2sf_to_sse (op3, operands[3]));
1379   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
1380   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1381
1382   emit_insn (gen_fnmav4sf4 (op0, op1, op2, op3));
1383
1384   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1385   DONE;
1386 })
1387
1388 (define_expand "fnmsv2sf4"
1389   [(set (match_operand:V2SF 0 "register_operand" "=v,v,x")
1390         (fma:V2SF
1391           (neg:V2SF
1392             (match_operand:V2SF 1 "nonimmediate_operand"))
1393           (match_operand:V2SF   2 "nonimmediate_operand")
1394           (neg:V2SF
1395             (match_operand:V2SF 3 "nonimmediate_operand"))))]
1396   "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL)
1397    && TARGET_MMX_WITH_SSE"
1398 {
1399   rtx op3 = gen_reg_rtx (V4SFmode);
1400   rtx op2 = gen_reg_rtx (V4SFmode);
1401   rtx op1 = gen_reg_rtx (V4SFmode);
1402   rtx op0 = gen_reg_rtx (V4SFmode);
1403
1404   emit_insn (gen_movq_v2sf_to_sse (op3, operands[3]));
1405   emit_insn (gen_movq_v2sf_to_sse (op2, operands[2]));
1406   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1407
1408   emit_insn (gen_fnmsv4sf4 (op0, op1, op2, op3));
1409
1410   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1411   DONE;
1412 })
1413
1414 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1415 ;;
1416 ;; Parallel single-precision floating point conversion operations
1417 ;;
1418 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1419
1420 (define_expand "fix_truncv2sfv2si2"
1421   [(set (match_operand:V2SI 0 "register_operand")
1422         (fix:V2SI (match_operand:V2SF 1 "nonimmediate_operand")))]
1423   "TARGET_MMX_WITH_SSE"
1424 {
1425   rtx op1 = gen_reg_rtx (V4SFmode);
1426   rtx op0 = gen_reg_rtx (V4SImode);
1427
1428   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1429
1430   emit_insn (gen_fix_truncv4sfv4si2 (op0, op1));
1431
1432   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1433   DONE;
1434 })
1435
1436 (define_expand "fixuns_truncv2sfv2si2"
1437   [(set (match_operand:V2SI 0 "register_operand")
1438         (unsigned_fix:V2SI (match_operand:V2SF 1 "nonimmediate_operand")))]
1439   "TARGET_AVX512VL && TARGET_MMX_WITH_SSE"
1440 {
1441   rtx op1 = gen_reg_rtx (V4SFmode);
1442   rtx op0 = gen_reg_rtx (V4SImode);
1443
1444   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1445
1446   emit_insn (gen_fixuns_truncv4sfv4si2 (op0, op1));
1447
1448   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1449   DONE;
1450 })
1451
1452 (define_insn "mmx_fix_truncv2sfv2si2"
1453   [(set (match_operand:V2SI 0 "register_operand" "=y")
1454         (fix:V2SI (match_operand:V2SF 1 "nonimmediate_operand" "ym")))]
1455   "TARGET_3DNOW"
1456   "pf2id\t{%1, %0|%0, %1}"
1457   [(set_attr "type" "mmxcvt")
1458    (set_attr "prefix_extra" "1")
1459    (set_attr "mode" "V2SF")])
1460
1461 (define_expand "floatv2siv2sf2"
1462   [(set (match_operand:V2SF 0 "register_operand")
1463         (float:V2SF (match_operand:V2SI 1 "nonimmediate_operand")))]
1464   "TARGET_MMX_WITH_SSE"
1465 {
1466   rtx op1 = gen_reg_rtx (V4SImode);
1467   rtx op0 = gen_reg_rtx (V4SFmode);
1468
1469   emit_insn (gen_movq_v2si_to_sse (op1, operands[1]));
1470
1471   emit_insn (gen_floatv4siv4sf2 (op0, op1));
1472
1473   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1474   DONE;
1475 })
1476
1477 (define_expand "floatunsv2siv2sf2"
1478   [(set (match_operand:V2SF 0 "register_operand")
1479         (unsigned_float:V2SF (match_operand:V2SI 1 "nonimmediate_operand")))]
1480   "TARGET_AVX512VL && TARGET_MMX_WITH_SSE"
1481 {
1482   rtx op1 = gen_reg_rtx (V4SImode);
1483   rtx op0 = gen_reg_rtx (V4SFmode);
1484
1485   emit_insn (gen_movq_v2si_to_sse (op1, operands[1]));
1486
1487   emit_insn (gen_floatunsv4siv4sf2 (op0, op1));
1488
1489   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1490   DONE;
1491 })
1492
1493 (define_insn "mmx_floatv2siv2sf2"
1494   [(set (match_operand:V2SF 0 "register_operand" "=y")
1495         (float:V2SF (match_operand:V2SI 1 "nonimmediate_operand" "ym")))]
1496   "TARGET_3DNOW"
1497   "pi2fd\t{%1, %0|%0, %1}"
1498   [(set_attr "type" "mmxcvt")
1499    (set_attr "prefix_extra" "1")
1500    (set_attr "mode" "V2SF")])
1501
1502 (define_insn "mmx_pf2iw"
1503   [(set (match_operand:V2SI 0 "register_operand" "=y")
1504         (sign_extend:V2SI
1505           (ss_truncate:V2HI
1506             (fix:V2SI
1507               (match_operand:V2SF 1 "nonimmediate_operand" "ym")))))]
1508   "TARGET_3DNOW_A"
1509   "pf2iw\t{%1, %0|%0, %1}"
1510   [(set_attr "type" "mmxcvt")
1511    (set_attr "prefix_extra" "1")
1512    (set_attr "mode" "V2SF")])
1513
1514 (define_insn "mmx_pi2fw"
1515   [(set (match_operand:V2SF 0 "register_operand" "=y")
1516         (float:V2SF
1517           (sign_extend:V2SI
1518             (truncate:V2HI
1519               (match_operand:V2SI 1 "nonimmediate_operand" "ym")))))]
1520   "TARGET_3DNOW_A"
1521   "pi2fw\t{%1, %0|%0, %1}"
1522   [(set_attr "type" "mmxcvt")
1523    (set_attr "prefix_extra" "1")
1524    (set_attr "mode" "V2SF")])
1525
1526 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1527 ;;
1528 ;; Parallel single-precision floating point element swizzling
1529 ;;
1530 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1531
1532 (define_insn "mmx_pswapdv2sf2"
1533   [(set (match_operand:V2SF 0 "register_operand" "=y,x,Yv")
1534         (vec_select:V2SF
1535           (match_operand:V2SF 1 "register_mmxmem_operand" "ym,0,Yv")
1536           (parallel [(const_int 1) (const_int 0)])))]
1537   "TARGET_3DNOW_A || TARGET_MMX_WITH_SSE"
1538   "@
1539    pswapd\t{%1, %0|%0, %1}
1540    shufps\t{$0xe1, %1, %0|%0, %1, 0xe1}
1541    vshufps\t{$0xe1, %1, %1, %0|%0, %1, %1, 0xe1}"
1542   [(set_attr "isa" "*,sse_noavx,avx")
1543    (set_attr "mmx_isa" "native,*,*")
1544    (set_attr "type" "mmxcvt,ssemov,ssemov")
1545    (set_attr "prefix_extra" "1,*,*")
1546    (set_attr "mode" "V2SF,V4SF,V4SF")])
1547
1548 (define_insn "*mmx_movshdup"
1549   [(set (match_operand:V2SF 0 "register_operand" "=v,x")
1550         (vec_select:V2SF
1551           (match_operand:V2SF 1 "register_operand" "v,0")
1552           (parallel [(const_int 1) (const_int 1)])))]
1553   "TARGET_MMX_WITH_SSE"
1554   "@
1555    %vmovshdup\t{%1, %0|%0, %1}
1556    shufps\t{$0xe5, %0, %0|%0, %0, 0xe5}"
1557   [(set_attr "isa" "sse3,*")
1558    (set_attr "type" "sse,sseshuf1")
1559    (set_attr "length_immediate" "*,1")
1560    (set_attr "prefix_rep" "1,*")
1561    (set_attr "prefix" "maybe_vex,orig")
1562    (set_attr "mode" "V4SF")])
1563
1564 (define_insn "*mmx_movsldup"
1565   [(set (match_operand:V2SF 0 "register_operand" "=v,x")
1566         (vec_select:V2SF
1567           (match_operand:V2SF 1 "register_operand" "v,0")
1568           (parallel [(const_int 0) (const_int 0)])))]
1569   "TARGET_MMX_WITH_SSE"
1570   "@
1571    %vmovsldup\t{%1, %0|%0, %1}
1572    shufps\t{$0xe0, %0, %0|%0, %0, 0xe0}"
1573   [(set_attr "isa" "sse3,*")
1574    (set_attr "type" "sse,sseshuf1")
1575    (set_attr "length_immediate" "*,1")
1576    (set_attr "prefix_rep" "1,*")
1577    (set_attr "prefix" "maybe_vex,orig")
1578    (set_attr "mode" "V4SF")])
1579
1580 (define_insn_and_split "*vec_interleave_lowv2sf"
1581   [(set (match_operand:V2SF 0 "register_operand" "=x,v")
1582         (vec_select:V2SF
1583           (vec_concat:V4SF
1584             (match_operand:V2SF 1 "register_operand" "0,v")
1585             (match_operand:V2SF 2 "register_operand" "x,v"))
1586           (parallel [(const_int 0) (const_int 2)])))]
1587   "TARGET_MMX_WITH_SSE"
1588   "#"
1589   "&& reload_completed"
1590   [(const_int 0)]
1591   "ix86_split_mmx_punpck (operands, false); DONE;"
1592   [(set_attr "isa" "noavx,avx")
1593    (set_attr "type" "sselog")
1594    (set_attr "prefix" "orig,maybe_evex")
1595    (set_attr "mode" "V4SF")])
1596
1597 (define_insn_and_split "*vec_interleave_highv2sf"
1598   [(set (match_operand:V2SF 0 "register_operand" "=x,v")
1599         (vec_select:V2SF
1600           (vec_concat:V4SF
1601             (match_operand:V2SF 1 "register_operand" "0,v")
1602             (match_operand:V2SF 2 "register_operand" "x,v"))
1603           (parallel [(const_int 1) (const_int 3)])))]
1604   "TARGET_MMX_WITH_SSE"
1605   "#"
1606   "&& reload_completed"
1607   [(const_int 0)]
1608   "ix86_split_mmx_punpck (operands, true); DONE;"
1609   [(set_attr "isa" "noavx,avx")
1610    (set_attr "type" "sselog")
1611    (set_attr "prefix" "orig,vex")
1612    (set_attr "mode" "V4SF")])
1613
1614 (define_insn "*vec_dupv2sf"
1615   [(set (match_operand:V2SF 0 "register_operand" "=y,Yv,x")
1616         (vec_duplicate:V2SF
1617           (match_operand:SF 1 "register_operand" "0,Yv,0")))]
1618   "TARGET_MMX || TARGET_MMX_WITH_SSE"
1619   "@
1620    punpckldq\t%0, %0
1621    %vmovsldup\t{%1, %0|%0, %1}
1622    shufps\t{$0xe0, %0, %0|%0, %0, 0xe0}"
1623   [(set_attr "isa" "*,sse3,sse_noavx")
1624    (set_attr "mmx_isa" "native,*,*")
1625    (set_attr "type" "mmxcvt,sse,sseshuf1")
1626    (set_attr "length_immediate" "*,*,1")
1627    (set_attr "prefix_rep" "*,1,*")
1628    (set_attr "prefix" "*,maybe_vex,orig")
1629    (set_attr "mode" "DI,V4SF,V4SF")])
1630
1631 (define_insn "*mmx_movss_<mode>"
1632   [(set (match_operand:V2FI 0 "register_operand"   "=x,v")
1633         (vec_merge:V2FI
1634           (match_operand:V2FI 2 "register_operand" " x,v")
1635           (match_operand:V2FI 1 "register_operand" " 0,v")
1636           (const_int 1)))]
1637   "TARGET_MMX_WITH_SSE"
1638   "@
1639    movss\t{%2, %0|%0, %2}
1640    vmovss\t{%2, %1, %0|%0, %1, %2}"
1641   [(set_attr "isa" "noavx,avx")
1642    (set_attr "type" "ssemov")
1643    (set_attr "prefix" "orig,maybe_evex")
1644    (set_attr "mode" "SF")])
1645
1646 (define_insn "*mmx_concatv2sf"
1647   [(set (match_operand:V2SF 0 "register_operand"     "=y,y")
1648         (vec_concat:V2SF
1649           (match_operand:SF 1 "nonimmediate_operand" " 0,rm")
1650           (match_operand:SF 2 "nonimm_or_0_operand"  "ym,C")))]
1651   "TARGET_MMX && !TARGET_SSE"
1652   "@
1653    punpckldq\t{%2, %0|%0, %2}
1654    movd\t{%1, %0|%0, %1}"
1655   [(set_attr "type" "mmxcvt,mmxmov")
1656    (set_attr "mode" "DI")])
1657
1658 (define_expand "vec_setv2sf"
1659   [(match_operand:V2SF 0 "register_operand")
1660    (match_operand:SF 1 "register_operand")
1661    (match_operand 2 "vec_setm_mmx_operand")]
1662   "TARGET_MMX || TARGET_MMX_WITH_SSE"
1663 {
1664   if (CONST_INT_P (operands[2]))
1665     ix86_expand_vector_set (TARGET_MMX_WITH_SSE, operands[0], operands[1],
1666                             INTVAL (operands[2]));
1667   else
1668     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
1669   DONE;
1670 })
1671
1672 ;; Avoid combining registers from different units in a single alternative,
1673 ;; see comment above inline_secondary_memory_needed function in i386.cc
1674 (define_insn_and_split "*vec_extractv2sf_0"
1675   [(set (match_operand:SF 0 "nonimmediate_operand"     "=x, m,y ,m,f,r")
1676         (vec_select:SF
1677           (match_operand:V2SF 1 "nonimmediate_operand" " xm,x,ym,y,m,m")
1678           (parallel [(const_int 0)])))]
1679   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
1680    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
1681   "#"
1682   "&& reload_completed"
1683   [(set (match_dup 0) (match_dup 1))]
1684   "operands[1] = gen_lowpart (SFmode, operands[1]);"
1685   [(set_attr "mmx_isa" "*,*,native,native,*,*")])
1686
1687 ;; Avoid combining registers from different units in a single alternative,
1688 ;; see comment above inline_secondary_memory_needed function in i386.cc
1689 (define_insn "*vec_extractv2sf_1"
1690   [(set (match_operand:SF 0 "nonimmediate_operand"     "=y,x,x,y,x,f,r")
1691         (vec_select:SF
1692           (match_operand:V2SF 1 "nonimmediate_operand" " 0,x,0,o,o,o,o")
1693           (parallel [(const_int 1)])))]
1694   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
1695    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
1696   "@
1697    punpckhdq\t%0, %0
1698    %vmovshdup\t{%1, %0|%0, %1}
1699    shufps\t{$0xe5, %0, %0|%0, %0, 0xe5}
1700    #
1701    #
1702    #
1703    #"
1704   [(set_attr "isa" "*,sse3,noavx,*,*,*,*")
1705    (set_attr "mmx_isa" "native,*,*,native,*,*,*")
1706    (set_attr "type" "mmxcvt,sse,sseshuf1,mmxmov,ssemov,fmov,imov")
1707    (set (attr "length_immediate")
1708      (if_then_else (eq_attr "alternative" "2")
1709                    (const_string "1")
1710                    (const_string "*")))
1711    (set (attr "prefix_rep")
1712      (if_then_else (eq_attr "alternative" "1")
1713                    (const_string "1")
1714                    (const_string "*")))
1715    (set_attr "prefix" "orig,maybe_vex,orig,orig,orig,orig,orig")
1716    (set_attr "mode" "DI,V4SF,V4SF,SF,SF,SF,SF")])
1717
1718 (define_split
1719   [(set (match_operand:SF 0 "register_operand")
1720         (vec_select:SF
1721           (match_operand:V2SF 1 "memory_operand")
1722           (parallel [(const_int 1)])))]
1723   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && reload_completed"
1724   [(set (match_dup 0) (match_dup 1))]
1725   "operands[1] = adjust_address (operands[1], SFmode, 4);")
1726
1727 (define_expand "vec_extractv2sfsf"
1728   [(match_operand:SF 0 "register_operand")
1729    (match_operand:V2SF 1 "register_operand")
1730    (match_operand 2 "const_int_operand")]
1731   "TARGET_MMX || TARGET_MMX_WITH_SSE"
1732 {
1733   ix86_expand_vector_extract (TARGET_MMX_WITH_SSE, operands[0],
1734                               operands[1], INTVAL (operands[2]));
1735   DONE;
1736 })
1737
1738 (define_expand "vec_initv2sfsf"
1739   [(match_operand:V2SF 0 "register_operand")
1740    (match_operand 1)]
1741   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
1742 {
1743   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
1744                            operands[1]);
1745   DONE;
1746 })
1747
1748 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1749 ;;
1750 ;; Parallel single-precision floating point rounding operations.
1751 ;;
1752 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1753
1754 (define_expand "nearbyintv2sf2"
1755   [(match_operand:V2SF 0 "register_operand")
1756    (match_operand:V2SF 1 "nonimmediate_operand")]
1757   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
1758 {
1759   rtx op1 = gen_reg_rtx (V4SFmode);
1760   rtx op0 = gen_reg_rtx (V4SFmode);
1761
1762   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1763
1764   emit_insn (gen_nearbyintv4sf2 (op0, op1));
1765
1766   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1767   DONE;
1768 })
1769
1770 (define_expand "rintv2sf2"
1771   [(match_operand:V2SF 0 "register_operand")
1772    (match_operand:V2SF 1 "nonimmediate_operand")]
1773   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
1774 {
1775   rtx op1 = gen_reg_rtx (V4SFmode);
1776   rtx op0 = gen_reg_rtx (V4SFmode);
1777
1778   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1779
1780   emit_insn (gen_rintv4sf2 (op0, op1));
1781
1782   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1783   DONE;
1784 })
1785
1786 (define_expand "lrintv2sfv2si2"
1787   [(match_operand:V2SI 0 "register_operand")
1788    (match_operand:V2SF 1 "nonimmediate_operand")]
1789  "TARGET_SSE4_1 && !flag_trapping_math
1790   && TARGET_MMX_WITH_SSE"
1791 {
1792   rtx op1 = gen_reg_rtx (V4SFmode);
1793   rtx op0 = gen_reg_rtx (V4SImode);
1794
1795   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1796
1797   emit_insn (gen_lrintv4sfv4si2 (op0, op1));
1798
1799   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1800   DONE;
1801 })
1802
1803 (define_expand "ceilv2sf2"
1804   [(match_operand:V2SF 0 "register_operand")
1805    (match_operand:V2SF 1 "nonimmediate_operand")]
1806   "TARGET_SSE4_1 && !flag_trapping_math
1807    && TARGET_MMX_WITH_SSE"
1808 {
1809   rtx op1 = gen_reg_rtx (V4SFmode);
1810   rtx op0 = gen_reg_rtx (V4SFmode);
1811
1812   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1813
1814   emit_insn (gen_ceilv4sf2 (op0, op1));
1815
1816   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1817   DONE;
1818 })
1819
1820 (define_expand "lceilv2sfv2si2"
1821   [(match_operand:V2SI 0 "register_operand")
1822    (match_operand:V2SF 1 "nonimmediate_operand")]
1823  "TARGET_SSE4_1 && !flag_trapping_math
1824   && TARGET_MMX_WITH_SSE"
1825 {
1826   rtx op1 = gen_reg_rtx (V4SFmode);
1827   rtx op0 = gen_reg_rtx (V4SImode);
1828
1829   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1830
1831   emit_insn (gen_lceilv4sfv4si2 (op0, op1));
1832
1833   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1834   DONE;
1835 })
1836
1837 (define_expand "floorv2sf2"
1838   [(match_operand:V2SF 0 "register_operand")
1839    (match_operand:V2SF 1 "nonimmediate_operand")]
1840   "TARGET_SSE4_1 && !flag_trapping_math
1841   && TARGET_MMX_WITH_SSE"
1842 {
1843   rtx op1 = gen_reg_rtx (V4SFmode);
1844   rtx op0 = gen_reg_rtx (V4SFmode);
1845
1846   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1847
1848   emit_insn (gen_floorv4sf2 (op0, op1));
1849
1850   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1851   DONE;
1852 })
1853
1854 (define_expand "lfloorv2sfv2si2"
1855   [(match_operand:V2SI 0 "register_operand")
1856    (match_operand:V2SF 1 "nonimmediate_operand")]
1857  "TARGET_SSE4_1 && !flag_trapping_math
1858   && TARGET_MMX_WITH_SSE"
1859 {
1860   rtx op1 = gen_reg_rtx (V4SFmode);
1861   rtx op0 = gen_reg_rtx (V4SImode);
1862
1863   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1864
1865   emit_insn (gen_lfloorv4sfv4si2 (op0, op1));
1866
1867   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1868   DONE;
1869 })
1870
1871 (define_expand "btruncv2sf2"
1872   [(match_operand:V2SF 0 "register_operand")
1873    (match_operand:V2SF 1 "nonimmediate_operand")]
1874   "TARGET_SSE4_1 && !flag_trapping_math
1875   && TARGET_MMX_WITH_SSE"
1876 {
1877   rtx op1 = gen_reg_rtx (V4SFmode);
1878   rtx op0 = gen_reg_rtx (V4SFmode);
1879
1880   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1881
1882   emit_insn (gen_btruncv4sf2 (op0, op1));
1883
1884   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1885   DONE;
1886 })
1887
1888 (define_expand "roundv2sf2"
1889   [(match_operand:V2SF 0 "register_operand")
1890    (match_operand:V2SF 1 "nonimmediate_operand")]
1891   "TARGET_SSE4_1 && !flag_trapping_math
1892    && TARGET_MMX_WITH_SSE"
1893 {
1894   rtx op1 = gen_reg_rtx (V4SFmode);
1895   rtx op0 = gen_reg_rtx (V4SFmode);
1896
1897   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1898
1899   emit_insn (gen_roundv4sf2 (op0, op1));
1900
1901   emit_move_insn (operands[0], lowpart_subreg (V2SFmode, op0, V4SFmode));
1902   DONE;
1903 })
1904
1905 (define_expand "lroundv2sfv2si2"
1906   [(match_operand:V2SI 0 "register_operand")
1907    (match_operand:V2SF 1 "nonimmediate_operand")]
1908  "TARGET_SSE4_1 && !flag_trapping_math
1909   && TARGET_MMX_WITH_SSE"
1910 {
1911   rtx op1 = gen_reg_rtx (V4SFmode);
1912   rtx op0 = gen_reg_rtx (V4SImode);
1913
1914   emit_insn (gen_movq_v2sf_to_sse (op1, operands[1]));
1915
1916   emit_insn (gen_lroundv4sfv4si2 (op0, op1));
1917
1918   emit_move_insn (operands[0], lowpart_subreg (V2SImode, op0, V4SImode));
1919   DONE;
1920 })
1921
1922 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1923 ;;
1924 ;; Parallel half-precision floating point arithmetic
1925 ;;
1926 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1927
1928 (define_expand "<insn>v4hf3"
1929   [(set (match_operand:V4HF 0 "register_operand")
1930         (plusminusmult:V4HF
1931           (match_operand:V4HF 1 "nonimmediate_operand")
1932           (match_operand:V4HF 2 "nonimmediate_operand")))]
1933   "TARGET_AVX512FP16 && TARGET_AVX512VL"
1934 {
1935   rtx op2 = gen_reg_rtx (V8HFmode);
1936   rtx op1 = gen_reg_rtx (V8HFmode);
1937   rtx op0 = gen_reg_rtx (V8HFmode);
1938
1939   emit_insn (gen_movq_v4hf_to_sse (op2, operands[2]));
1940   emit_insn (gen_movq_v4hf_to_sse (op1, operands[1]));
1941
1942   emit_insn (gen_<insn>v8hf3 (op0, op1, op2));
1943
1944   emit_move_insn (operands[0], lowpart_subreg (V4HFmode, op0, V8HFmode));
1945   DONE;
1946 })
1947
1948 (define_expand "divv4hf3"
1949   [(set (match_operand:V4HF 0 "register_operand")
1950         (div:V4HF
1951           (match_operand:V4HF 1 "nonimmediate_operand")
1952           (match_operand:V4HF 2 "nonimmediate_operand")))]
1953   "TARGET_AVX512FP16 && TARGET_AVX512VL"
1954 {
1955   rtx op2 = gen_reg_rtx (V8HFmode);
1956   rtx op1 = gen_reg_rtx (V8HFmode);
1957   rtx op0 = gen_reg_rtx (V8HFmode);
1958
1959   emit_insn (gen_movq_v4hf_to_sse (op1, operands[1]));
1960   rtx tmp = gen_rtx_VEC_CONCAT (V8HFmode, operands[2],
1961                                 force_reg (V4HFmode, CONST1_RTX (V4HFmode)));
1962   emit_insn (gen_rtx_SET (op2, tmp));
1963   emit_insn (gen_divv8hf3 (op0, op1, op2));
1964   emit_move_insn (operands[0], lowpart_subreg (V4HFmode, op0, V8HFmode));
1965   DONE;
1966 })
1967
1968 (define_expand "movd_v2hf_to_sse"
1969   [(set (match_operand:V8HF 0 "register_operand")
1970         (vec_merge:V8HF
1971           (vec_duplicate:V8HF
1972             (match_operand:V2HF 1 "nonimmediate_operand"))
1973           (match_operand:V8HF 2 "reg_or_0_operand")
1974           (const_int 3)))]
1975   "TARGET_SSE")
1976
1977 (define_expand "<insn>v2hf3"
1978   [(set (match_operand:V2HF 0 "register_operand")
1979         (plusminusmult:V2HF
1980           (match_operand:V2HF 1 "nonimmediate_operand")
1981           (match_operand:V2HF 2 "nonimmediate_operand")))]
1982   "TARGET_AVX512FP16 && TARGET_AVX512VL"
1983 {
1984   rtx op2 = gen_reg_rtx (V8HFmode);
1985   rtx op1 = gen_reg_rtx (V8HFmode);
1986   rtx op0 = gen_reg_rtx (V8HFmode);
1987
1988   emit_insn (gen_movd_v2hf_to_sse (op2, operands[2], CONST0_RTX (V8HFmode)));
1989   emit_insn (gen_movd_v2hf_to_sse (op1, operands[1], CONST0_RTX (V8HFmode)));
1990   emit_insn (gen_<insn>v8hf3 (op0, op1, op2));
1991
1992   emit_move_insn (operands[0], lowpart_subreg (V2HFmode, op0, V8HFmode));
1993   DONE;
1994 })
1995
1996 (define_expand "divv2hf3"
1997   [(set (match_operand:V2HF 0 "register_operand")
1998         (div:V2HF
1999           (match_operand:V2HF 1 "nonimmediate_operand")
2000           (match_operand:V2HF 2 "nonimmediate_operand")))]
2001   "TARGET_AVX512FP16 && TARGET_AVX512VL"
2002 {
2003   rtx op2 = gen_reg_rtx (V8HFmode);
2004   rtx op1 = gen_reg_rtx (V8HFmode);
2005   rtx op0 = gen_reg_rtx (V8HFmode);
2006
2007   emit_insn (gen_movd_v2hf_to_sse (op2, operands[2],
2008                                   force_reg (V8HFmode, CONST1_RTX (V8HFmode))));
2009   emit_insn (gen_movd_v2hf_to_sse (op1, operands[1], CONST0_RTX (V8HFmode)));
2010   emit_insn (gen_divv8hf3 (op0, op1, op2));
2011
2012   emit_move_insn (operands[0], lowpart_subreg (V2HFmode, op0, V8HFmode));
2013   DONE;
2014 })
2015
2016
2017 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2018 ;;
2019 ;; Parallel integral arithmetic
2020 ;;
2021 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2022
2023 (define_expand "neg<mode>2"
2024   [(set (match_operand:MMXMODEI 0 "register_operand")
2025         (minus:MMXMODEI
2026           (match_dup 2)
2027           (match_operand:MMXMODEI 1 "register_operand")))]
2028   "TARGET_MMX_WITH_SSE"
2029   "operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
2030
2031 (define_expand "neg<mode>2"
2032   [(set (match_operand:VI_32 0 "register_operand")
2033         (minus:VI_32
2034           (match_dup 2)
2035           (match_operand:VI_32 1 "register_operand")))]
2036   "TARGET_SSE2"
2037   "operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
2038
2039 (define_insn "negv2qi2"
2040   [(set (match_operand:V2QI 0 "register_operand" "=?Q,&Yw")
2041         (neg:V2QI
2042           (match_operand:V2QI 1 "register_operand" "0,Yw")))
2043    (clobber (reg:CC FLAGS_REG))]
2044   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
2045   "#"
2046   [(set_attr "isa" "*,sse2")
2047    (set_attr "type" "multi")
2048    (set_attr "mode" "QI,TI")])
2049
2050 (define_split
2051   [(set (match_operand:V2QI 0 "general_reg_operand")
2052         (neg:V2QI
2053           (match_operand:V2QI 1 "general_reg_operand")))
2054    (clobber (reg:CC FLAGS_REG))]
2055   "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
2056    && reload_completed"
2057   [(parallel
2058      [(set (strict_low_part (match_dup 0))
2059            (neg:QI (match_dup 1)))
2060       (clobber (reg:CC FLAGS_REG))])
2061    (parallel
2062      [(set (zero_extract:HI (match_dup 2) (const_int 8) (const_int 8))
2063            (subreg:HI
2064              (neg:QI
2065                (subreg:QI
2066                  (zero_extract:HI (match_dup 3)
2067                                   (const_int 8)
2068                                   (const_int 8)) 0)) 0))
2069       (clobber (reg:CC FLAGS_REG))])]
2070 {
2071   operands[3] = lowpart_subreg (HImode, operands[1], V2QImode);
2072   operands[2] = lowpart_subreg (HImode, operands[0], V2QImode);
2073   operands[1] = lowpart_subreg (QImode, operands[1], V2QImode);
2074   operands[0] = lowpart_subreg (QImode, operands[0], V2QImode);
2075 })
2076
2077 (define_split
2078   [(set (match_operand:V2QI 0 "sse_reg_operand")
2079         (neg:V2QI
2080           (match_operand:V2QI 1 "sse_reg_operand")))
2081    (clobber (reg:CC FLAGS_REG))]
2082   "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
2083    && TARGET_SSE2 && reload_completed"
2084   [(set (match_dup 0) (match_dup 2))
2085    (set (match_dup 0)
2086         (minus:V16QI (match_dup 0) (match_dup 1)))]
2087 {
2088   operands[2] = CONST0_RTX (V16QImode);
2089   operands[1] = lowpart_subreg (V16QImode, operands[1], V2QImode);
2090   operands[0] = lowpart_subreg (V16QImode, operands[0], V2QImode);
2091 })
2092
2093 (define_expand "mmx_<insn><mode>3"
2094   [(set (match_operand:MMXMODEI8 0 "register_operand")
2095         (plusminus:MMXMODEI8
2096           (match_operand:MMXMODEI8 1 "register_mmxmem_operand")
2097           (match_operand:MMXMODEI8 2 "register_mmxmem_operand")))]
2098   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2099   "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
2100
2101 (define_expand "<insn><mode>3"
2102   [(set (match_operand:MMXMODEI 0 "register_operand")
2103         (plusminus:MMXMODEI
2104           (match_operand:MMXMODEI 1 "register_operand")
2105           (match_operand:MMXMODEI 2 "register_operand")))]
2106   "TARGET_MMX_WITH_SSE")
2107
2108 (define_insn "*mmx_<insn><mode>3"
2109   [(set (match_operand:MMXMODEI8 0 "register_operand" "=y,x,<Yv_Yw>")
2110         (plusminus:MMXMODEI8
2111           (match_operand:MMXMODEI8 1 "register_mmxmem_operand"
2112             "<comm>0,0,<Yv_Yw>")
2113           (match_operand:MMXMODEI8 2 "register_mmxmem_operand"
2114             "ym,x,<Yv_Yw>")))]
2115   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2116    && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
2117   "@
2118    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2119    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2120    vp<plusminus_mnemonic><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2121   [(set_attr "isa" "*,sse2_noavx,avx")
2122    (set_attr "mmx_isa" "native,*,*")
2123    (set_attr "type" "mmxadd,sseadd,sseadd")
2124    (set_attr "mode" "DI,TI,TI")])
2125
2126 (define_insn "<insn><mode>3"
2127   [(set (match_operand:VI_32 0 "register_operand" "=x,Yw")
2128         (plusminus:VI_32
2129           (match_operand:VI_32 1 "register_operand" "<comm>0,Yw")
2130           (match_operand:VI_32 2 "register_operand" "x,Yw")))]
2131   "TARGET_SSE2"
2132   "@
2133    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2134    vp<plusminus_mnemonic><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2135   [(set_attr "isa" "noavx,avx")
2136    (set_attr "type" "sseadd")
2137    (set_attr "mode" "TI")])
2138
2139 (define_insn "<insn>v2qi3"
2140   [(set (match_operand:V2QI 0 "register_operand" "=?Q,x,Yw")
2141         (plusminus:V2QI
2142           (match_operand:V2QI 1 "register_operand" "<comm>0,0,Yw")
2143           (match_operand:V2QI 2 "register_operand" "Q,x,Yw")))
2144    (clobber (reg:CC FLAGS_REG))]
2145   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
2146   "#"
2147   [(set_attr "isa" "*,sse2_noavx,avx")
2148    (set_attr "type" "multi,sseadd,sseadd")
2149    (set_attr "mode" "QI,TI,TI")])
2150
2151 (define_split
2152   [(set (match_operand:V2QI 0 "general_reg_operand")
2153         (plusminus:V2QI
2154           (match_operand:V2QI 1 "general_reg_operand")
2155           (match_operand:V2QI 2 "general_reg_operand")))
2156    (clobber (reg:CC FLAGS_REG))]
2157   "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
2158    && reload_completed"
2159   [(parallel
2160      [(set (strict_low_part (match_dup 0))
2161            (plusminus:QI (match_dup 1) (match_dup 2)))
2162       (clobber (reg:CC FLAGS_REG))])
2163    (parallel
2164      [(set (zero_extract:HI (match_dup 3) (const_int 8) (const_int 8))
2165            (subreg:HI
2166              (plusminus:QI
2167                (subreg:QI
2168                  (zero_extract:HI (match_dup 4)
2169                                   (const_int 8)
2170                                   (const_int 8)) 0)
2171                (subreg:QI
2172                  (zero_extract:HI (match_dup 5)
2173                                   (const_int 8)
2174                                   (const_int 8)) 0)) 0))
2175       (clobber (reg:CC FLAGS_REG))])]
2176 {
2177   operands[5] = lowpart_subreg (HImode, operands[2], V2QImode);
2178   operands[4] = lowpart_subreg (HImode, operands[1], V2QImode);
2179   operands[3] = lowpart_subreg (HImode, operands[0], V2QImode);
2180   operands[2] = lowpart_subreg (QImode, operands[2], V2QImode);
2181   operands[1] = lowpart_subreg (QImode, operands[1], V2QImode);
2182   operands[0] = lowpart_subreg (QImode, operands[0], V2QImode);
2183 })
2184
2185 (define_split
2186   [(set (match_operand:V2QI 0 "sse_reg_operand")
2187         (plusminus:V2QI
2188           (match_operand:V2QI 1 "sse_reg_operand")
2189           (match_operand:V2QI 2 "sse_reg_operand")))
2190    (clobber (reg:CC FLAGS_REG))]
2191   "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
2192    && TARGET_SSE2 && reload_completed"
2193   [(set (match_dup 0)
2194         (plusminus:V16QI (match_dup 1) (match_dup 2)))]
2195 {
2196   operands[2] = lowpart_subreg (V16QImode, operands[2], V2QImode);
2197   operands[1] = lowpart_subreg (V16QImode, operands[1], V2QImode);
2198   operands[0] = lowpart_subreg (V16QImode, operands[0], V2QImode);
2199 })
2200
2201 (define_expand "mmx_<insn><mode>3"
2202   [(set (match_operand:MMXMODE12 0 "register_operand")
2203         (sat_plusminus:MMXMODE12
2204           (match_operand:MMXMODE12 1 "register_mmxmem_operand")
2205           (match_operand:MMXMODE12 2 "register_mmxmem_operand")))]
2206   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2207   "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
2208
2209 (define_insn "*mmx_<insn><mode>3"
2210   [(set (match_operand:MMXMODE12 0 "register_operand" "=y,x,Yw")
2211         (sat_plusminus:MMXMODE12
2212           (match_operand:MMXMODE12 1 "register_mmxmem_operand" "<comm>0,0,Yw")
2213           (match_operand:MMXMODE12 2 "register_mmxmem_operand" "ym,x,Yw")))]
2214   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2215    && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
2216   "@
2217    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2218    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2219    vp<plusminus_mnemonic><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2220   [(set_attr "isa" "*,sse2_noavx,avx")
2221    (set_attr "mmx_isa" "native,*,*")
2222    (set_attr "type" "mmxadd,sseadd,sseadd")
2223    (set_attr "mode" "DI,TI,TI")])
2224
2225 (define_insn "*<insn><mode>3"
2226   [(set (match_operand:VI_16_32 0 "register_operand" "=x,Yw")
2227         (sat_plusminus:VI_16_32
2228           (match_operand:VI_16_32 1 "register_operand" "<comm>0,Yw")
2229           (match_operand:VI_16_32 2 "register_operand" "x,Yw")))]
2230   "TARGET_SSE2"
2231   "@
2232    p<plusminus_mnemonic><mmxvecsize>\t{%2, %0|%0, %2}
2233    vp<plusminus_mnemonic><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2234   [(set_attr "isa" "noavx,avx")
2235    (set_attr "type" "sseadd")
2236    (set_attr "mode" "TI")])
2237
2238 (define_insn "mulv2si3"
2239   [(set (match_operand:V2SI 0 "register_operand" "=Yr,*x,v")
2240         (mult:V2SI
2241           (match_operand:V2SI 1 "register_operand" "%0,0,v")
2242           (match_operand:V2SI 2 "register_operand" "Yr,*x,v")))]
2243   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
2244   "@
2245    pmulld\t{%2, %0|%0, %2}
2246    pmulld\t{%2, %0|%0, %2}
2247    vpmulld\t{%2, %1, %0|%0, %1, %2}"
2248   [(set_attr "isa" "noavx,noavx,avx")
2249    (set_attr "type" "sseimul")
2250    (set_attr "prefix_extra" "1")
2251    (set_attr "prefix" "orig,orig,vex")
2252    (set_attr "btver2_decode" "vector")
2253    (set_attr "mode" "TI")])
2254
2255 (define_expand "mmx_mulv4hi3"
2256   [(set (match_operand:V4HI 0 "register_operand")
2257         (mult:V4HI (match_operand:V4HI 1 "register_mmxmem_operand")
2258                    (match_operand:V4HI 2 "register_mmxmem_operand")))]
2259   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2260   "ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);")
2261
2262 (define_expand "mulv4hi3"
2263   [(set (match_operand:V4HI 0 "register_operand")
2264         (mult:V4HI (match_operand:V4HI 1 "register_operand")
2265                    (match_operand:V4HI 2 "register_operand")))]
2266   "TARGET_MMX_WITH_SSE")
2267
2268 (define_insn "*mmx_mulv4hi3"
2269   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
2270         (mult:V4HI (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw")
2271                    (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")))]
2272   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2273    && ix86_binary_operator_ok (MULT, V4HImode, operands)"
2274   "@
2275    pmullw\t{%2, %0|%0, %2}
2276    pmullw\t{%2, %0|%0, %2}
2277    vpmullw\t{%2, %1, %0|%0, %1, %2}"
2278   [(set_attr "isa" "*,sse2_noavx,avx")
2279    (set_attr "mmx_isa" "native,*,*")
2280    (set_attr "type" "mmxmul,ssemul,ssemul")
2281    (set_attr "mode" "DI,TI,TI")])
2282
2283 (define_insn "mulv2hi3"
2284   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
2285         (mult:V2HI (match_operand:V2HI 1 "register_operand" "%0,Yw")
2286                    (match_operand:V2HI 2 "register_operand" "x,Yw")))]
2287   "TARGET_SSE2"
2288   "@
2289    pmullw\t{%2, %0|%0, %2}
2290    vpmullw\t{%2, %1, %0|%0, %1, %2}"
2291   [(set_attr "isa" "noavx,avx")
2292    (set_attr "type" "ssemul")
2293    (set_attr "mode" "TI")])
2294
2295 (define_expand "mulv8qi3"
2296   [(set (match_operand:V8QI 0 "register_operand")
2297         (mult:V8QI (match_operand:V8QI 1 "register_operand")
2298                    (match_operand:V8QI 2 "register_operand")))]
2299   "TARGET_MMX_WITH_SSE"
2300 {
2301   ix86_expand_vecop_qihi_partial (MULT, operands[0], operands[1], operands[2]);
2302   DONE;
2303 })
2304
2305 (define_expand "mulv4qi3"
2306   [(set (match_operand:V4QI 0 "register_operand")
2307         (mult:V4QI (match_operand:V4QI 1 "register_operand")
2308                    (match_operand:V4QI 2 "register_operand")))]
2309   "TARGET_SSE2"
2310 {
2311   ix86_expand_vecop_qihi_partial (MULT, operands[0], operands[1], operands[2]);
2312   DONE;
2313 })
2314
2315 (define_expand "mmx_smulv4hi3_highpart"
2316   [(set (match_operand:V4HI 0 "register_operand")
2317         (truncate:V4HI
2318           (lshiftrt:V4SI
2319             (mult:V4SI
2320               (sign_extend:V4SI
2321                 (match_operand:V4HI 1 "register_mmxmem_operand"))
2322               (sign_extend:V4SI
2323                 (match_operand:V4HI 2 "register_mmxmem_operand")))
2324             (const_int 16))))]
2325   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2326   "ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);")
2327
2328 (define_insn "*mmx_smulv4hi3_highpart"
2329   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
2330         (truncate:V4HI
2331           (lshiftrt:V4SI
2332             (mult:V4SI
2333               (sign_extend:V4SI
2334                 (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw"))
2335               (sign_extend:V4SI
2336                 (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")))
2337             (const_int 16))))]
2338   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2339    && ix86_binary_operator_ok (MULT, V4HImode, operands)"
2340   "@
2341    pmulhw\t{%2, %0|%0, %2}
2342    pmulhw\t{%2, %0|%0, %2}
2343    vpmulhw\t{%2, %1, %0|%0, %1, %2}"
2344   [(set_attr "isa" "*,sse2_noavx,avx")
2345    (set_attr "mmx_isa" "native,*,*")
2346    (set_attr "type" "mmxmul,ssemul,ssemul")
2347    (set_attr "mode" "DI,TI,TI")])
2348
2349 (define_expand "mmx_umulv4hi3_highpart"
2350   [(set (match_operand:V4HI 0 "register_operand")
2351         (truncate:V4HI
2352           (lshiftrt:V4SI
2353             (mult:V4SI
2354               (zero_extend:V4SI
2355                 (match_operand:V4HI 1 "register_mmxmem_operand"))
2356               (zero_extend:V4SI
2357                 (match_operand:V4HI 2 "register_mmxmem_operand")))
2358             (const_int 16))))]
2359   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2360    && (TARGET_SSE || TARGET_3DNOW_A)"
2361   "ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);")
2362
2363 (define_insn "*mmx_umulv4hi3_highpart"
2364   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
2365         (truncate:V4HI
2366           (lshiftrt:V4SI
2367             (mult:V4SI
2368               (zero_extend:V4SI
2369                 (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw"))
2370               (zero_extend:V4SI
2371                 (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")))
2372           (const_int 16))))]
2373   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2374    && (TARGET_SSE || TARGET_3DNOW_A)
2375    && ix86_binary_operator_ok (MULT, V4HImode, operands)"
2376   "@
2377    pmulhuw\t{%2, %0|%0, %2}
2378    pmulhuw\t{%2, %0|%0, %2}
2379    vpmulhuw\t{%2, %1, %0|%0, %1, %2}"
2380   [(set_attr "isa" "*,sse2_noavx,avx")
2381    (set_attr "mmx_isa" "native,*,*")
2382    (set_attr "type" "mmxmul,ssemul,ssemul")
2383    (set_attr "mode" "DI,TI,TI")])
2384
2385 (define_expand "<s>mulv4hi3_highpart"
2386   [(set (match_operand:V4HI 0 "register_operand")
2387         (truncate:V4HI
2388           (lshiftrt:V4SI
2389             (mult:V4SI
2390               (any_extend:V4SI
2391                 (match_operand:V4HI 1 "register_operand"))
2392               (any_extend:V4SI
2393                 (match_operand:V4HI 2 "register_operand")))
2394             (const_int 16))))]
2395   "TARGET_MMX_WITH_SSE")
2396
2397 (define_insn "<s>mulv2hi3_highpart"
2398   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
2399         (truncate:V2HI
2400           (lshiftrt:V2SI
2401             (mult:V2SI
2402               (any_extend:V2SI
2403                 (match_operand:V2HI 1 "register_operand" "%0,Yw"))
2404               (any_extend:V2SI
2405                 (match_operand:V2HI 2 "register_operand" "x,Yw")))
2406             (const_int 16))))]
2407   "TARGET_SSE2"
2408   "@
2409    pmulh<u>w\t{%2, %0|%0, %2}
2410    vpmulh<u>w\t{%2, %1, %0|%0, %1, %2}"
2411   [(set_attr "isa" "noavx,avx")
2412    (set_attr "type" "ssemul")
2413    (set_attr "mode" "TI")])
2414
2415 (define_expand "mmx_pmaddwd"
2416   [(set (match_operand:V2SI 0 "register_operand")
2417         (plus:V2SI
2418           (mult:V2SI
2419             (sign_extend:V2SI
2420               (vec_select:V2HI
2421                 (match_operand:V4HI 1 "register_mmxmem_operand")
2422                 (parallel [(const_int 0) (const_int 2)])))
2423             (sign_extend:V2SI
2424               (vec_select:V2HI
2425                 (match_operand:V4HI 2 "register_mmxmem_operand")
2426                 (parallel [(const_int 0) (const_int 2)]))))
2427           (mult:V2SI
2428             (sign_extend:V2SI
2429               (vec_select:V2HI (match_dup 1)
2430                 (parallel [(const_int 1) (const_int 3)])))
2431             (sign_extend:V2SI
2432               (vec_select:V2HI (match_dup 2)
2433                 (parallel [(const_int 1) (const_int 3)]))))))]
2434   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2435   "ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);")
2436
2437 (define_insn "*mmx_pmaddwd"
2438   [(set (match_operand:V2SI 0 "register_operand" "=y,x,Yw")
2439         (plus:V2SI
2440           (mult:V2SI
2441             (sign_extend:V2SI
2442               (vec_select:V2HI
2443                 (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw")
2444                 (parallel [(const_int 0) (const_int 2)])))
2445             (sign_extend:V2SI
2446               (vec_select:V2HI
2447                 (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")
2448                 (parallel [(const_int 0) (const_int 2)]))))
2449           (mult:V2SI
2450             (sign_extend:V2SI
2451               (vec_select:V2HI (match_dup 1)
2452                 (parallel [(const_int 1) (const_int 3)])))
2453             (sign_extend:V2SI
2454               (vec_select:V2HI (match_dup 2)
2455                 (parallel [(const_int 1) (const_int 3)]))))))]
2456   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2457    && ix86_binary_operator_ok (MULT, V4HImode, operands)"
2458   "@
2459    pmaddwd\t{%2, %0|%0, %2}
2460    pmaddwd\t{%2, %0|%0, %2}
2461    vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
2462   [(set_attr "isa" "*,sse2_noavx,avx")
2463    (set_attr "mmx_isa" "native,*,*")
2464    (set_attr "type" "mmxmul,sseiadd,sseiadd")
2465    (set_attr "mode" "DI,TI,TI")])
2466
2467 (define_expand "mmx_pmulhrwv4hi3"
2468   [(set (match_operand:V4HI 0 "register_operand")
2469         (truncate:V4HI
2470           (lshiftrt:V4SI
2471             (plus:V4SI
2472               (mult:V4SI
2473                 (sign_extend:V4SI
2474                   (match_operand:V4HI 1 "nonimmediate_operand"))
2475                 (sign_extend:V4SI
2476                   (match_operand:V4HI 2 "nonimmediate_operand")))
2477               (const_vector:V4SI [(const_int 32768) (const_int 32768)
2478                                   (const_int 32768) (const_int 32768)]))
2479             (const_int 16))))]
2480   "TARGET_3DNOW"
2481   "ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);")
2482
2483 (define_insn "*mmx_pmulhrwv4hi3"
2484   [(set (match_operand:V4HI 0 "register_operand" "=y")
2485         (truncate:V4HI
2486           (lshiftrt:V4SI
2487             (plus:V4SI
2488               (mult:V4SI
2489                 (sign_extend:V4SI
2490                   (match_operand:V4HI 1 "nonimmediate_operand" "%0"))
2491                 (sign_extend:V4SI
2492                   (match_operand:V4HI 2 "nonimmediate_operand" "ym")))
2493               (const_vector:V4SI [(const_int 32768) (const_int 32768)
2494                                   (const_int 32768) (const_int 32768)]))
2495             (const_int 16))))]
2496   "TARGET_3DNOW && ix86_binary_operator_ok (MULT, V4HImode, operands)"
2497   "pmulhrw\t{%2, %0|%0, %2}"
2498   [(set_attr "type" "mmxmul")
2499    (set_attr "prefix_extra" "1")
2500    (set_attr "mode" "DI")])
2501
2502 (define_expand "sse2_umulv1siv1di3"
2503   [(set (match_operand:V1DI 0 "register_operand")
2504         (mult:V1DI
2505           (zero_extend:V1DI
2506             (vec_select:V1SI
2507               (match_operand:V2SI 1 "register_mmxmem_operand")
2508               (parallel [(const_int 0)])))
2509           (zero_extend:V1DI
2510             (vec_select:V1SI
2511               (match_operand:V2SI 2 "register_mmxmem_operand")
2512               (parallel [(const_int 0)])))))]
2513   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE2"
2514   "ix86_fixup_binary_operands_no_copy (MULT, V2SImode, operands);")
2515
2516 (define_insn "*sse2_umulv1siv1di3"
2517   [(set (match_operand:V1DI 0 "register_operand" "=y,x,Yv")
2518         (mult:V1DI
2519           (zero_extend:V1DI
2520             (vec_select:V1SI
2521               (match_operand:V2SI 1 "register_mmxmem_operand" "%0,0,Yv")
2522               (parallel [(const_int 0)])))
2523           (zero_extend:V1DI
2524             (vec_select:V1SI
2525               (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yv")
2526               (parallel [(const_int 0)])))))]
2527   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2528    && TARGET_SSE2
2529    && ix86_binary_operator_ok (MULT, V2SImode, operands)"
2530   "@
2531    pmuludq\t{%2, %0|%0, %2}
2532    pmuludq\t{%2, %0|%0, %2}
2533    vpmuludq\t{%2, %1, %0|%0, %1, %2}"
2534   [(set_attr "isa" "*,sse2_noavx,avx")
2535    (set_attr "mmx_isa" "native,*,*")
2536    (set_attr "type" "mmxmul,ssemul,ssemul")
2537    (set_attr "mode" "DI,TI,TI")])
2538
2539 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2540 ;;
2541 ;; Parallel integral shifts
2542 ;;
2543 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2544
2545 (define_insn "<code><mode>3"
2546   [(set (match_operand:MMXMODE14 0 "register_operand" "=Yr,*x,Yv")
2547         (smaxmin:MMXMODE14
2548           (match_operand:MMXMODE14 1 "register_operand" "%0,0,Yv")
2549           (match_operand:MMXMODE14 2 "register_operand" "Yr,*x,Yv")))]
2550   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
2551   "@
2552    p<maxmin_int><mmxvecsize>\t{%2, %0|%0, %2}
2553    p<maxmin_int><mmxvecsize>\t{%2, %0|%0, %2}
2554    vp<maxmin_int><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2555   [(set_attr "isa" "noavx,noavx,avx")
2556    (set_attr "type" "sseiadd")
2557    (set_attr "prefix_extra" "1")
2558    (set_attr "prefix" "orig,orig,vex")
2559    (set_attr "mode" "TI")])
2560
2561 (define_expand "mmx_<code>v4hi3"
2562   [(set (match_operand:V4HI 0 "register_operand")
2563         (smaxmin:V4HI
2564           (match_operand:V4HI 1 "register_mmxmem_operand")
2565           (match_operand:V4HI 2 "register_mmxmem_operand")))]
2566   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2567    && (TARGET_SSE || TARGET_3DNOW_A)"
2568   "ix86_fixup_binary_operands_no_copy (<CODE>, V4HImode, operands);")
2569
2570 (define_insn "*mmx_<code>v4hi3"
2571   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
2572         (smaxmin:V4HI
2573           (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw")
2574           (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")))]
2575   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2576    && (TARGET_SSE || TARGET_3DNOW_A)
2577    && ix86_binary_operator_ok (<CODE>, V4HImode, operands)"
2578   "@
2579    p<maxmin_int>w\t{%2, %0|%0, %2}
2580    p<maxmin_int>w\t{%2, %0|%0, %2}
2581    vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
2582   [(set_attr "isa" "*,sse2_noavx,avx")
2583    (set_attr "mmx_isa" "native,*,*")
2584    (set_attr "type" "mmxadd,sseiadd,sseiadd")
2585    (set_attr "mode" "DI,TI,TI")])
2586
2587 (define_expand "<code>v4hi3"
2588   [(set (match_operand:V4HI 0 "register_operand")
2589         (smaxmin:V4HI
2590           (match_operand:V4HI 1 "register_operand")
2591           (match_operand:V4HI 2 "register_operand")))]
2592   "TARGET_MMX_WITH_SSE")
2593
2594 (define_insn "<code><mode>3"
2595   [(set (match_operand:VI1_16_32 0 "register_operand" "=Yr,*x,Yv")
2596         (smaxmin:VI1_16_32
2597           (match_operand:VI1_16_32 1 "register_operand" "%0,0,Yv")
2598           (match_operand:VI1_16_32 2 "register_operand" "Yr,*x,Yv")))]
2599   "TARGET_SSE4_1"
2600   "@
2601    p<maxmin_int>b\t{%2, %0|%0, %2}
2602    p<maxmin_int>b\t{%2, %0|%0, %2}
2603    vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
2604   [(set_attr "isa" "noavx,noavx,avx")
2605    (set_attr "type" "sseiadd")
2606    (set_attr "prefix_extra" "1")
2607    (set_attr "prefix" "orig,orig,vex")
2608    (set_attr "mode" "TI")])
2609
2610 (define_insn "<code>v2hi3"
2611   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
2612         (smaxmin:V2HI
2613           (match_operand:V2HI 1 "register_operand" "%0,Yw")
2614           (match_operand:V2HI 2 "register_operand" "x,Yw")))]
2615   "TARGET_SSE2"
2616   "@
2617    p<maxmin_int>w\t{%2, %0|%0, %2}
2618    vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
2619   [(set_attr "isa" "noavx,avx")
2620    (set_attr "type" "sseiadd")
2621    (set_attr "mode" "TI")])
2622
2623 (define_insn "<code><mode>3"
2624   [(set (match_operand:MMXMODE24 0 "register_operand" "=Yr,*x,Yv")
2625         (umaxmin:MMXMODE24
2626           (match_operand:MMXMODE24 1 "register_operand" "%0,0,Yv")
2627           (match_operand:MMXMODE24 2 "register_operand" "Yr,*x,Yv")))]
2628   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
2629   "@
2630    p<maxmin_int><mmxvecsize>\t{%2, %0|%0, %2}
2631    p<maxmin_int><mmxvecsize>\t{%2, %0|%0, %2}
2632    vp<maxmin_int><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2633   [(set_attr "isa" "noavx,noavx,avx")
2634    (set_attr "type" "sseiadd")
2635    (set_attr "prefix_extra" "1")
2636    (set_attr "prefix" "orig,orig,vex")
2637    (set_attr "mode" "TI")])
2638
2639 (define_expand "mmx_<code>v8qi3"
2640   [(set (match_operand:V8QI 0 "register_operand")
2641         (umaxmin:V8QI
2642           (match_operand:V8QI 1 "register_mmxmem_operand")
2643           (match_operand:V8QI 2 "register_mmxmem_operand")))]
2644   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2645    && (TARGET_SSE || TARGET_3DNOW_A)"
2646   "ix86_fixup_binary_operands_no_copy (<CODE>, V8QImode, operands);")
2647
2648 (define_insn "*mmx_<code>v8qi3"
2649   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
2650         (umaxmin:V8QI
2651           (match_operand:V8QI 1 "register_mmxmem_operand" "%0,0,Yw")
2652           (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yw")))]
2653   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2654    && (TARGET_SSE || TARGET_3DNOW_A)
2655    && ix86_binary_operator_ok (<CODE>, V8QImode, operands)"
2656   "@
2657    p<maxmin_int>b\t{%2, %0|%0, %2}
2658    p<maxmin_int>b\t{%2, %0|%0, %2}
2659    vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
2660   [(set_attr "isa" "*,sse2_noavx,avx")
2661    (set_attr "mmx_isa" "native,*,*")
2662    (set_attr "type" "mmxadd,sseiadd,sseiadd")
2663    (set_attr "mode" "DI,TI,TI")])
2664
2665 (define_expand "<code>v8qi3"
2666   [(set (match_operand:V8QI 0 "register_operand")
2667         (umaxmin:V8QI
2668           (match_operand:V8QI 1 "register_operand")
2669           (match_operand:V8QI 2 "register_operand")))]
2670   "TARGET_MMX_WITH_SSE")
2671
2672 (define_insn "<code><mode>3"
2673   [(set (match_operand:VI1_16_32 0 "register_operand" "=x,Yw")
2674         (umaxmin:VI1_16_32
2675           (match_operand:VI1_16_32 1 "register_operand" "%0,Yw")
2676           (match_operand:VI1_16_32 2 "register_operand" "x,Yw")))]
2677   "TARGET_SSE2"
2678   "@
2679    p<maxmin_int>b\t{%2, %0|%0, %2}
2680    vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
2681   [(set_attr "isa" "noavx,avx")
2682    (set_attr "type" "sseiadd")
2683    (set_attr "mode" "TI")])
2684
2685 (define_insn "<code>v2hi3"
2686   [(set (match_operand:V2HI 0 "register_operand" "=Yr,*x,Yv")
2687         (umaxmin:V2HI
2688           (match_operand:V2HI 1 "register_operand" "%0,0,Yv")
2689           (match_operand:V2HI 2 "register_operand" "Yr,*x,Yv")))]
2690   "TARGET_SSE4_1"
2691   "@
2692    p<maxmin_int>w\t{%2, %0|%0, %2}
2693    p<maxmin_int>w\t{%2, %0|%0, %2}
2694    vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
2695   [(set_attr "isa" "noavx,noavx,avx")
2696    (set_attr "type" "sseiadd")
2697    (set_attr "prefix_extra" "1")
2698    (set_attr "prefix" "orig,orig,vex")
2699    (set_attr "mode" "TI")])
2700
2701 (define_insn "ssse3_abs<mode>2"
2702   [(set (match_operand:MMXMODEI 0 "register_operand" "=y,Yv")
2703         (abs:MMXMODEI
2704           (match_operand:MMXMODEI 1 "register_mmxmem_operand" "ym,Yv")))]
2705   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
2706   "@
2707    pabs<mmxvecsize>\t{%1, %0|%0, %1}
2708    %vpabs<mmxvecsize>\t{%1, %0|%0, %1}"
2709   [(set_attr "mmx_isa" "native,*")
2710    (set_attr "type" "sselog1")
2711    (set_attr "prefix_rep" "0")
2712    (set_attr "prefix_extra" "1")
2713    (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
2714    (set_attr "mode" "DI,TI")])
2715
2716 (define_expand "abs<mode>2"
2717   [(set (match_operand:MMXMODEI 0 "register_operand")
2718         (abs:MMXMODEI
2719           (match_operand:MMXMODEI 1 "register_operand")))]
2720   "TARGET_SSSE3 && TARGET_MMX_WITH_SSE")
2721
2722 (define_insn "abs<mode>2"
2723   [(set (match_operand:VI_16_32 0 "register_operand" "=Yv")
2724         (abs:VI_16_32
2725           (match_operand:VI_16_32 1 "register_operand" "Yv")))]
2726   "TARGET_SSSE3"
2727   "%vpabs<mmxvecsize>\t{%1, %0|%0, %1}"
2728   [(set_attr "type" "sselog1")
2729    (set_attr "prefix_rep" "0")
2730    (set_attr "prefix_extra" "1")
2731    (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
2732    (set_attr "mode" "TI")])
2733
2734 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2735 ;;
2736 ;; Parallel integral shifts
2737 ;;
2738 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2739
2740 (define_insn "mmx_ashr<mode>3"
2741   [(set (match_operand:MMXMODE24 0 "register_operand" "=y,x,<Yv_Yw>")
2742         (ashiftrt:MMXMODE24
2743           (match_operand:MMXMODE24 1 "register_operand" "0,0,<Yv_Yw>")
2744           (match_operand:DI 2 "nonmemory_operand" "yN,xN,<Yv_Yw>N")))]
2745   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2746   "@
2747    psra<mmxvecsize>\t{%2, %0|%0, %2}
2748    psra<mmxvecsize>\t{%2, %0|%0, %2}
2749    vpsra<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2750   [(set_attr "isa" "*,sse2_noavx,avx")
2751    (set_attr "mmx_isa" "native,*,*")
2752    (set_attr "type" "mmxshft,sseishft,sseishft")
2753    (set (attr "length_immediate")
2754      (if_then_else (match_operand 2 "const_int_operand")
2755        (const_string "1")
2756        (const_string "0")))
2757    (set_attr "mode" "DI,TI,TI")])
2758
2759 (define_expand "ashr<mode>3"
2760   [(set (match_operand:MMXMODE24 0 "register_operand")
2761         (ashiftrt:MMXMODE24
2762           (match_operand:MMXMODE24 1 "register_operand")
2763           (match_operand:DI 2 "nonmemory_operand")))]
2764   "TARGET_MMX_WITH_SSE")
2765
2766 (define_insn "mmx_<insn><mode>3"
2767   [(set (match_operand:MMXMODE248 0 "register_operand" "=y,x,<Yv_Yw>")
2768         (any_lshift:MMXMODE248
2769           (match_operand:MMXMODE248 1 "register_operand" "0,0,<Yv_Yw>")
2770           (match_operand:DI 2 "nonmemory_operand" "yN,xN,<Yv_Yw>N")))]
2771   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2772   "@
2773    p<vshift><mmxvecsize>\t{%2, %0|%0, %2}
2774    p<vshift><mmxvecsize>\t{%2, %0|%0, %2}
2775    vp<vshift><mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2776   [(set_attr "isa" "*,sse2_noavx,avx")
2777    (set_attr "mmx_isa" "native,*,*")
2778    (set_attr "type" "mmxshft,sseishft,sseishft")
2779    (set (attr "length_immediate")
2780      (if_then_else (match_operand 2 "const_int_operand")
2781        (const_string "1")
2782        (const_string "0")))
2783    (set_attr "mode" "DI,TI,TI")])
2784
2785 (define_expand "<insn><mode>3"
2786   [(set (match_operand:MMXMODE24 0 "register_operand")
2787         (any_lshift:MMXMODE24
2788           (match_operand:MMXMODE24 1 "register_operand")
2789           (match_operand:DI 2 "nonmemory_operand")))]
2790   "TARGET_MMX_WITH_SSE")
2791
2792 (define_insn "mmx_<insn>v1si3"
2793   [(set (match_operand:V1SI 0 "register_operand" "=x,Yw")
2794         (any_lshift:V1SI
2795           (match_operand:V1SI 1 "register_operand" "0,Yw")
2796           (match_operand:DI 2 "nonmemory_operand" "xN,YwN")))]
2797   "TARGET_SSE2"
2798   "@
2799    p<vshift>d\t{%2, %0|%0, %2}
2800    vp<vshift>d\t{%2, %1, %0|%0, %1, %2}"
2801   [(set_attr "isa" "noavx,avx")
2802    (set_attr "type" "sseishft")
2803    (set (attr "length_immediate")
2804      (if_then_else (match_operand 2 "const_int_operand")
2805        (const_string "1")
2806        (const_string "0")))
2807    (set_attr "mode" "TI")])
2808
2809 (define_insn "<insn>v2hi3"
2810   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
2811         (any_shift:V2HI
2812           (match_operand:V2HI 1 "register_operand" "0,Yw")
2813           (match_operand:DI 2 "nonmemory_operand" "xN,YwN")))]
2814   "TARGET_SSE2"
2815   "@
2816    p<vshift>w\t{%2, %0|%0, %2}
2817    vp<vshift>w\t{%2, %1, %0|%0, %1, %2}"
2818   [(set_attr "isa" "noavx,avx")
2819    (set_attr "type" "sseishft")
2820    (set (attr "length_immediate")
2821      (if_then_else (match_operand 2 "const_int_operand")
2822        (const_string "1")
2823        (const_string "0")))
2824    (set_attr "mode" "TI")])
2825
2826 (define_expand "<insn>v8qi3"
2827   [(set (match_operand:V8QI 0 "register_operand")
2828         (any_shift:V8QI (match_operand:V8QI 1 "register_operand")
2829                         (match_operand:DI 2 "nonmemory_operand")))]
2830   "TARGET_MMX_WITH_SSE"
2831 {
2832   ix86_expand_vecop_qihi_partial (<CODE>, operands[0],
2833                                   operands[1], operands[2]);
2834   DONE;
2835 })
2836
2837 (define_expand "<insn>v4qi3"
2838   [(set (match_operand:V4QI 0 "register_operand")
2839         (any_shift:V4QI (match_operand:V4QI 1 "register_operand")
2840                         (match_operand:DI 2 "nonmemory_operand")))]
2841   "TARGET_SSE2"
2842 {
2843   ix86_expand_vecop_qihi_partial (<CODE>, operands[0],
2844                                   operands[1], operands[2]);
2845   DONE;
2846 })
2847
2848 (define_insn_and_split "<insn>v2qi3"
2849   [(set (match_operand:V2QI 0 "register_operand" "=Q")
2850         (any_shift:V2QI
2851           (match_operand:V2QI 1 "register_operand" "0")
2852           (match_operand:QI 2 "nonmemory_operand" "cI")))
2853    (clobber (reg:CC FLAGS_REG))]
2854   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
2855   "#"
2856   "&& reload_completed"
2857   [(parallel
2858      [(set (zero_extract:HI (match_dup 3) (const_int 8) (const_int 8))
2859            (subreg:HI
2860              (any_shift:QI
2861                (subreg:QI
2862                  (zero_extract:HI (match_dup 4)
2863                                   (const_int 8)
2864                                   (const_int 8)) 0)
2865                (match_dup 2)) 0))
2866       (clobber (reg:CC FLAGS_REG))])
2867    (parallel
2868      [(set (strict_low_part (match_dup 0))
2869            (any_shift:QI (match_dup 1) (match_dup 2)))
2870       (clobber (reg:CC FLAGS_REG))])]
2871 {
2872   operands[4] = lowpart_subreg (HImode, operands[1], V2QImode);
2873   operands[3] = lowpart_subreg (HImode, operands[0], V2QImode);
2874   operands[1] = lowpart_subreg (QImode, operands[1], V2QImode);
2875   operands[0] = lowpart_subreg (QImode, operands[0], V2QImode);
2876 }
2877   [(set_attr "type" "multi")
2878    (set_attr "mode" "QI")])
2879
2880 (define_expand "v<insn>v8qi3"
2881   [(set (match_operand:V8QI 0 "register_operand")
2882         (any_shift:V8QI
2883           (match_operand:V8QI 1 "register_operand")
2884           (match_operand:V8QI 2 "register_operand")))]
2885   "TARGET_AVX512BW && TARGET_AVX512VL && TARGET_MMX_WITH_SSE"
2886 {
2887   ix86_expand_vecop_qihi_partial (<CODE>, operands[0],
2888                                   operands[1], operands[2]);
2889   DONE;
2890 })
2891
2892 (define_expand "v<insn>v4qi3"
2893   [(set (match_operand:V4QI 0 "register_operand")
2894         (any_shift:V4QI
2895           (match_operand:V4QI 1 "register_operand")
2896           (match_operand:V4QI 2 "register_operand")))]
2897   "TARGET_AVX512BW && TARGET_AVX512VL"
2898 {
2899   ix86_expand_vecop_qihi_partial (<CODE>, operands[0],
2900                                   operands[1], operands[2]);
2901   DONE;
2902 })
2903
2904 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2905 ;;
2906 ;; Parallel integral comparisons
2907 ;;
2908 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2909
2910 (define_expand "mmx_eq<mode>3"
2911   [(set (match_operand:MMXMODEI 0 "register_operand")
2912         (eq:MMXMODEI
2913           (match_operand:MMXMODEI 1 "register_mmxmem_operand")
2914           (match_operand:MMXMODEI 2 "register_mmxmem_operand")))]
2915   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2916   "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
2917
2918 (define_insn "*mmx_eq<mode>3"
2919   [(set (match_operand:MMXMODEI 0 "register_operand" "=y,x,x")
2920         (eq:MMXMODEI
2921           (match_operand:MMXMODEI 1 "register_mmxmem_operand" "%0,0,x")
2922           (match_operand:MMXMODEI 2 "register_mmxmem_operand" "ym,x,x")))]
2923   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
2924    && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
2925   "@
2926    pcmpeq<mmxvecsize>\t{%2, %0|%0, %2}
2927    pcmpeq<mmxvecsize>\t{%2, %0|%0, %2}
2928    vpcmpeq<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2929   [(set_attr "isa" "*,sse2_noavx,avx")
2930    (set_attr "mmx_isa" "native,*,*")
2931    (set_attr "type" "mmxcmp,ssecmp,ssecmp")
2932    (set_attr "mode" "DI,TI,TI")])
2933
2934 (define_insn "*eq<mode>3"
2935   [(set (match_operand:VI_16_32 0 "register_operand" "=x,x")
2936         (eq:VI_16_32
2937           (match_operand:VI_16_32 1 "register_operand" "%0,x")
2938           (match_operand:VI_16_32 2 "register_operand" "x,x")))]
2939   "TARGET_SSE2"
2940   "@
2941    pcmpeq<mmxvecsize>\t{%2, %0|%0, %2}
2942    vpcmpeq<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2943   [(set_attr "isa" "noavx,avx")
2944    (set_attr "type" "ssecmp")
2945    (set_attr "mode" "TI")])
2946
2947 (define_insn "mmx_gt<mode>3"
2948   [(set (match_operand:MMXMODEI 0 "register_operand" "=y,x,x")
2949         (gt:MMXMODEI
2950           (match_operand:MMXMODEI 1 "register_operand" "0,0,x")
2951           (match_operand:MMXMODEI 2 "register_mmxmem_operand" "ym,x,x")))]
2952   "TARGET_MMX || TARGET_MMX_WITH_SSE"
2953   "@
2954    pcmpgt<mmxvecsize>\t{%2, %0|%0, %2}
2955    pcmpgt<mmxvecsize>\t{%2, %0|%0, %2}
2956    vpcmpgt<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2957   [(set_attr "isa" "*,sse2_noavx,avx")
2958    (set_attr "mmx_isa" "native,*,*")
2959    (set_attr "type" "mmxcmp,ssecmp,ssecmp")
2960    (set_attr "mode" "DI,TI,TI")])
2961
2962 (define_insn "*gt<mode>3"
2963   [(set (match_operand:VI_16_32 0 "register_operand" "=x,x")
2964         (gt:VI_16_32
2965           (match_operand:VI_16_32 1 "register_operand" "0,x")
2966           (match_operand:VI_16_32 2 "register_operand" "x,x")))]
2967   "TARGET_SSE2"
2968   "@
2969    pcmpgt<mmxvecsize>\t{%2, %0|%0, %2}
2970    vpcmpgt<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
2971   [(set_attr "isa" "noavx,avx")
2972    (set_attr "type" "ssecmp")
2973    (set_attr "mode" "TI")])
2974
2975 (define_insn "*xop_maskcmp<mode>3"
2976   [(set (match_operand:MMXMODEI 0 "register_operand" "=x")
2977         (match_operator:MMXMODEI 1 "ix86_comparison_int_operator"
2978          [(match_operand:MMXMODEI 2 "register_operand" "x")
2979           (match_operand:MMXMODEI 3 "register_operand" "x")]))]
2980   "TARGET_XOP"
2981   "vpcom%Y1<mmxvecsize>\t{%3, %2, %0|%0, %2, %3}"
2982   [(set_attr "type" "sse4arg")
2983    (set_attr "mode" "TI")])
2984
2985 (define_insn "*xop_maskcmp<mode>3"
2986   [(set (match_operand:VI_16_32 0 "register_operand" "=x")
2987         (match_operator:VI_16_32 1 "ix86_comparison_int_operator"
2988          [(match_operand:VI_16_32 2 "register_operand" "x")
2989           (match_operand:VI_16_32 3 "register_operand" "x")]))]
2990   "TARGET_XOP"
2991   "vpcom%Y1<mmxvecsize>\t{%3, %2, %0|%0, %2, %3}"
2992   [(set_attr "type" "sse4arg")
2993    (set_attr "mode" "TI")])
2994
2995 (define_insn "*xop_maskcmp_uns<mode>3"
2996   [(set (match_operand:MMXMODEI 0 "register_operand" "=x")
2997         (match_operator:MMXMODEI 1 "ix86_comparison_uns_operator"
2998          [(match_operand:MMXMODEI 2 "register_operand" "x")
2999           (match_operand:MMXMODEI 3 "register_operand" "x")]))]
3000   "TARGET_XOP"
3001   "vpcom%Y1u<mmxvecsize>\t{%3, %2, %0|%0, %2, %3}"
3002   [(set_attr "type" "sse4arg")
3003    (set_attr "mode" "TI")])
3004
3005 (define_insn "*xop_maskcmp_uns<mode>3"
3006   [(set (match_operand:VI_16_32 0 "register_operand" "=x")
3007         (match_operator:VI_16_32 1 "ix86_comparison_uns_operator"
3008          [(match_operand:VI_16_32 2 "register_operand" "x")
3009           (match_operand:VI_16_32 3 "register_operand" "x")]))]
3010   "TARGET_XOP"
3011   "vpcom%Y1u<mmxvecsize>\t{%3, %2, %0|%0, %2, %3}"
3012   [(set_attr "type" "sse4arg")
3013    (set_attr "mode" "TI")])
3014
3015 (define_expand "vec_cmp<mode><mode>"
3016   [(set (match_operand:MMXMODEI 0 "register_operand")
3017         (match_operator:MMXMODEI 1 ""
3018           [(match_operand:MMXMODEI 2 "register_operand")
3019            (match_operand:MMXMODEI 3 "register_operand")]))]
3020   "TARGET_MMX_WITH_SSE"
3021 {
3022   bool ok = ix86_expand_int_vec_cmp (operands);
3023   gcc_assert (ok);
3024   DONE;
3025 })
3026
3027 (define_expand "vec_cmp<mode><mode>"
3028   [(set (match_operand:VI_16_32 0 "register_operand")
3029         (match_operator:VI_16_32 1 ""
3030           [(match_operand:VI_16_32 2 "register_operand")
3031            (match_operand:VI_16_32 3 "register_operand")]))]
3032   "TARGET_SSE2"
3033 {
3034   bool ok = ix86_expand_int_vec_cmp (operands);
3035   gcc_assert (ok);
3036   DONE;
3037 })
3038
3039 (define_expand "vec_cmpu<mode><mode>"
3040   [(set (match_operand:MMXMODEI 0 "register_operand")
3041         (match_operator:MMXMODEI 1 ""
3042           [(match_operand:MMXMODEI 2 "register_operand")
3043            (match_operand:MMXMODEI 3 "register_operand")]))]
3044   "TARGET_MMX_WITH_SSE"
3045 {
3046   bool ok = ix86_expand_int_vec_cmp (operands);
3047   gcc_assert (ok);
3048   DONE;
3049 })
3050
3051 (define_expand "vec_cmpu<mode><mode>"
3052   [(set (match_operand:VI_16_32 0 "register_operand")
3053         (match_operator:VI_16_32 1 ""
3054           [(match_operand:VI_16_32 2 "register_operand")
3055            (match_operand:VI_16_32 3 "register_operand")]))]
3056   "TARGET_SSE2"
3057 {
3058   bool ok = ix86_expand_int_vec_cmp (operands);
3059   gcc_assert (ok);
3060   DONE;
3061 })
3062
3063 (define_expand "vcond<MMXMODE124:mode><MMXMODEI:mode>"
3064   [(set (match_operand:MMXMODE124 0 "register_operand")
3065         (if_then_else:MMXMODE124
3066           (match_operator 3 ""
3067             [(match_operand:MMXMODEI 4 "register_operand")
3068              (match_operand:MMXMODEI 5 "register_operand")])
3069           (match_operand:MMXMODE124 1)
3070           (match_operand:MMXMODE124 2)))]
3071   "TARGET_MMX_WITH_SSE
3072    && (GET_MODE_NUNITS (<MMXMODE124:MODE>mode)
3073        == GET_MODE_NUNITS (<MMXMODEI:MODE>mode))"
3074 {
3075   bool ok = ix86_expand_int_vcond (operands);
3076   gcc_assert (ok);
3077   DONE;
3078 })
3079
3080 (define_expand "vcond<mode><mode>"
3081   [(set (match_operand:VI_16_32 0 "register_operand")
3082         (if_then_else:VI_16_32
3083           (match_operator 3 ""
3084             [(match_operand:VI_16_32 4 "register_operand")
3085              (match_operand:VI_16_32 5 "register_operand")])
3086           (match_operand:VI_16_32 1)
3087           (match_operand:VI_16_32 2)))]
3088   "TARGET_SSE2"
3089 {
3090   bool ok = ix86_expand_int_vcond (operands);
3091   gcc_assert (ok);
3092   DONE;
3093 })
3094
3095 (define_expand "vcondu<MMXMODE124:mode><MMXMODEI:mode>"
3096   [(set (match_operand:MMXMODE124 0 "register_operand")
3097         (if_then_else:MMXMODE124
3098           (match_operator 3 ""
3099             [(match_operand:MMXMODEI 4 "register_operand")
3100              (match_operand:MMXMODEI 5 "register_operand")])
3101           (match_operand:MMXMODE124 1)
3102           (match_operand:MMXMODE124 2)))]
3103   "TARGET_MMX_WITH_SSE
3104    && (GET_MODE_NUNITS (<MMXMODE124:MODE>mode)
3105        == GET_MODE_NUNITS (<MMXMODEI:MODE>mode))"
3106 {
3107   bool ok = ix86_expand_int_vcond (operands);
3108   gcc_assert (ok);
3109   DONE;
3110 })
3111
3112 (define_expand "vcondu<mode><mode>"
3113   [(set (match_operand:VI_16_32 0 "register_operand")
3114         (if_then_else:VI_16_32
3115           (match_operator 3 ""
3116             [(match_operand:VI_16_32 4 "register_operand")
3117              (match_operand:VI_16_32 5 "register_operand")])
3118           (match_operand:VI_16_32 1)
3119           (match_operand:VI_16_32 2)))]
3120   "TARGET_SSE2"
3121 {
3122   bool ok = ix86_expand_int_vcond (operands);
3123   gcc_assert (ok);
3124   DONE;
3125 })
3126
3127 (define_expand "vcond_mask_<mode><mmxintvecmodelower>"
3128   [(set (match_operand:MMXMODE124 0 "register_operand")
3129         (vec_merge:MMXMODE124
3130           (match_operand:MMXMODE124 1 "register_operand")
3131           (match_operand:MMXMODE124 2 "register_operand")
3132           (match_operand:<mmxintvecmode> 3 "register_operand")))]
3133   "TARGET_MMX_WITH_SSE"
3134 {
3135   ix86_expand_sse_movcc (operands[0], operands[3],
3136                          operands[1], operands[2]);
3137   DONE;
3138 })
3139
3140 (define_expand "vcond_mask_<mode><mode>"
3141   [(set (match_operand:VI_16_32 0 "register_operand")
3142         (vec_merge:VI_16_32
3143           (match_operand:VI_16_32 1 "register_operand")
3144           (match_operand:VI_16_32 2 "register_operand")
3145           (match_operand:VI_16_32 3 "register_operand")))]
3146   "TARGET_SSE2"
3147 {
3148   ix86_expand_sse_movcc (operands[0], operands[3],
3149                          operands[1], operands[2]);
3150   DONE;
3151 })
3152
3153 (define_insn "mmx_pblendvb_v8qi"
3154   [(set (match_operand:V8QI 0 "register_operand" "=Yr,*x,x")
3155         (unspec:V8QI
3156           [(match_operand:V8QI 1 "register_operand" "0,0,x")
3157            (match_operand:V8QI 2 "register_operand" "Yr,*x,x")
3158            (match_operand:V8QI 3 "register_operand" "Yz,Yz,x")]
3159           UNSPEC_BLENDV))]
3160   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3161   "@
3162    pblendvb\t{%3, %2, %0|%0, %2, %3}
3163    pblendvb\t{%3, %2, %0|%0, %2, %3}
3164    vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3165   [(set_attr "isa" "noavx,noavx,avx")
3166    (set_attr "type" "ssemov")
3167    (set_attr "prefix_extra" "1")
3168    (set_attr "length_immediate" "1")
3169    (set_attr "prefix" "orig,orig,vex")
3170    (set_attr "btver2_decode" "vector")
3171    (set_attr "mode" "TI")])
3172
3173 (define_insn "mmx_pblendvb_<mode>"
3174   [(set (match_operand:VI_16_32 0 "register_operand" "=Yr,*x,x")
3175         (unspec:VI_16_32
3176           [(match_operand:VI_16_32 1 "register_operand" "0,0,x")
3177            (match_operand:VI_16_32 2 "register_operand" "Yr,*x,x")
3178            (match_operand:VI_16_32 3 "register_operand" "Yz,Yz,x")]
3179           UNSPEC_BLENDV))]
3180   "TARGET_SSE4_1"
3181   "@
3182    pblendvb\t{%3, %2, %0|%0, %2, %3}
3183    pblendvb\t{%3, %2, %0|%0, %2, %3}
3184    vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3185   [(set_attr "isa" "noavx,noavx,avx")
3186    (set_attr "type" "ssemov")
3187    (set_attr "prefix_extra" "1")
3188    (set_attr "length_immediate" "1")
3189    (set_attr "prefix" "orig,orig,vex")
3190    (set_attr "btver2_decode" "vector")
3191    (set_attr "mode" "TI")])
3192
3193 ;; XOP parallel XMM conditional moves
3194 (define_insn "*xop_pcmov_<mode>"
3195   [(set (match_operand:MMXMODE124 0 "register_operand" "=x")
3196         (if_then_else:MMXMODE124
3197           (match_operand:MMXMODE124 3 "register_operand" "x")
3198           (match_operand:MMXMODE124 1 "register_operand" "x")
3199           (match_operand:MMXMODE124 2 "register_operand" "x")))]
3200   "TARGET_XOP && TARGET_MMX_WITH_SSE"
3201   "vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3202   [(set_attr "type" "sse4arg")
3203    (set_attr "mode" "TI")])
3204
3205 (define_insn "*xop_pcmov_<mode>"
3206   [(set (match_operand:VI_16_32 0 "register_operand" "=x")
3207         (if_then_else:VI_16_32
3208           (match_operand:VI_16_32 3 "register_operand" "x")
3209           (match_operand:VI_16_32 1 "register_operand" "x")
3210           (match_operand:VI_16_32 2 "register_operand" "x")))]
3211   "TARGET_XOP"
3212   "vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3213   [(set_attr "type" "sse4arg")
3214    (set_attr "mode" "TI")])
3215
3216 ;; XOP permute instructions
3217 (define_insn "mmx_ppermv64"
3218   [(set (match_operand:V8QI 0 "register_operand" "=x")
3219         (unspec:V8QI
3220           [(match_operand:V8QI 1 "register_operand" "x")
3221            (match_operand:V8QI 2 "register_operand" "x")
3222            (match_operand:V16QI 3 "nonimmediate_operand" "xm")]
3223           UNSPEC_XOP_PERMUTE))]
3224   "TARGET_XOP && TARGET_MMX_WITH_SSE"
3225   "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3226   [(set_attr "type" "sse4arg")
3227    (set_attr "mode" "TI")])
3228
3229 (define_insn "mmx_ppermv32"
3230   [(set (match_operand:V4QI 0 "register_operand" "=x")
3231         (unspec:V4QI
3232           [(match_operand:V4QI 1 "register_operand" "x")
3233            (match_operand:V4QI 2 "register_operand" "x")
3234            (match_operand:V16QI 3 "nonimmediate_operand" "xm")]
3235           UNSPEC_XOP_PERMUTE))]
3236   "TARGET_XOP"
3237   "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3238   [(set_attr "type" "sse4arg")
3239    (set_attr "mode" "TI")])
3240
3241 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3242 ;;
3243 ;; Parallel integral logical operations
3244 ;;
3245 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3246
3247 (define_expand "one_cmpl<mode>2"
3248   [(set (match_operand:MMXMODEI 0 "register_operand")
3249         (xor:MMXMODEI
3250           (match_operand:MMXMODEI 1 "register_operand")
3251           (match_dup 2)))]
3252   "TARGET_MMX_WITH_SSE"
3253   "operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));")
3254
3255 (define_insn "one_cmpl<mode>2"
3256   [(set (match_operand:VI_16_32 0 "register_operand" "=?r,&x,&v")
3257         (not:VI_16_32
3258           (match_operand:VI_16_32 1 "register_operand" "0,x,v")))]
3259   ""
3260   "#"
3261   [(set_attr "isa" "*,sse2,avx512vl")
3262    (set_attr "type" "negnot,sselog1,sselog1")
3263    (set_attr "mode" "SI,TI,TI")])
3264
3265 (define_split
3266   [(set (match_operand:VI_16_32 0 "general_reg_operand")
3267         (not:VI_16_32
3268           (match_operand:VI_16_32 1 "general_reg_operand")))]
3269   "reload_completed"
3270   [(set (match_dup 0)
3271         (not:SI (match_dup 1)))]
3272 {
3273   operands[1] = lowpart_subreg (SImode, operands[1], <MODE>mode);
3274   operands[0] = lowpart_subreg (SImode, operands[0], <MODE>mode);
3275 })
3276
3277 (define_split
3278   [(set (match_operand:VI_16_32 0 "sse_reg_operand")
3279         (not:VI_16_32
3280           (match_operand:VI_16_32 1 "sse_reg_operand")))]
3281   "TARGET_SSE2 && reload_completed"
3282   [(set (match_dup 0) (match_dup 2))
3283    (set (match_dup 0)
3284         (xor:V16QI
3285           (match_dup 0) (match_dup 1)))]
3286 {
3287   operands[2] = CONSTM1_RTX (V16QImode);
3288   operands[1] = lowpart_subreg (V16QImode, operands[1], <MODE>mode);
3289   operands[0] = lowpart_subreg (V16QImode, operands[0], <MODE>mode);
3290 })
3291
3292 (define_insn "mmx_andnot<mode>3"
3293   [(set (match_operand:MMXMODEI 0 "register_operand" "=y,x,x,v")
3294         (and:MMXMODEI
3295           (not:MMXMODEI (match_operand:MMXMODEI 1 "register_operand" "0,0,x,v"))
3296           (match_operand:MMXMODEI 2 "register_mmxmem_operand" "ym,x,x,v")))]
3297   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3298   "@
3299    pandn\t{%2, %0|%0, %2}
3300    pandn\t{%2, %0|%0, %2}
3301    vpandn\t{%2, %1, %0|%0, %1, %2}
3302    vpandnd\t{%2, %1, %0|%0, %1, %2}"
3303   [(set_attr "isa" "*,sse2_noavx,avx,avx512vl")
3304    (set_attr "mmx_isa" "native,*,*,*")
3305    (set_attr "type" "mmxadd,sselog,sselog,sselog")
3306    (set_attr "mode" "DI,TI,TI,TI")])
3307
3308 (define_insn "*andnot<mode>3"
3309   [(set (match_operand:VI_16_32 0 "register_operand" "=?&r,?r,x,x,v")
3310         (and:VI_16_32
3311           (not:VI_16_32
3312             (match_operand:VI_16_32 1 "register_operand" "0,r,0,x,v"))
3313           (match_operand:VI_16_32 2 "register_operand" "r,r,x,x,v")))
3314    (clobber (reg:CC FLAGS_REG))]
3315   ""
3316   "#"
3317   [(set_attr "isa" "*,bmi,sse2_noavx,avx,avx512vl")
3318    (set_attr "type" "alu,bitmanip,sselog,sselog,sselog")
3319    (set_attr "mode" "SI,SI,TI,TI,TI")])
3320
3321 (define_split
3322   [(set (match_operand:VI_16_32 0 "general_reg_operand")
3323         (and:VI_16_32
3324           (not:VI_16_32 (match_operand:VI_16_32 1 "general_reg_operand"))
3325           (match_operand:VI_16_32 2 "general_reg_operand")))
3326    (clobber (reg:CC FLAGS_REG))]
3327   "TARGET_BMI && reload_completed"
3328   [(parallel
3329      [(set (match_dup 0)
3330            (and:SI (not:SI (match_dup 1)) (match_dup 2)))
3331       (clobber (reg:CC FLAGS_REG))])]
3332 {
3333   operands[2] = lowpart_subreg (SImode, operands[2], <MODE>mode);
3334   operands[1] = lowpart_subreg (SImode, operands[1], <MODE>mode);
3335   operands[0] = lowpart_subreg (SImode, operands[0], <MODE>mode);
3336 })
3337
3338 (define_split
3339   [(set (match_operand:VI_16_32 0 "general_reg_operand")
3340         (and:VI_16_32
3341           (not:VI_16_32 (match_operand:VI_16_32 1 "general_reg_operand"))
3342           (match_operand:VI_16_32 2 "general_reg_operand")))
3343    (clobber (reg:CC FLAGS_REG))]
3344   "!TARGET_BMI && reload_completed"
3345   [(set (match_dup 0)
3346         (not:SI (match_dup 1)))
3347    (parallel
3348      [(set (match_dup 0)
3349            (and:SI (match_dup 0) (match_dup 2)))
3350       (clobber (reg:CC FLAGS_REG))])]
3351 {
3352   operands[2] = lowpart_subreg (SImode, operands[2], <MODE>mode);
3353   operands[1] = lowpart_subreg (SImode, operands[1], <MODE>mode);
3354   operands[0] = lowpart_subreg (SImode, operands[0], <MODE>mode);
3355 })
3356
3357 (define_split
3358   [(set (match_operand:VI_16_32 0 "sse_reg_operand")
3359         (and:VI_16_32
3360           (not:VI_16_32 (match_operand:VI_16_32 1 "sse_reg_operand"))
3361           (match_operand:VI_16_32 2 "sse_reg_operand")))
3362    (clobber (reg:CC FLAGS_REG))]
3363   "TARGET_SSE2 && reload_completed"
3364   [(set (match_dup 0)
3365         (and:V16QI (not:V16QI (match_dup 1)) (match_dup 2)))]
3366 {
3367   operands[2] = lowpart_subreg (V16QImode, operands[2], <MODE>mode);
3368   operands[1] = lowpart_subreg (V16QImode, operands[1], <MODE>mode);
3369   operands[0] = lowpart_subreg (V16QImode, operands[0], <MODE>mode);
3370 })
3371
3372 (define_expand "mmx_<code><mode>3"
3373   [(set (match_operand:MMXMODEI 0 "register_operand")
3374         (any_logic:MMXMODEI
3375           (match_operand:MMXMODEI 1 "register_mmxmem_operand")
3376           (match_operand:MMXMODEI 2 "register_mmxmem_operand")))]
3377   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3378   "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
3379
3380 (define_expand "<code><mode>3"
3381   [(set (match_operand:MMXMODEI 0 "register_operand")
3382         (any_logic:MMXMODEI
3383           (match_operand:MMXMODEI 1 "register_operand")
3384           (match_operand:MMXMODEI 2 "register_operand")))]
3385   "TARGET_MMX_WITH_SSE")
3386
3387 (define_insn "*mmx_<code><mode>3"
3388   [(set (match_operand:MMXMODEI 0 "register_operand" "=y,x,x,v")
3389         (any_logic:MMXMODEI
3390           (match_operand:MMXMODEI 1 "register_mmxmem_operand" "%0,0,x,v")
3391           (match_operand:MMXMODEI 2 "register_mmxmem_operand" "ym,x,x,v")))]
3392   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
3393    && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
3394   "@
3395    p<logic>\t{%2, %0|%0, %2}
3396    p<logic>\t{%2, %0|%0, %2}
3397    vp<logic>\t{%2, %1, %0|%0, %1, %2}
3398    vp<logic>d\t{%2, %1, %0|%0, %1, %2}"
3399   [(set_attr "isa" "*,sse2_noavx,avx,avx512vl")
3400    (set_attr "mmx_isa" "native,*,*,*")
3401    (set_attr "type" "mmxadd,sselog,sselog,sselog")
3402    (set_attr "mode" "DI,TI,TI,TI")])
3403
3404 (define_expand "<code><mode>3"
3405   [(set (match_operand:VI_16_32 0 "nonimmediate_operand")
3406         (any_logic:VI_16_32
3407           (match_operand:VI_16_32 1 "nonimmediate_operand")
3408           (match_operand:VI_16_32 2 "nonimmediate_or_x86_64_const_vector_operand")))]
3409   ""
3410   "ix86_expand_binary_operator (<CODE>, <MODE>mode, operands); DONE;")
3411
3412 (define_insn "*<code><mode>3"
3413   [(set (match_operand:VI_16_32 0 "nonimmediate_operand" "=?r,m,x,x,v")
3414         (any_logic:VI_16_32
3415           (match_operand:VI_16_32 1 "nonimmediate_operand" "%0,0,0,x,v")
3416           (match_operand:VI_16_32 2 "nonimmediate_or_x86_64_const_vector_operand" "r,i,x,x,v")))
3417    (clobber (reg:CC FLAGS_REG))]
3418   "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
3419   "#"
3420   [(set_attr "isa" "*,*,sse2_noavx,avx,avx512vl")
3421    (set_attr "type" "alu,alu,sselog,sselog,sselog")
3422    (set_attr "mode" "SI,SI,TI,TI,TI")])
3423
3424 (define_split
3425   [(set (match_operand:VI_16_32 0 "nonimmediate_gr_operand")
3426         (any_logic:VI_16_32
3427           (match_operand:VI_16_32 1 "nonimmediate_gr_operand")
3428           (match_operand:VI_16_32 2 "reg_or_const_vector_operand")))
3429    (clobber (reg:CC FLAGS_REG))]
3430   "reload_completed"
3431   [(parallel
3432      [(set (match_dup 0)
3433            (any_logic:<mmxinsnmode> (match_dup 1) (match_dup 2)))
3434       (clobber (reg:CC FLAGS_REG))])]
3435 {
3436   if (!register_operand (operands[2], <MODE>mode))
3437     {
3438       HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (operands[2],
3439                                                                 <MODE>mode);
3440       operands[2] = GEN_INT (val);
3441     }
3442   else
3443     operands[2] = lowpart_subreg (<mmxinsnmode>mode, operands[2], <MODE>mode);
3444   operands[1] = lowpart_subreg (<mmxinsnmode>mode, operands[1], <MODE>mode);
3445   operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
3446 })
3447
3448 (define_split
3449   [(set (match_operand:VI_16_32 0 "sse_reg_operand")
3450         (any_logic:VI_16_32
3451           (match_operand:VI_16_32 1 "sse_reg_operand")
3452           (match_operand:VI_16_32 2 "sse_reg_operand")))
3453    (clobber (reg:CC FLAGS_REG))]
3454   "TARGET_SSE2 && reload_completed"
3455   [(set (match_dup 0)
3456         (any_logic:V16QI (match_dup 1) (match_dup 2)))]
3457 {
3458   operands[2] = lowpart_subreg (V16QImode, operands[2], <MODE>mode);
3459   operands[1] = lowpart_subreg (V16QImode, operands[1], <MODE>mode);
3460   operands[0] = lowpart_subreg (V16QImode, operands[0], <MODE>mode);
3461 })
3462
3463 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3464 ;;
3465 ;; Parallel integral element swizzling
3466 ;;
3467 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3468
3469 (define_insn_and_split "mmx_packsswb"
3470   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
3471         (vec_concat:V8QI
3472           (ss_truncate:V4QI
3473             (match_operand:V4HI 1 "register_operand" "0,0,Yw"))
3474           (ss_truncate:V4QI
3475             (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw"))))]
3476   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3477   "@
3478    packsswb\t{%2, %0|%0, %2}
3479    #
3480    #"
3481   "&& reload_completed
3482    && SSE_REGNO_P (REGNO (operands[0]))"
3483   [(const_int 0)]
3484   "ix86_split_mmx_pack (operands, SS_TRUNCATE); DONE;"
3485   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3486    (set_attr "type" "mmxshft,sselog,sselog")
3487    (set_attr "mode" "DI,TI,TI")])
3488
3489 ;; This instruction does unsigned saturation of signed source
3490 ;; and is different from generic us_truncate RTX.
3491 (define_insn_and_split "mmx_packuswb"
3492   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
3493         (unspec:V8QI
3494           [(match_operand:V4HI 1 "register_operand" "0,0,Yw")
3495            (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")]
3496           UNSPEC_US_TRUNCATE))]
3497   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3498   "@
3499    packuswb\t{%2, %0|%0, %2}
3500    #
3501    #"
3502   "&& reload_completed
3503    && SSE_REGNO_P (REGNO (operands[0]))"
3504   [(const_int 0)]
3505   "ix86_split_mmx_pack (operands, US_TRUNCATE); DONE;"
3506   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3507    (set_attr "type" "mmxshft,sselog,sselog")
3508    (set_attr "mode" "DI,TI,TI")])
3509
3510 (define_insn_and_split "mmx_packssdw"
3511   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
3512         (vec_concat:V4HI
3513           (ss_truncate:V2HI
3514             (match_operand:V2SI 1 "register_operand" "0,0,Yw"))
3515           (ss_truncate:V2HI
3516             (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yw"))))]
3517   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3518   "@
3519    packssdw\t{%2, %0|%0, %2}
3520    #
3521    #"
3522   "&& reload_completed
3523    && SSE_REGNO_P (REGNO (operands[0]))"
3524   [(const_int 0)]
3525   "ix86_split_mmx_pack (operands, SS_TRUNCATE); DONE;"
3526   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3527    (set_attr "type" "mmxshft,sselog,sselog")
3528    (set_attr "mode" "DI,TI,TI")])
3529
3530 (define_insn_and_split "mmx_packusdw"
3531   [(set (match_operand:V4HI 0 "register_operand" "=Yr,*x,Yw")
3532         (unspec:V4HI
3533           [(match_operand:V2SI 1 "register_operand" "0,0,Yw")
3534            (match_operand:V2SI 2 "register_operand" "Yr,*x,Yw")]
3535            UNSPEC_US_TRUNCATE))]
3536   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3537   "#"
3538   "&& reload_completed"
3539   [(const_int 0)]
3540   "ix86_split_mmx_pack (operands, US_TRUNCATE); DONE;"
3541   [(set_attr "isa" "noavx,noavx,avx")
3542    (set_attr "type" "sselog")
3543    (set_attr "mode" "TI")])
3544
3545 (define_insn_and_split "mmx_punpckhbw"
3546   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
3547         (vec_select:V8QI
3548           (vec_concat:V16QI
3549             (match_operand:V8QI 1 "register_operand" "0,0,Yw")
3550             (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yw"))
3551           (parallel [(const_int 4) (const_int 12)
3552                      (const_int 5) (const_int 13)
3553                      (const_int 6) (const_int 14)
3554                      (const_int 7) (const_int 15)])))]
3555   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3556   "@
3557    punpckhbw\t{%2, %0|%0, %2}
3558    #
3559    #"
3560   "&& reload_completed
3561    && SSE_REGNO_P (REGNO (operands[0]))"
3562   [(const_int 0)]
3563   "ix86_split_mmx_punpck (operands, true); DONE;"
3564   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3565    (set_attr "type" "mmxcvt,sselog,sselog")
3566    (set_attr "mode" "DI,TI,TI")])
3567
3568 (define_insn_and_split "mmx_punpckhbw_low"
3569   [(set (match_operand:V4QI 0 "register_operand" "=x,Yw")
3570         (vec_select:V4QI
3571           (vec_concat:V8QI
3572             (match_operand:V4QI 1 "register_operand" "0,Yw")
3573             (match_operand:V4QI 2 "register_operand" "x,Yw"))
3574           (parallel [(const_int 2) (const_int 6)
3575                      (const_int 3) (const_int 7)])))]
3576   "TARGET_SSE2"
3577   "#"
3578   "&& reload_completed"
3579   [(const_int 0)]
3580   "ix86_split_mmx_punpck (operands, true); DONE;"
3581   [(set_attr "isa" "noavx,avx")
3582    (set_attr "type" "sselog")
3583    (set_attr "mode" "TI")])
3584
3585 (define_insn_and_split "mmx_punpcklbw"
3586   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
3587         (vec_select:V8QI
3588           (vec_concat:V16QI
3589             (match_operand:V8QI 1 "register_operand" "0,0,Yw")
3590             (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yw"))
3591           (parallel [(const_int 0) (const_int 8)
3592                      (const_int 1) (const_int 9)
3593                      (const_int 2) (const_int 10)
3594                      (const_int 3) (const_int 11)])))]
3595   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3596   "@
3597    punpcklbw\t{%2, %0|%0, %k2}
3598    #
3599    #"
3600   "&& reload_completed
3601    && SSE_REGNO_P (REGNO (operands[0]))"
3602   [(const_int 0)]
3603   "ix86_split_mmx_punpck (operands, false); DONE;"
3604   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3605    (set_attr "type" "mmxcvt,sselog,sselog")
3606    (set_attr "mode" "DI,TI,TI")])
3607
3608 (define_insn_and_split "mmx_punpcklbw_low"
3609   [(set (match_operand:V4QI 0 "register_operand" "=x,Yw")
3610         (vec_select:V4QI
3611           (vec_concat:V8QI
3612             (match_operand:V4QI 1 "register_operand" "0,Yw")
3613             (match_operand:V4QI 2 "register_operand" "x,Yw"))
3614           (parallel [(const_int 0) (const_int 4)
3615                      (const_int 1) (const_int 5)])))]
3616   "TARGET_SSE2"
3617   "#"
3618   "&& reload_completed"
3619   [(const_int 0)]
3620   "ix86_split_mmx_punpck (operands, false); DONE;"
3621   [(set_attr "isa" "noavx,avx")
3622    (set_attr "type" "sselog")
3623    (set_attr "mode" "TI")])
3624
3625 (define_insn_and_split "mmx_punpckhwd"
3626   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
3627         (vec_select:V4HI
3628           (vec_concat:V8HI
3629             (match_operand:V4HI 1 "register_operand" "0,0,Yw")
3630             (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw"))
3631           (parallel [(const_int 2) (const_int 6)
3632                      (const_int 3) (const_int 7)])))]
3633   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3634   "@
3635    punpckhwd\t{%2, %0|%0, %2}
3636    #
3637    #"
3638   "&& reload_completed
3639    && SSE_REGNO_P (REGNO (operands[0]))"
3640   [(const_int 0)]
3641   "ix86_split_mmx_punpck (operands, true); DONE;"
3642   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3643    (set_attr "type" "mmxcvt,sselog,sselog")
3644    (set_attr "mode" "DI,TI,TI")])
3645
3646 (define_insn_and_split "mmx_punpcklwd"
3647   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
3648         (vec_select:V4HI
3649           (vec_concat:V8HI
3650             (match_operand:V4HI 1 "register_operand" "0,0,Yw")
3651             (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw"))
3652           (parallel [(const_int 0) (const_int 4)
3653                      (const_int 1) (const_int 5)])))]
3654   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3655   "@
3656    punpcklwd\t{%2, %0|%0, %k2}
3657    #
3658    #"
3659   "&& reload_completed
3660    && SSE_REGNO_P (REGNO (operands[0]))"
3661   [(const_int 0)]
3662   "ix86_split_mmx_punpck (operands, false); DONE;"
3663   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3664    (set_attr "type" "mmxcvt,sselog,sselog")
3665    (set_attr "mode" "DI,TI,TI")])
3666
3667 (define_insn_and_split "mmx_punpckhdq"
3668   [(set (match_operand:V2SI 0 "register_operand" "=y,x,Yv")
3669         (vec_select:V2SI
3670           (vec_concat:V4SI
3671             (match_operand:V2SI 1 "register_operand" "0,0,Yv")
3672             (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yv"))
3673           (parallel [(const_int 1)
3674                      (const_int 3)])))]
3675   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3676   "@
3677    punpckhdq\t{%2, %0|%0, %2}
3678    #
3679    #"
3680   "&& reload_completed
3681    && SSE_REGNO_P (REGNO (operands[0]))"
3682   [(const_int 0)]
3683   "ix86_split_mmx_punpck (operands, true); DONE;"
3684   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3685    (set_attr "type" "mmxcvt,sselog,sselog")
3686    (set_attr "mode" "DI,TI,TI")])
3687
3688 (define_insn_and_split "mmx_punpckldq"
3689   [(set (match_operand:V2SI 0 "register_operand" "=y,x,Yv")
3690         (vec_select:V2SI
3691           (vec_concat:V4SI
3692             (match_operand:V2SI 1 "register_operand" "0,0,Yv")
3693             (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yv"))
3694           (parallel [(const_int 0)
3695                      (const_int 2)])))]
3696   "TARGET_MMX || TARGET_MMX_WITH_SSE"
3697   "@
3698    punpckldq\t{%2, %0|%0, %k2}
3699    #
3700    #"
3701   "&& reload_completed
3702    && SSE_REGNO_P (REGNO (operands[0]))"
3703   [(const_int 0)]
3704   "ix86_split_mmx_punpck (operands, false); DONE;"
3705   [(set_attr "mmx_isa" "native,sse_noavx,avx")
3706    (set_attr "type" "mmxcvt,sselog,sselog")
3707    (set_attr "mode" "DI,TI,TI")])
3708
3709 (define_insn "sse4_1_<code>v4qiv4hi2"
3710   [(set (match_operand:V4HI 0 "register_operand" "=Yr,*x,Yw")
3711         (any_extend:V4HI
3712           (vec_select:V4QI
3713             (match_operand:V8QI 1 "register_operand" "Yr,*x,Yw")
3714             (parallel [(const_int 0) (const_int 1)
3715                        (const_int 2) (const_int 3)]))))]
3716   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3717   "%vpmov<extsuffix>bw\t{%1, %0|%0, %1}"
3718   [(set_attr "isa" "noavx,noavx,avx")
3719    (set_attr "type" "ssemov")
3720    (set_attr "prefix_extra" "1")
3721    (set_attr "prefix" "orig,orig,maybe_evex")
3722    (set_attr "mode" "TI")])
3723
3724 (define_expand "<insn>v4qiv4hi2"
3725   [(set (match_operand:V4HI 0 "register_operand")
3726         (any_extend:V4HI
3727           (match_operand:V4QI 1 "register_operand")))]
3728   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3729 {
3730   rtx op1 = force_reg (V4QImode, operands[1]);
3731   op1 = lowpart_subreg (V8QImode, op1, V4QImode);
3732   emit_insn (gen_sse4_1_<code>v4qiv4hi2 (operands[0], op1));
3733   DONE;
3734 })
3735
3736 (define_insn "sse4_1_<code>v2hiv2si2"
3737   [(set (match_operand:V2SI 0 "register_operand" "=Yr,*x,v")
3738         (any_extend:V2SI
3739           (vec_select:V2HI
3740             (match_operand:V4HI 1 "register_operand" "Yr,*x,v")
3741             (parallel [(const_int 0) (const_int 1)]))))]
3742   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3743   "%vpmov<extsuffix>wd\t{%1, %0|%0, %1}"
3744   [(set_attr "isa" "noavx,noavx,avx")
3745    (set_attr "type" "ssemov")
3746    (set_attr "prefix_extra" "1")
3747    (set_attr "prefix" "orig,orig,maybe_evex")
3748    (set_attr "mode" "TI")])
3749
3750 (define_expand "<insn>v2hiv2si2"
3751   [(set (match_operand:V2SI 0 "register_operand")
3752         (any_extend:V2SI
3753           (match_operand:V2HI 1 "register_operand")))]
3754   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3755 {
3756   rtx op1 = force_reg (V2HImode, operands[1]);
3757   op1 = lowpart_subreg (V4HImode, op1, V2HImode);
3758   emit_insn (gen_sse4_1_<code>v2hiv2si2 (operands[0], op1));
3759   DONE;
3760 })
3761
3762 (define_insn "sse4_1_<code>v2qiv2si2"
3763   [(set (match_operand:V2SI 0 "register_operand" "=Yr,*x,v")
3764         (any_extend:V2SI
3765           (vec_select:V2QI
3766             (match_operand:V4QI 1 "register_operand" "Yr,*x,v")
3767             (parallel [(const_int 0) (const_int 1)]))))]
3768   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3769   "%vpmov<extsuffix>bd\t{%1, %0|%0, %1}"
3770   [(set_attr "isa" "noavx,noavx,avx")
3771    (set_attr "type" "ssemov")
3772    (set_attr "prefix_extra" "1")
3773    (set_attr "prefix" "orig,orig,maybe_evex")
3774    (set_attr "mode" "TI")])
3775
3776 (define_expand "<insn>v2qiv2si2"
3777   [(set (match_operand:V2SI 0 "register_operand")
3778         (any_extend:V2SI
3779           (match_operand:V2QI 1 "register_operand")))]
3780   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
3781 {
3782   rtx op1 = force_reg (V2QImode, operands[1]);
3783   op1 = lowpart_subreg (V4QImode, op1, V2QImode);
3784   emit_insn (gen_sse4_1_<code>v2qiv2si2 (operands[0], op1));
3785   DONE;
3786 })
3787
3788 (define_insn "sse4_1_<code>v2qiv2hi2"
3789   [(set (match_operand:V2HI 0 "register_operand" "=Yr,*x,Yw")
3790         (any_extend:V2HI
3791           (vec_select:V2QI
3792             (match_operand:V4QI 1 "register_operand" "Yr,*x,Yw")
3793             (parallel [(const_int 0) (const_int 1)]))))]
3794   "TARGET_SSE4_1"
3795   "%vpmov<extsuffix>bw\t{%1, %0|%0, %1}"
3796   [(set_attr "isa" "noavx,noavx,avx")
3797    (set_attr "type" "ssemov")
3798    (set_attr "prefix_extra" "1")
3799    (set_attr "prefix" "orig,orig,maybe_evex")
3800    (set_attr "mode" "TI")])
3801
3802 (define_expand "<insn>v2qiv2hi2"
3803   [(set (match_operand:V2HI 0 "register_operand")
3804         (any_extend:V2HI
3805           (match_operand:V2QI 1 "register_operand")))]
3806   "TARGET_SSE4_1"
3807 {
3808   rtx op1 = force_reg (V2QImode, operands[1]);
3809   op1 = lowpart_subreg (V4QImode, op1, V2QImode);
3810   emit_insn (gen_sse4_1_<code>v2qiv2hi2 (operands[0], op1));
3811   DONE;
3812 })
3813
3814 (define_insn "truncv2hiv2qi2"
3815   [(set (match_operand:V2QI 0 "register_operand" "=v")
3816         (truncate:V2QI
3817           (match_operand:V2HI 1 "register_operand" "v")))]
3818   "TARGET_AVX512VL && TARGET_AVX512BW"
3819   "vpmovwb\t{%1, %0|%0, %1}"
3820   [(set_attr "type" "ssemov")
3821    (set_attr "prefix" "evex")
3822    (set_attr "mode" "TI")])
3823
3824 (define_mode_iterator V2QI_V2HI [V2QI V2HI])
3825 (define_insn "truncv2si<mode>2"
3826   [(set (match_operand:V2QI_V2HI 0 "register_operand" "=v")
3827         (truncate:V2QI_V2HI
3828           (match_operand:V2SI 1 "register_operand" "v")))]
3829   "TARGET_AVX512VL && TARGET_MMX_WITH_SSE"
3830   "vpmovd<mmxvecsize>\t{%1, %0|%0, %1}"
3831   [(set_attr "type" "ssemov")
3832    (set_attr "prefix" "evex")
3833    (set_attr "mode" "TI")])
3834
3835 ;; Pack/unpack vector modes
3836 (define_mode_attr mmxpackmode
3837   [(V4HI "V8QI") (V2SI "V4HI")])
3838
3839 (define_expand "vec_pack_trunc_<mode>"
3840   [(match_operand:<mmxpackmode> 0 "register_operand")
3841    (match_operand:MMXMODE24 1 "register_operand")
3842    (match_operand:MMXMODE24 2 "register_operand")]
3843   "TARGET_MMX_WITH_SSE"
3844 {
3845   rtx op1 = gen_lowpart (<mmxpackmode>mode, operands[1]);
3846   rtx op2 = gen_lowpart (<mmxpackmode>mode, operands[2]);
3847   ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
3848   DONE;
3849 })
3850
3851 (define_expand "vec_pack_trunc_v2hi"
3852   [(match_operand:V4QI 0 "register_operand")
3853    (match_operand:V2HI 1 "register_operand")
3854    (match_operand:V2HI 2 "register_operand")]
3855   "TARGET_SSE2"
3856 {
3857   rtx op1 = gen_lowpart (V4QImode, operands[1]);
3858   rtx op2 = gen_lowpart (V4QImode, operands[2]);
3859   ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
3860   DONE;
3861 })
3862
3863 (define_mode_attr mmxunpackmode
3864   [(V8QI "V4HI") (V4HI "V2SI")])
3865
3866 (define_expand "vec_unpacks_lo_<mode>"
3867   [(match_operand:<mmxunpackmode> 0 "register_operand")
3868    (match_operand:MMXMODE12 1 "register_operand")]
3869   "TARGET_MMX_WITH_SSE"
3870   "ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
3871
3872 (define_expand "vec_unpacks_hi_<mode>"
3873   [(match_operand:<mmxunpackmode> 0 "register_operand")
3874    (match_operand:MMXMODE12 1 "register_operand")]
3875   "TARGET_MMX_WITH_SSE"
3876   "ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
3877
3878 (define_expand "vec_unpacku_lo_<mode>"
3879   [(match_operand:<mmxunpackmode> 0 "register_operand")
3880    (match_operand:MMXMODE12 1 "register_operand")]
3881   "TARGET_MMX_WITH_SSE"
3882   "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
3883
3884 (define_expand "vec_unpacku_hi_<mode>"
3885   [(match_operand:<mmxunpackmode> 0 "register_operand")
3886    (match_operand:MMXMODE12 1 "register_operand")]
3887   "TARGET_MMX_WITH_SSE"
3888   "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
3889
3890 (define_expand "vec_unpacks_lo_v4qi"
3891   [(match_operand:V2HI 0 "register_operand")
3892    (match_operand:V4QI 1 "register_operand")]
3893   "TARGET_SSE2"
3894   "ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
3895
3896 (define_expand "vec_unpacks_hi_v4qi"
3897   [(match_operand:V2HI 0 "register_operand")
3898    (match_operand:V4QI 1 "register_operand")]
3899   "TARGET_SSE2"
3900   "ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
3901
3902 (define_expand "vec_unpacku_lo_v4qi"
3903   [(match_operand:V2HI 0 "register_operand")
3904    (match_operand:V4QI 1 "register_operand")]
3905   "TARGET_SSE2"
3906   "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
3907
3908 (define_expand "vec_unpacku_hi_v4qi"
3909   [(match_operand:V2HI 0 "register_operand")
3910    (match_operand:V4QI 1 "register_operand")]
3911   "TARGET_SSE2"
3912   "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
3913
3914 (define_insn "*mmx_pinsrd"
3915   [(set (match_operand:V2SI 0 "register_operand" "=x,Yv")
3916         (vec_merge:V2SI
3917           (vec_duplicate:V2SI
3918             (match_operand:SI 2 "nonimmediate_operand" "rm,rm"))
3919           (match_operand:V2SI 1 "register_operand" "0,Yv")
3920           (match_operand:SI 3 "const_int_operand")))]
3921   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE
3922    && ((unsigned) exact_log2 (INTVAL (operands[3]))
3923        < GET_MODE_NUNITS (V2SImode))"
3924 {
3925   operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
3926   switch (which_alternative)
3927     {
3928     case 1:
3929       return "vpinsrd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
3930     case 0:
3931       return "pinsrd\t{%3, %2, %0|%0, %2, %3}";
3932     default:
3933       gcc_unreachable ();
3934     }
3935 }
3936   [(set_attr "isa" "noavx,avx")
3937    (set_attr "prefix_extra" "1")
3938    (set_attr "type" "sselog")
3939    (set_attr "length_immediate" "1")
3940    (set_attr "prefix" "orig,vex")
3941    (set_attr "mode" "TI")])
3942
3943 (define_expand "mmx_pinsrw"
3944   [(set (match_operand:V4HI 0 "register_operand")
3945         (vec_merge:V4HI
3946           (vec_duplicate:V4HI
3947             (match_operand:SI 2 "nonimmediate_operand"))
3948           (match_operand:V4HI 1 "register_operand")
3949           (match_operand:SI 3 "const_0_to_3_operand")))]
3950   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
3951    && (TARGET_SSE || TARGET_3DNOW_A)"
3952 {
3953   operands[2] = gen_lowpart (HImode, operands[2]);
3954   operands[3] = GEN_INT (1 << INTVAL (operands[3]));
3955 })
3956
3957 (define_insn "*mmx_pinsrw"
3958   [(set (match_operand:V4HI 0 "register_operand" "=y,x,YW")
3959         (vec_merge:V4HI
3960           (vec_duplicate:V4HI
3961             (match_operand:HI 2 "nonimmediate_operand" "rm,rm,rm"))
3962           (match_operand:V4HI 1 "register_operand" "0,0,YW")
3963           (match_operand:SI 3 "const_int_operand")))]
3964   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
3965    && (TARGET_SSE || TARGET_3DNOW_A)
3966    && ((unsigned) exact_log2 (INTVAL (operands[3]))
3967        < GET_MODE_NUNITS (V4HImode))"
3968 {
3969   operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
3970   switch (which_alternative)
3971     {
3972     case 2:
3973       if (MEM_P (operands[2]))
3974         return "vpinsrw\t{%3, %2, %1, %0|%0, %1, %2, %3}";
3975       else
3976         return "vpinsrw\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
3977     case 1:
3978     case 0:
3979       if (MEM_P (operands[2]))
3980         return "pinsrw\t{%3, %2, %0|%0, %2, %3}";
3981       else
3982         return "pinsrw\t{%3, %k2, %0|%0, %k2, %3}";
3983     default:
3984       gcc_unreachable ();
3985     }
3986 }
3987   [(set_attr "isa" "*,sse2_noavx,avx")
3988    (set_attr "mmx_isa" "native,*,*")
3989    (set_attr "type" "mmxcvt,sselog,sselog")
3990    (set_attr "length_immediate" "1")
3991    (set_attr "mode" "DI,TI,TI")])
3992
3993 (define_insn "*mmx_pinsrb"
3994   [(set (match_operand:V8QI 0 "register_operand" "=x,YW")
3995         (vec_merge:V8QI
3996           (vec_duplicate:V8QI
3997             (match_operand:QI 2 "nonimmediate_operand" "rm,rm"))
3998           (match_operand:V8QI 1 "register_operand" "0,YW")
3999           (match_operand:SI 3 "const_int_operand")))]
4000   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE
4001    && ((unsigned) exact_log2 (INTVAL (operands[3]))
4002        < GET_MODE_NUNITS (V8QImode))"
4003 {
4004   operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
4005   switch (which_alternative)
4006     {
4007     case 1:
4008       if (MEM_P (operands[2]))
4009         return "vpinsrb\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4010       else
4011         return "vpinsrb\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
4012     case 0:
4013       if (MEM_P (operands[2]))
4014         return "pinsrb\t{%3, %2, %0|%0, %2, %3}";
4015       else
4016         return "pinsrb\t{%3, %k2, %0|%0, %k2, %3}";
4017     default:
4018       gcc_unreachable ();
4019     }
4020 }
4021   [(set_attr "isa" "noavx,avx")
4022    (set_attr "type" "sselog")
4023    (set_attr "prefix_extra" "1")
4024    (set_attr "length_immediate" "1")
4025    (set_attr "prefix" "orig,vex")
4026    (set_attr "mode" "TI")])
4027
4028 (define_insn "*mmx_pextrw"
4029   [(set (match_operand:HI 0 "register_sse4nonimm_operand" "=r,r,m")
4030         (vec_select:HI
4031           (match_operand:V4HI 1 "register_operand" "y,YW,YW")
4032           (parallel [(match_operand:SI 2 "const_0_to_3_operand")])))]
4033   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4034    && (TARGET_SSE || TARGET_3DNOW_A)"
4035   "@
4036    pextrw\t{%2, %1, %k0|%k0, %1, %2}
4037    %vpextrw\t{%2, %1, %k0|%k0, %1, %2}
4038    %vpextrw\t{%2, %1, %0|%0, %1, %2}"
4039   [(set_attr "isa" "*,sse2,sse4")
4040    (set_attr "mmx_isa" "native,*,*")
4041    (set_attr "type" "mmxcvt,sselog1,sselog1")
4042    (set_attr "length_immediate" "1")
4043    (set_attr "prefix" "orig,maybe_vex,maybe_vex")
4044    (set_attr "mode" "DI,TI,TI")])
4045
4046 (define_insn "*mmx_pextrw_zext"
4047   [(set (match_operand:SWI48 0 "register_operand" "=r,r")
4048         (zero_extend:SWI48
4049           (vec_select:HI
4050             (match_operand:V4HI 1 "register_operand" "y,YW")
4051             (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
4052   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4053    && (TARGET_SSE || TARGET_3DNOW_A)"
4054   "@
4055    pextrw\t{%2, %1, %k0|%k0, %1, %2}
4056    %vpextrw\t{%2, %1, %k0|%k0, %1, %2}"
4057   [(set_attr "isa" "*,sse2")
4058    (set_attr "mmx_isa" "native,*")
4059    (set_attr "type" "mmxcvt,sselog1")
4060    (set_attr "length_immediate" "1")
4061    (set_attr "prefix" "orig,maybe_vex")
4062    (set_attr "mode" "DI,TI")])
4063
4064 (define_insn "*mmx_pextrb"
4065   [(set (match_operand:QI 0 "nonimmediate_operand" "=r,m")
4066         (vec_select:QI
4067           (match_operand:V8QI 1 "register_operand" "YW,YW")
4068           (parallel [(match_operand:SI 2 "const_0_to_7_operand")])))]
4069   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
4070   "@
4071    %vpextrb\t{%2, %1, %k0|%k0, %1, %2}
4072    %vpextrb\t{%2, %1, %0|%0, %1, %2}"
4073   [(set_attr "type" "sselog1")
4074    (set_attr "prefix_extra" "1")
4075    (set_attr "length_immediate" "1")
4076    (set_attr "prefix" "maybe_vex")
4077    (set_attr "mode" "TI")])
4078
4079 (define_insn "*mmx_pextrb_zext"
4080   [(set (match_operand:SWI248 0 "register_operand" "=r")
4081         (zero_extend:SWI248
4082           (vec_select:QI
4083             (match_operand:V8QI 1 "register_operand" "YW")
4084             (parallel [(match_operand:SI 2 "const_0_to_7_operand")]))))]
4085   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
4086   "%vpextrb\t{%2, %1, %k0|%k0, %1, %2}"
4087   [(set_attr "type" "sselog1")
4088    (set_attr "prefix_extra" "1")
4089    (set_attr "length_immediate" "1")
4090    (set_attr "prefix" "maybe_vex")
4091    (set_attr "mode" "TI")])
4092
4093 (define_insn "mmx_pshufbv8qi3"
4094   [(set (match_operand:V8QI 0 "register_operand" "=x,Yw")
4095         (unspec:V8QI
4096           [(match_operand:V8QI 1 "register_operand" "0,Yw")
4097            (match_operand:V16QI 2 "vector_operand" "xBm,Ywm")]
4098           UNSPEC_PSHUFB))]
4099   "TARGET_SSSE3 && TARGET_MMX_WITH_SSE"
4100   "@
4101    pshufb\t{%2, %0|%0, %2}
4102    vpshufb\t{%2, %1, %0|%0, %1, %2}"
4103   [(set_attr "isa" "noavx,avx")
4104    (set_attr "type" "sselog1")
4105    (set_attr "prefix_extra" "1")
4106    (set_attr "prefix" "orig,maybe_evex")
4107    (set_attr "btver2_decode" "vector")
4108    (set_attr "mode" "TI")])
4109
4110 (define_insn "mmx_pshufbv4qi3"
4111   [(set (match_operand:V4QI 0 "register_operand" "=x,Yw")
4112         (unspec:V4QI
4113           [(match_operand:V4QI 1 "register_operand" "0,Yw")
4114            (match_operand:V16QI 2 "vector_operand" "xBm,Ywm")]
4115           UNSPEC_PSHUFB))]
4116   "TARGET_SSSE3"
4117   "@
4118    pshufb\t{%2, %0|%0, %2}
4119    vpshufb\t{%2, %1, %0|%0, %1, %2}"
4120   [(set_attr "isa" "noavx,avx")
4121    (set_attr "type" "sselog1")
4122    (set_attr "prefix_extra" "1")
4123    (set_attr "prefix" "orig,maybe_evex")
4124    (set_attr "btver2_decode" "vector")
4125    (set_attr "mode" "TI")])
4126
4127 (define_expand "mmx_pshufw"
4128   [(match_operand:V4HI 0 "register_operand")
4129    (match_operand:V4HI 1 "register_mmxmem_operand")
4130    (match_operand:SI 2 "const_int_operand")]
4131   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4132    && (TARGET_SSE || TARGET_3DNOW_A)"
4133 {
4134   int mask = INTVAL (operands[2]);
4135   emit_insn (gen_mmx_pshufw_1 (operands[0], operands[1],
4136                                GEN_INT ((mask >> 0) & 3),
4137                                GEN_INT ((mask >> 2) & 3),
4138                                GEN_INT ((mask >> 4) & 3),
4139                                GEN_INT ((mask >> 6) & 3)));
4140   DONE;
4141 })
4142
4143 (define_insn "mmx_pshufw_1"
4144   [(set (match_operand:V4HI 0 "register_operand" "=y,Yw")
4145         (vec_select:V4HI
4146           (match_operand:V4HI 1 "register_mmxmem_operand" "ym,Yw")
4147           (parallel [(match_operand 2 "const_0_to_3_operand")
4148                      (match_operand 3 "const_0_to_3_operand")
4149                      (match_operand 4 "const_0_to_3_operand")
4150                      (match_operand 5 "const_0_to_3_operand")])))]
4151   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4152    && (TARGET_SSE || TARGET_3DNOW_A)"
4153 {
4154   int mask = 0;
4155   mask |= INTVAL (operands[2]) << 0;
4156   mask |= INTVAL (operands[3]) << 2;
4157   mask |= INTVAL (operands[4]) << 4;
4158   mask |= INTVAL (operands[5]) << 6;
4159   operands[2] = GEN_INT (mask);
4160
4161   switch (which_alternative)
4162     {
4163     case 0:
4164       return "pshufw\t{%2, %1, %0|%0, %1, %2}";
4165     case 1:
4166       return "%vpshuflw\t{%2, %1, %0|%0, %1, %2}";
4167     default:
4168       gcc_unreachable ();
4169     }
4170 }
4171   [(set_attr "isa" "*,sse2")
4172    (set_attr "mmx_isa" "native,*")
4173    (set_attr "type" "mmxcvt,sselog1")
4174    (set_attr "length_immediate" "1")
4175    (set_attr "mode" "DI,TI")])
4176
4177 (define_insn "*mmx_pshufd_1"
4178   [(set (match_operand:V2SI 0 "register_operand" "=Yv")
4179         (vec_select:V2SI
4180           (match_operand:V2SI 1 "register_operand" "Yv")
4181           (parallel [(match_operand 2 "const_0_to_1_operand")
4182                      (match_operand 3 "const_0_to_1_operand")])))]
4183   "TARGET_MMX_WITH_SSE"
4184 {
4185   int mask = 0;
4186   mask |= INTVAL (operands[2]) << 0;
4187   mask |= INTVAL (operands[3]) << 2;
4188   mask |= 2 << 4;
4189   mask |= 3 << 6;
4190   operands[2] = GEN_INT (mask);
4191
4192   return "%vpshufd\t{%2, %1, %0|%0, %1, %2}";
4193 }
4194   [(set_attr "type" "sselog1")
4195    (set_attr "prefix_data16" "1")
4196    (set_attr "length_immediate" "1")
4197    (set_attr "mode" "TI")])
4198
4199 (define_insn "*mmx_pblendw64"
4200   [(set (match_operand:V4HI 0 "register_operand" "=Yr,*x,x")
4201         (vec_merge:V4HI
4202           (match_operand:V4HI 2 "register_operand" "Yr,*x,x")
4203           (match_operand:V4HI 1 "register_operand" "0,0,x")
4204           (match_operand:SI 3 "const_0_to_15_operand")))]
4205   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
4206   "@
4207    pblendw\t{%3, %2, %0|%0, %2, %3}
4208    pblendw\t{%3, %2, %0|%0, %2, %3}
4209    vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
4210   [(set_attr "isa" "noavx,noavx,avx")
4211    (set_attr "type" "ssemov")
4212    (set_attr "prefix_extra" "1")
4213    (set_attr "length_immediate" "1")
4214    (set_attr "prefix" "orig,orig,vex")
4215    (set_attr "mode" "TI")])
4216
4217 (define_insn "*mmx_pblendw32"
4218   [(set (match_operand:V2HI 0 "register_operand" "=Yr,*x,x")
4219         (vec_merge:V2HI
4220           (match_operand:V2HI 2 "register_operand" "Yr,*x,x")
4221           (match_operand:V2HI 1 "register_operand" "0,0,x")
4222           (match_operand:SI 3 "const_0_to_7_operand")))]
4223   "TARGET_SSE4_1"
4224   "@
4225    pblendw\t{%3, %2, %0|%0, %2, %3}
4226    pblendw\t{%3, %2, %0|%0, %2, %3}
4227    vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
4228   [(set_attr "isa" "noavx,noavx,avx")
4229    (set_attr "type" "ssemov")
4230    (set_attr "prefix_extra" "1")
4231    (set_attr "length_immediate" "1")
4232    (set_attr "prefix" "orig,orig,vex")
4233    (set_attr "mode" "TI")])
4234
4235 ;; Optimize V2SImode load from memory, swapping the elements and
4236 ;; storing back into the memory into DImode rotate of the memory by 32.
4237 (define_split
4238   [(set (match_operand:V2SI 0 "memory_operand")
4239         (vec_select:V2SI (match_dup 0)
4240           (parallel [(const_int 1) (const_int 0)])))]
4241   "TARGET_64BIT && (TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())"
4242   [(set (match_dup 0)
4243         (rotate:DI (match_dup 0) (const_int 32)))]
4244   "operands[0] = adjust_address (operands[0], DImode, 0);")
4245
4246 (define_insn "mmx_pswapdv2si2"
4247   [(set (match_operand:V2SI 0 "register_operand" "=y,Yv")
4248         (vec_select:V2SI
4249           (match_operand:V2SI 1 "register_mmxmem_operand" "ym,Yv")
4250           (parallel [(const_int 1) (const_int 0)])))]
4251   "TARGET_3DNOW_A"
4252   "@
4253    pswapd\t{%1, %0|%0, %1}
4254    %vpshufd\t{$0xe1, %1, %0|%0, %1, 0xe1}";
4255   [(set_attr "isa" "*,sse2")
4256    (set_attr "mmx_isa" "native,*")
4257    (set_attr "type" "mmxcvt,sselog1")
4258    (set_attr "prefix_extra" "1,*")
4259    (set_attr "length_immediate" "*,1")
4260    (set_attr "mode" "DI,TI")])
4261
4262 (define_insn "*vec_dupv4hi"
4263   [(set (match_operand:V4HI 0 "register_operand" "=y,Yw")
4264         (vec_duplicate:V4HI
4265           (truncate:HI
4266             (match_operand:SI 1 "register_operand" "0,Yw"))))]
4267   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4268    && (TARGET_SSE || TARGET_3DNOW_A)"
4269   "@
4270    pshufw\t{$0, %0, %0|%0, %0, 0}
4271    %vpshuflw\t{$0, %1, %0|%0, %1, 0}"
4272   [(set_attr "isa" "*,sse2")
4273    (set_attr "mmx_isa" "native,*")
4274    (set_attr "type" "mmxcvt,sselog1")
4275    (set_attr "length_immediate" "1")
4276    (set_attr "mode" "DI,TI")])
4277
4278
4279 (define_insn "*vec_dupv2si"
4280   [(set (match_operand:V2SI 0 "register_operand" "=y,Yv")
4281         (vec_duplicate:V2SI
4282           (match_operand:SI 1 "register_operand" "0,Yv")))]
4283   "TARGET_MMX || TARGET_MMX_WITH_SSE"
4284   "@
4285    punpckldq\t%0, %0
4286    %vpshufd\t{$0xe0, %1, %0|%0, %1, 0xe0}"
4287   [(set_attr "isa" "*,sse2")
4288    (set_attr "mmx_isa" "native,*")
4289    (set_attr "type" "mmxcvt,sselog1")
4290    (set_attr "prefix_data16" "*,1")
4291    (set_attr "length_immediate" "*,1")
4292    (set_attr "mode" "DI,TI")])
4293
4294 (define_insn "*mmx_concatv2si"
4295   [(set (match_operand:V2SI 0 "register_operand"     "=y,y")
4296         (vec_concat:V2SI
4297           (match_operand:SI 1 "nonimmediate_operand" " 0,rm")
4298           (match_operand:SI 2 "nonimm_or_0_operand"  "ym,C")))]
4299   "TARGET_MMX && !TARGET_SSE"
4300   "@
4301    punpckldq\t{%2, %0|%0, %2}
4302    movd\t{%1, %0|%0, %1}"
4303   [(set_attr "type" "mmxcvt,mmxmov")
4304    (set_attr "mode" "DI")])
4305
4306 (define_expand "vec_setv2si"
4307   [(match_operand:V2SI 0 "register_operand")
4308    (match_operand:SI 1 "register_operand")
4309    (match_operand 2 "vec_setm_mmx_operand")]
4310   "TARGET_MMX || TARGET_MMX_WITH_SSE"
4311 {
4312   if (CONST_INT_P (operands[2]))
4313     ix86_expand_vector_set (TARGET_MMX_WITH_SSE, operands[0], operands[1],
4314                             INTVAL (operands[2]));
4315   else
4316     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
4317   DONE;
4318 })
4319
4320 ;; Avoid combining registers from different units in a single alternative,
4321 ;; see comment above inline_secondary_memory_needed function in i386.cc
4322 (define_insn_and_split "*vec_extractv2si_0"
4323   [(set (match_operand:SI 0 "nonimmediate_operand"     "=x,m,y, m,r,r")
4324         (vec_select:SI
4325           (match_operand:V2SI 1 "nonimmediate_operand" "xm,x,ym,y,m,x")
4326           (parallel [(const_int 0)])))]
4327   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4328    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4329   "#"
4330   "&& reload_completed"
4331   [(set (match_dup 0) (match_dup 1))]
4332   "operands[1] = gen_lowpart (SImode, operands[1]);"
4333   [(set_attr "isa" "*,*,*,*,*,sse2")
4334    (set_attr "mmx_isa" "*,*,native,native,*,*")
4335    (set (attr "preferred_for_speed")
4336      (cond [(eq_attr "alternative" "5")
4337               (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
4338            ]
4339            (symbol_ref "true")))])
4340
4341 (define_insn "*vec_extractv2si_0_zext_sse4"
4342   [(set (match_operand:DI 0 "register_operand" "=r,x")
4343         (zero_extend:DI
4344           (vec_select:SI
4345             (match_operand:V2SI 1 "register_operand" "x,x")
4346             (parallel [(const_int 0)]))))]
4347   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE4_1"
4348   "#"
4349   [(set_attr "isa" "x64,*")
4350    (set (attr "preferred_for_speed")
4351      (cond [(eq_attr "alternative" "0")
4352               (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
4353            ]
4354            (symbol_ref "true")))])
4355
4356 (define_insn "*vec_extractv2si_0_zext"
4357   [(set (match_operand:DI 0 "register_operand" "=r")
4358         (zero_extend:DI
4359           (vec_select:SI
4360             (match_operand:V2SI 1 "register_operand" "x")
4361             (parallel [(const_int 0)]))))]
4362   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4363    && TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_FROM_VEC"
4364   "#")
4365
4366 (define_split
4367   [(set (match_operand:DI 0 "register_operand")
4368         (zero_extend:DI
4369           (vec_select:SI
4370             (match_operand:V2SI 1 "register_operand")
4371             (parallel [(const_int 0)]))))]
4372   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4373    && TARGET_SSE2 && reload_completed"
4374   [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
4375   "operands[1] = gen_lowpart (SImode, operands[1]);")
4376
4377 ;; Avoid combining registers from different units in a single alternative,
4378 ;; see comment above inline_secondary_memory_needed function in i386.cc
4379 (define_insn "*vec_extractv2si_1"
4380   [(set (match_operand:SI 0 "nonimmediate_operand"     "=y,rm,x,x,y,x,r")
4381         (vec_select:SI
4382           (match_operand:V2SI 1 "nonimmediate_operand" " 0,x ,x,0,o,o,o")
4383           (parallel [(const_int 1)])))]
4384   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4385    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4386   "@
4387    punpckhdq\t%0, %0
4388    %vpextrd\t{$1, %1, %0|%0, %1, 1}
4389    %vpshufd\t{$0xe5, %1, %0|%0, %1, 0xe5}
4390    shufps\t{$0xe5, %0, %0|%0, %0, 0xe5}
4391    #
4392    #
4393    #"
4394   [(set_attr "isa" "*,sse4,sse2,noavx,*,*,*")
4395    (set_attr "mmx_isa" "native,*,*,*,native,*,*")
4396    (set_attr "type" "mmxcvt,ssemov,sseshuf1,sseshuf1,mmxmov,ssemov,imov")
4397    (set (attr "length_immediate")
4398      (if_then_else (eq_attr "alternative" "1,2,3")
4399                    (const_string "1")
4400                    (const_string "*")))
4401    (set_attr "prefix" "orig,maybe_vex,maybe_vex,orig,orig,orig,orig")
4402    (set_attr "mode" "DI,TI,TI,V4SF,SI,SI,SI")])
4403
4404 (define_split
4405   [(set (match_operand:SI 0 "register_operand")
4406         (vec_select:SI
4407           (match_operand:V2SI 1 "memory_operand")
4408           (parallel [(const_int 1)])))]
4409   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && reload_completed"
4410   [(set (match_dup 0) (match_dup 1))]
4411   "operands[1] = adjust_address (operands[1], SImode, 4);")
4412
4413 (define_insn "*vec_extractv2si_1_zext"
4414   [(set (match_operand:DI 0 "register_operand" "=r")
4415         (zero_extend:DI
4416           (vec_select:SI
4417             (match_operand:V2SI 1 "register_operand" "x")
4418             (parallel [(const_int 1)]))))]
4419   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4420    && TARGET_64BIT && TARGET_SSE4_1"
4421   "%vpextrd\t{$1, %1, %k0|%k0, %1, 1}"
4422   [(set_attr "type" "sselog1")
4423    (set_attr "prefix_extra" "1")
4424    (set_attr "length_immediate" "1")
4425    (set_attr "prefix" "maybe_vex")
4426    (set_attr "mode" "TI")])
4427
4428 (define_insn_and_split "*vec_extractv2si_zext_mem"
4429   [(set (match_operand:DI 0 "register_operand" "=y,x,r")
4430         (zero_extend:DI
4431           (vec_select:SI
4432             (match_operand:V2SI 1 "memory_operand" "o,o,o")
4433             (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))))]
4434   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_64BIT"
4435   "#"
4436   "&& reload_completed"
4437   [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
4438 {
4439   operands[1] = adjust_address (operands[1], SImode, INTVAL (operands[2]) * 4);
4440 }
4441   [(set_attr "isa" "*,sse2,*")
4442    (set_attr "mmx_isa" "native,*,*")])
4443
4444 (define_expand "vec_extractv2sisi"
4445   [(match_operand:SI 0 "register_operand")
4446    (match_operand:V2SI 1 "register_operand")
4447    (match_operand 2 "const_int_operand")]
4448   "TARGET_MMX || TARGET_MMX_WITH_SSE"
4449 {
4450   ix86_expand_vector_extract (TARGET_MMX_WITH_SSE, operands[0],
4451                               operands[1], INTVAL (operands[2]));
4452   DONE;
4453 })
4454
4455 (define_expand "vec_initv2sisi"
4456   [(match_operand:V2SI 0 "register_operand")
4457    (match_operand 1)]
4458   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4459 {
4460   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
4461                            operands[1]);
4462   DONE;
4463 })
4464
4465 (define_expand "vec_setv4hi"
4466   [(match_operand:V4HI 0 "register_operand")
4467    (match_operand:HI 1 "register_operand")
4468    (match_operand 2 "vec_setm_mmx_operand")]
4469   "TARGET_MMX || TARGET_MMX_WITH_SSE"
4470 {
4471   if (CONST_INT_P (operands[2]))
4472     ix86_expand_vector_set (TARGET_MMX_WITH_SSE, operands[0], operands[1],
4473                             INTVAL (operands[2]));
4474   else
4475     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
4476   DONE;
4477 })
4478
4479 (define_expand "vec_extractv4hihi"
4480   [(match_operand:HI 0 "register_operand")
4481    (match_operand:V4HI 1 "register_operand")
4482    (match_operand 2 "const_int_operand")]
4483   "TARGET_MMX || TARGET_MMX_WITH_SSE"
4484 {
4485   ix86_expand_vector_extract (TARGET_MMX_WITH_SSE, operands[0],
4486                               operands[1], INTVAL (operands[2]));
4487   DONE;
4488 })
4489
4490 (define_expand "vec_initv4hihi"
4491   [(match_operand:V4HI 0 "register_operand")
4492    (match_operand 1)]
4493   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4494 {
4495   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
4496                            operands[1]);
4497   DONE;
4498 })
4499
4500 (define_expand "vec_setv8qi"
4501   [(match_operand:V8QI 0 "register_operand")
4502    (match_operand:QI 1 "register_operand")
4503    (match_operand 2 "vec_setm_mmx_operand")]
4504   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
4505 {
4506   if (CONST_INT_P (operands[2]))
4507     ix86_expand_vector_set (TARGET_MMX_WITH_SSE, operands[0], operands[1],
4508                             INTVAL (operands[2]));
4509   else
4510     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
4511   DONE;
4512 })
4513
4514 (define_expand "vec_extractv8qiqi"
4515   [(match_operand:QI 0 "register_operand")
4516    (match_operand:V8QI 1 "register_operand")
4517    (match_operand 2 "const_int_operand")]
4518   "TARGET_SSE4_1 && TARGET_MMX_WITH_SSE"
4519 {
4520   ix86_expand_vector_extract (TARGET_MMX_WITH_SSE, operands[0],
4521                               operands[1], INTVAL (operands[2]));
4522   DONE;
4523 })
4524
4525 (define_expand "vec_initv8qiqi"
4526   [(match_operand:V8QI 0 "register_operand")
4527    (match_operand 1)]
4528   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4529 {
4530   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
4531                            operands[1]);
4532   DONE;
4533 })
4534
4535 (define_insn "*pinsrw"
4536   [(set (match_operand:V2HI 0 "register_operand" "=x,YW")
4537         (vec_merge:V2HI
4538           (vec_duplicate:V2HI
4539             (match_operand:HI 2 "nonimmediate_operand" "rm,rm"))
4540           (match_operand:V2HI 1 "register_operand" "0,YW")
4541           (match_operand:SI 3 "const_int_operand")))]
4542   "TARGET_SSE2
4543    && ((unsigned) exact_log2 (INTVAL (operands[3]))
4544        < GET_MODE_NUNITS (V2HImode))"
4545 {
4546   operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
4547   switch (which_alternative)
4548     {
4549     case 1:
4550       if (MEM_P (operands[2]))
4551         return "vpinsrw\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4552       else
4553         return "vpinsrw\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
4554     case 0:
4555       if (MEM_P (operands[2]))
4556         return "pinsrw\t{%3, %2, %0|%0, %2, %3}";
4557       else
4558         return "pinsrw\t{%3, %k2, %0|%0, %k2, %3}";
4559     default:
4560       gcc_unreachable ();
4561     }
4562 }
4563   [(set_attr "isa" "noavx,avx")
4564    (set_attr "type" "sselog")
4565    (set_attr "length_immediate" "1")
4566    (set_attr "mode" "TI")])
4567
4568 (define_insn "*pinsrb"
4569   [(set (match_operand:V4QI 0 "register_operand" "=x,YW")
4570         (vec_merge:V4QI
4571           (vec_duplicate:V4QI
4572             (match_operand:QI 2 "nonimmediate_operand" "rm,rm"))
4573           (match_operand:V4QI 1 "register_operand" "0,YW")
4574           (match_operand:SI 3 "const_int_operand")))]
4575   "TARGET_SSE4_1
4576    && ((unsigned) exact_log2 (INTVAL (operands[3]))
4577        < GET_MODE_NUNITS (V4QImode))"
4578 {
4579   operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
4580   switch (which_alternative)
4581     {
4582     case 1:
4583       if (MEM_P (operands[2]))
4584         return "vpinsrb\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4585       else
4586         return "vpinsrb\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
4587     case 0:
4588       if (MEM_P (operands[2]))
4589         return "pinsrb\t{%3, %2, %0|%0, %2, %3}";
4590       else
4591         return "pinsrb\t{%3, %k2, %0|%0, %k2, %3}";
4592     default:
4593       gcc_unreachable ();
4594     }
4595 }
4596   [(set_attr "isa" "noavx,avx")
4597    (set_attr "type" "sselog")
4598    (set_attr "prefix_extra" "1")
4599    (set_attr "length_immediate" "1")
4600    (set_attr "prefix" "orig,vex")
4601    (set_attr "mode" "TI")])
4602
4603 (define_insn "*pextrw"
4604   [(set (match_operand:HI 0 "register_sse4nonimm_operand" "=r,m")
4605         (vec_select:HI
4606           (match_operand:V2HI 1 "register_operand" "YW,YW")
4607           (parallel [(match_operand:SI 2 "const_0_to_1_operand")])))]
4608   "TARGET_SSE2"
4609   "@
4610    %vpextrw\t{%2, %1, %k0|%k0, %1, %2}
4611    %vpextrw\t{%2, %1, %0|%0, %1, %2}"
4612   [(set_attr "isa" "*,sse4")
4613    (set_attr "type" "sselog1")
4614    (set_attr "length_immediate" "1")
4615    (set_attr "prefix" "maybe_vex")
4616    (set_attr "mode" "TI")])
4617
4618 (define_insn "*pextrw_zext"
4619   [(set (match_operand:SWI48 0 "register_operand" "=r")
4620         (zero_extend:SWI48
4621           (vec_select:HI
4622             (match_operand:V2HI 1 "register_operand" "YW")
4623             (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))))]
4624   "TARGET_SSE2"
4625   "%vpextrw\t{%2, %1, %k0|%k0, %1, %2}"
4626   [(set_attr "type" "sselog1")
4627    (set_attr "length_immediate" "1")
4628    (set_attr "prefix" "maybe_vex")
4629    (set_attr "mode" "TI")])
4630
4631 (define_insn "*pextrb"
4632   [(set (match_operand:QI 0 "nonimmediate_operand" "=r,m")
4633         (vec_select:QI
4634           (match_operand:V4QI 1 "register_operand" "YW,YW")
4635           (parallel [(match_operand:SI 2 "const_0_to_3_operand")])))]
4636   "TARGET_SSE4_1"
4637   "@
4638    %vpextrb\t{%2, %1, %k0|%k0, %1, %2}
4639    %vpextrb\t{%2, %1, %0|%0, %1, %2}"
4640   [(set_attr "type" "sselog1")
4641    (set_attr "prefix_extra" "1")
4642    (set_attr "length_immediate" "1")
4643    (set_attr "prefix" "maybe_vex")
4644    (set_attr "mode" "TI")])
4645
4646 (define_insn "*pextrb_zext"
4647   [(set (match_operand:SWI248 0 "register_operand" "=r")
4648         (zero_extend:SWI248
4649           (vec_select:QI
4650             (match_operand:V4QI 1 "register_operand" "YW")
4651             (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
4652   "TARGET_SSE4_1"
4653   "%vpextrb\t{%2, %1, %k0|%k0, %1, %2}"
4654   [(set_attr "type" "sselog1")
4655    (set_attr "prefix_extra" "1")
4656    (set_attr "length_immediate" "1")
4657    (set_attr "prefix" "maybe_vex")
4658    (set_attr "mode" "TI")])
4659
4660 (define_expand "vec_setv2hi"
4661   [(match_operand:V2HI 0 "register_operand")
4662    (match_operand:HI 1 "register_operand")
4663    (match_operand 2 "vec_setm_sse41_operand")]
4664   "TARGET_SSE2"
4665 {
4666   if (CONST_INT_P (operands[2]))
4667     ix86_expand_vector_set (false, operands[0], operands[1],
4668                             INTVAL (operands[2]));
4669   else
4670     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
4671   DONE;
4672 })
4673
4674 (define_expand "vec_extractv2hihi"
4675   [(match_operand:HI 0 "register_operand")
4676    (match_operand:V2HI 1 "register_operand")
4677    (match_operand 2 "const_int_operand")]
4678   "TARGET_SSE2"
4679 {
4680   ix86_expand_vector_extract (false, operands[0],
4681                               operands[1], INTVAL (operands[2]));
4682   DONE;
4683 })
4684
4685 (define_expand "vec_setv4qi"
4686   [(match_operand:V4QI 0 "register_operand")
4687    (match_operand:QI 1 "register_operand")
4688    (match_operand 2 "vec_setm_mmx_operand")]
4689   "TARGET_SSE4_1"
4690 {
4691   if (CONST_INT_P (operands[2]))
4692     ix86_expand_vector_set (false, operands[0], operands[1],
4693                             INTVAL (operands[2]));
4694   else
4695     ix86_expand_vector_set_var (operands[0], operands[1], operands[2]);
4696   DONE;
4697 })
4698
4699 (define_expand "vec_extractv4qiqi"
4700   [(match_operand:QI 0 "register_operand")
4701    (match_operand:V4QI 1 "register_operand")
4702    (match_operand 2 "const_int_operand")]
4703   "TARGET_SSE4_1"
4704 {
4705   ix86_expand_vector_extract (false, operands[0],
4706                               operands[1], INTVAL (operands[2]));
4707   DONE;
4708 })
4709
4710 (define_insn_and_split "*punpckwd"
4711   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
4712         (vec_select:V2HI
4713           (vec_concat:V4HI
4714             (match_operand:V2HI 1 "register_operand" "0,Yw")
4715             (match_operand:V2HI 2 "register_operand" "x,Yw"))
4716           (parallel [(match_operand 3 "const_0_to_3_operand")
4717                      (match_operand 4 "const_0_to_3_operand")])))]
4718   "TARGET_SSE2"
4719   "#"
4720   "&& reload_completed"
4721   [(set (match_dup 5)
4722         (vec_select:V8HI
4723           (match_dup 5)
4724           (parallel [(match_dup 3) (match_dup 4)
4725                      (const_int 2) (const_int 3)
4726                      (const_int 4) (const_int 5)
4727                      (const_int 6) (const_int 7)])))]
4728 {
4729   rtx dest = lowpart_subreg (V8HImode, operands[0], V2HImode);
4730   rtx op1 = lowpart_subreg (V8HImode, operands[1], V2HImode);
4731   rtx op2 = lowpart_subreg (V8HImode, operands[2], V2HImode);
4732
4733   emit_insn (gen_vec_interleave_lowv8hi (dest, op1, op2));
4734
4735   static const int map[4] = { 0, 2, 1, 3 };
4736
4737   int sel0 = map[INTVAL (operands[3])];
4738   int sel1 = map[INTVAL (operands[4])];
4739
4740   if (sel0 == 0 && sel1 == 1)
4741     DONE;
4742
4743   operands[3] = GEN_INT (sel0);
4744   operands[4] = GEN_INT (sel1);
4745   operands[5] = dest;
4746 }
4747   [(set_attr "isa" "noavx,avx")
4748    (set_attr "type" "sselog")
4749    (set_attr "mode" "TI")])
4750
4751 (define_insn "*pshufw_1"
4752   [(set (match_operand:V2HI 0 "register_operand" "=Yw")
4753         (vec_select:V2HI
4754           (match_operand:V2HI 1 "register_operand" "Yw")
4755           (parallel [(match_operand 2 "const_0_to_1_operand")
4756                      (match_operand 3 "const_0_to_1_operand")])))]
4757   "TARGET_SSE2"
4758 {
4759   int mask = 0;
4760   mask |= INTVAL (operands[2]) << 0;
4761   mask |= INTVAL (operands[3]) << 2;
4762   mask |= 2 << 4;
4763   mask |= 3 << 6;
4764   operands[2] = GEN_INT (mask);
4765
4766   return "%vpshuflw\t{%2, %1, %0|%0, %1, %2}";
4767 }
4768   [(set_attr "type" "sselog1")
4769    (set_attr "length_immediate" "1")
4770    (set_attr "mode" "TI")])
4771
4772 (define_insn "*vec_dupv2hi"
4773   [(set (match_operand:V2HI 0 "register_operand" "=Yw")
4774         (vec_duplicate:V2HI
4775           (truncate:HI
4776             (match_operand:SI 1 "register_operand" "Yw"))))]
4777   "TARGET_SSE2"
4778   "%vpshuflw\t{$0, %1, %0|%0, %1, 0}"
4779   [(set_attr "type" "sselog1")
4780    (set_attr "length_immediate" "1")
4781    (set_attr "mode" "TI")])
4782
4783 (define_expand "vec_initv2hihi"
4784   [(match_operand:V2HI 0 "register_operand")
4785    (match_operand 1)]
4786   "TARGET_SSE2"
4787 {
4788   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
4789                            operands[1]);
4790   DONE;
4791 })
4792
4793 (define_expand "vec_initv4qiqi"
4794   [(match_operand:V2HI 0 "register_operand")
4795    (match_operand 1)]
4796   "TARGET_SSE2"
4797 {
4798   ix86_expand_vector_init (TARGET_MMX_WITH_SSE, operands[0],
4799                            operands[1]);
4800   DONE;
4801 })
4802
4803 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4804 ;;
4805 ;; Miscellaneous
4806 ;;
4807 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4808
4809 (define_expand "mmx_uavg<mode>3"
4810   [(set (match_operand:MMXMODE12 0 "register_operand")
4811         (truncate:MMXMODE12
4812           (lshiftrt:<mmxdoublemode>
4813             (plus:<mmxdoublemode>
4814               (plus:<mmxdoublemode>
4815                 (zero_extend:<mmxdoublemode>
4816                   (match_operand:MMXMODE12 1 "register_mmxmem_operand"))
4817                 (zero_extend:<mmxdoublemode>
4818                   (match_operand:MMXMODE12 2 "register_mmxmem_operand")))
4819               (match_dup 3))
4820             (const_int 1))))]
4821   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4822    && (TARGET_SSE || TARGET_3DNOW)"
4823 {
4824   operands[3] = CONST1_RTX(<mmxdoublemode>mode);
4825   ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
4826 })
4827
4828 (define_insn "*mmx_uavgv8qi3"
4829   [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yw")
4830         (truncate:V8QI
4831           (lshiftrt:V8HI
4832             (plus:V8HI
4833               (plus:V8HI
4834                 (zero_extend:V8HI
4835                   (match_operand:V8QI 1 "register_mmxmem_operand" "%0,0,Yw"))
4836                 (zero_extend:V8HI
4837                   (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yw")))
4838               (const_vector:V8HI [(const_int 1) (const_int 1)
4839                                   (const_int 1) (const_int 1)
4840                                   (const_int 1) (const_int 1)
4841                                   (const_int 1) (const_int 1)]))
4842             (const_int 1))))]
4843   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4844    && (TARGET_SSE || TARGET_3DNOW)
4845    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
4846 {
4847   switch (which_alternative)
4848     {
4849     case 2:
4850       return "vpavgb\t{%2, %1, %0|%0, %1, %2}";
4851     case 1:
4852     case 0:
4853       /* These two instructions have the same operation, but their encoding
4854          is different.  Prefer the one that is de facto standard.  */
4855       if (TARGET_SSE || TARGET_3DNOW_A)
4856         return "pavgb\t{%2, %0|%0, %2}";
4857       else
4858         return "pavgusb\t{%2, %0|%0, %2}";
4859       default:
4860         gcc_unreachable ();
4861     }
4862 }
4863   [(set_attr "isa" "*,sse2_noavx,avx")
4864    (set_attr "mmx_isa" "native,*,*")
4865    (set_attr "type" "mmxshft,sseiadd,sseiadd")
4866    (set (attr "prefix_extra")
4867      (if_then_else
4868        (not (ior (match_test "TARGET_SSE")
4869                  (match_test "TARGET_3DNOW_A")))
4870        (const_string "1")
4871        (const_string "*")))
4872    (set_attr "mode" "DI,TI,TI")])
4873
4874 (define_insn "*mmx_uavgv4hi3"
4875   [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yw")
4876         (truncate:V4HI
4877           (lshiftrt:V4SI
4878             (plus:V4SI
4879               (plus:V4SI
4880                 (zero_extend:V4SI
4881                   (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yw"))
4882                 (zero_extend:V4SI
4883                   (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yw")))
4884               (const_vector:V4SI [(const_int 1) (const_int 1)
4885                                   (const_int 1) (const_int 1)]))
4886             (const_int 1))))]
4887   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
4888    && (TARGET_SSE || TARGET_3DNOW_A)
4889    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
4890   "@
4891    pavgw\t{%2, %0|%0, %2}
4892    pavgw\t{%2, %0|%0, %2}
4893    vpavgw\t{%2, %1, %0|%0, %1, %2}"
4894   [(set_attr "isa" "*,sse2_noavx,avx")
4895    (set_attr "mmx_isa" "native,*,*")
4896    (set_attr "type" "mmxshft,sseiadd,sseiadd")
4897    (set_attr "mode" "DI,TI,TI")])
4898
4899 (define_expand "uavg<mode>3_ceil"
4900   [(set (match_operand:MMXMODE12 0 "register_operand")
4901         (truncate:MMXMODE12
4902           (lshiftrt:<mmxdoublemode>
4903             (plus:<mmxdoublemode>
4904               (plus:<mmxdoublemode>
4905                 (zero_extend:<mmxdoublemode>
4906                   (match_operand:MMXMODE12 1 "register_operand"))
4907                 (zero_extend:<mmxdoublemode>
4908                   (match_operand:MMXMODE12 2 "register_operand")))
4909               (match_dup 3))
4910             (const_int 1))))]
4911   "TARGET_MMX_WITH_SSE"
4912   "operands[3] = CONST1_RTX(<mmxdoublemode>mode);")
4913
4914 (define_insn "uavgv4qi3_ceil"
4915   [(set (match_operand:V4QI 0 "register_operand" "=x,Yw")
4916         (truncate:V4QI
4917           (lshiftrt:V4HI
4918             (plus:V4HI
4919               (plus:V4HI
4920                 (zero_extend:V4HI
4921                   (match_operand:V4QI 1 "register_operand" "%0,Yw"))
4922                 (zero_extend:V4HI
4923                   (match_operand:V4QI 2 "register_operand" "x,Yw")))
4924               (const_vector:V4HI [(const_int 1) (const_int 1)
4925                                   (const_int 1) (const_int 1)]))
4926             (const_int 1))))]
4927   "TARGET_SSE2"
4928   "@
4929    pavgb\t{%2, %0|%0, %2}
4930    vpavgb\t{%2, %1, %0|%0, %1, %2}"
4931   [(set_attr "isa" "noavx,avx")
4932    (set_attr "type" "sseiadd")
4933    (set_attr "mode" "TI")])
4934
4935 (define_insn "uavgv2qi3_ceil"
4936   [(set (match_operand:V2QI 0 "register_operand" "=x,Yw")
4937         (truncate:V2QI
4938           (lshiftrt:V2HI
4939             (plus:V2HI
4940               (plus:V2HI
4941                 (zero_extend:V2HI
4942                   (match_operand:V2QI 1 "register_operand" "%0,Yw"))
4943                 (zero_extend:V2HI
4944                   (match_operand:V2QI 2 "register_operand" "x,Yw")))
4945               (const_vector:V2HI [(const_int 1) (const_int 1)]))
4946             (const_int 1))))]
4947   "TARGET_SSE2"
4948   "@
4949    pavgb\t{%2, %0|%0, %2}
4950    vpavgb\t{%2, %1, %0|%0, %1, %2}"
4951   [(set_attr "isa" "noavx,avx")
4952    (set_attr "type" "sseiadd")
4953    (set_attr "mode" "TI")])
4954
4955 (define_insn "uavgv2hi3_ceil"
4956   [(set (match_operand:V2HI 0 "register_operand" "=x,Yw")
4957         (truncate:V2HI
4958           (lshiftrt:V2SI
4959             (plus:V2SI
4960               (plus:V2SI
4961                 (zero_extend:V2SI
4962                   (match_operand:V2HI 1 "register_operand" "%0,Yw"))
4963                 (zero_extend:V2SI
4964                   (match_operand:V2HI 2 "register_operand" "x,Yw")))
4965               (const_vector:V2SI [(const_int 1) (const_int 1)]))
4966             (const_int 1))))]
4967   "TARGET_SSE2"
4968   "@
4969    pavgw\t{%2, %0|%0, %2}
4970    vpavgw\t{%2, %1, %0|%0, %1, %2}"
4971   [(set_attr "isa" "noavx,avx")
4972    (set_attr "type" "sseiadd")
4973    (set_attr "mode" "TI")])
4974
4975 (define_expand "mmx_psadbw"
4976   [(set (match_operand:V1DI 0 "register_operand")
4977         (unspec:V1DI [(match_operand:V8QI 1 "register_mmxmem_operand")
4978                       (match_operand:V8QI 2 "register_mmxmem_operand")]
4979                      UNSPEC_PSADBW))]
4980   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && (TARGET_SSE || TARGET_3DNOW_A)"
4981   "ix86_fixup_binary_operands_no_copy (PLUS, V8QImode, operands);")
4982
4983 (define_insn "*mmx_psadbw"
4984   [(set (match_operand:V1DI 0 "register_operand" "=y,x,Yw")
4985         (unspec:V1DI [(match_operand:V8QI 1 "register_mmxmem_operand" "%0,0,Yw")
4986                       (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yw")]
4987                      UNSPEC_PSADBW))]
4988   "(TARGET_MMX || TARGET_MMX_WITH_SSE) && (TARGET_SSE || TARGET_3DNOW_A)
4989    && ix86_binary_operator_ok (PLUS, V8QImode, operands)"
4990   "@
4991    psadbw\t{%2, %0|%0, %2}
4992    psadbw\t{%2, %0|%0, %2}
4993    vpsadbw\t{%2, %1, %0|%0, %1, %2}"
4994   [(set_attr "isa" "*,sse2_noavx,avx")
4995    (set_attr "mmx_isa" "native,*,*")
4996    (set_attr "type" "mmxshft,sseiadd,sseiadd")
4997    (set_attr "mode" "DI,TI,TI")])
4998
4999 (define_expand "reduc_plus_scal_v8qi"
5000  [(plus:V8QI
5001     (match_operand:QI 0 "register_operand")
5002     (match_operand:V8QI 1 "register_operand"))]
5003  "TARGET_MMX_WITH_SSE"
5004 {
5005   rtx tmp = gen_reg_rtx (V8QImode);
5006   emit_move_insn (tmp, CONST0_RTX (V8QImode));
5007   rtx tmp2 = gen_reg_rtx (V1DImode);
5008   emit_insn (gen_mmx_psadbw (tmp2, operands[1], tmp));
5009   tmp2 = gen_lowpart (V8QImode, tmp2);
5010   emit_insn (gen_vec_extractv8qiqi (operands[0], tmp2, const0_rtx));
5011   DONE;
5012 })
5013
5014 (define_expand "reduc_plus_scal_v4hi"
5015  [(plus:V4HI
5016     (match_operand:HI 0 "register_operand")
5017     (match_operand:V4HI 1 "register_operand"))]
5018  "TARGET_MMX_WITH_SSE"
5019 {
5020   rtx tmp = gen_reg_rtx (V4HImode);
5021   ix86_expand_reduc (gen_addv4hi3, tmp, operands[1]);
5022   emit_insn (gen_vec_extractv4hihi (operands[0], tmp, const0_rtx));
5023   DONE;
5024 })
5025
5026 (define_expand "reduc_<code>_scal_v4hi"
5027   [(smaxmin:V4HI
5028      (match_operand:HI 0 "register_operand")
5029      (match_operand:V4HI 1 "register_operand"))]
5030   "TARGET_MMX_WITH_SSE"
5031 {
5032   rtx tmp = gen_reg_rtx (V4HImode);
5033   ix86_expand_reduc (gen_<code>v4hi3, tmp, operands[1]);
5034   emit_insn (gen_vec_extractv4hihi (operands[0], tmp, const0_rtx));
5035   DONE;
5036 })
5037
5038 (define_expand "reduc_<code>_scal_v4qi"
5039   [(smaxmin:V4QI
5040      (match_operand:QI 0 "register_operand")
5041      (match_operand:V4QI 1 "register_operand"))]
5042   "TARGET_SSE4_1"
5043 {
5044   rtx tmp = gen_reg_rtx (V4QImode);
5045   ix86_expand_reduc (gen_<code>v4qi3, tmp, operands[1]);
5046   emit_insn (gen_vec_extractv4qiqi (operands[0], tmp, const0_rtx));
5047   DONE;
5048 })
5049
5050 (define_expand "reduc_<code>_scal_v4hi"
5051   [(umaxmin:V4HI
5052      (match_operand:HI 0 "register_operand")
5053      (match_operand:V4HI 1 "register_operand"))]
5054   "TARGET_MMX_WITH_SSE && TARGET_SSE4_1"
5055 {
5056   rtx tmp = gen_reg_rtx (V4HImode);
5057   ix86_expand_reduc (gen_<code>v4hi3, tmp, operands[1]);
5058   emit_insn (gen_vec_extractv4hihi (operands[0], tmp, const0_rtx));
5059   DONE;
5060 })
5061
5062 (define_expand "reduc_<code>_scal_v4qi"
5063   [(umaxmin:V4QI
5064      (match_operand:QI 0 "register_operand")
5065      (match_operand:V4QI 1 "register_operand"))]
5066   "TARGET_SSE4_1"
5067 {
5068   rtx tmp = gen_reg_rtx (V4QImode);
5069   ix86_expand_reduc (gen_<code>v4qi3, tmp, operands[1]);
5070   emit_insn (gen_vec_extractv4qiqi (operands[0], tmp, const0_rtx));
5071   DONE;
5072 })
5073
5074 (define_expand "reduc_plus_scal_v4qi"
5075  [(plus:V4QI
5076     (match_operand:QI 0 "register_operand")
5077     (match_operand:V4QI 1 "register_operand"))]
5078  "TARGET_SSE2"
5079 {
5080   rtx op1 = gen_reg_rtx (V16QImode);
5081   emit_insn (gen_vec_setv4si_0 (lowpart_subreg (V4SImode, op1, V16QImode),
5082                                 CONST0_RTX (V4SImode),
5083                                 lowpart_subreg (SImode,
5084                                                 operands[1],
5085                                                 V4QImode)));
5086   rtx tmp = gen_reg_rtx (V16QImode);
5087   emit_move_insn (tmp, CONST0_RTX (V16QImode));
5088   rtx tmp2 = gen_reg_rtx (V2DImode);
5089   emit_insn (gen_sse2_psadbw (tmp2, op1, tmp));
5090   tmp2 = gen_lowpart (V16QImode, tmp2);
5091   emit_insn (gen_vec_extractv16qiqi (operands[0], tmp2, const0_rtx));
5092   DONE;
5093 })
5094
5095 (define_expand "usadv8qi"
5096   [(match_operand:V2SI 0 "register_operand")
5097    (match_operand:V8QI 1 "register_operand")
5098    (match_operand:V8QI 2 "register_operand")
5099    (match_operand:V2SI 3 "register_operand")]
5100   "TARGET_MMX_WITH_SSE"
5101 {
5102   rtx t1 = gen_reg_rtx (V1DImode);
5103   rtx t2 = gen_reg_rtx (V2SImode);
5104   emit_insn (gen_mmx_psadbw (t1, operands[1], operands[2]));
5105   convert_move (t2, t1, 0);
5106   emit_insn (gen_addv2si3 (operands[0], t2, operands[3]));
5107   DONE;
5108 })
5109
5110 (define_insn_and_split "mmx_pmovmskb"
5111   [(set (match_operand:SI 0 "register_operand" "=r,r")
5112         (unspec:SI [(match_operand:V8QI 1 "register_operand" "y,x")]
5113                    UNSPEC_MOVMSK))]
5114   "(TARGET_MMX || TARGET_MMX_WITH_SSE)
5115    && (TARGET_SSE || TARGET_3DNOW_A)"
5116   "@
5117    pmovmskb\t{%1, %0|%0, %1}
5118    #"
5119   "TARGET_SSE2 && reload_completed
5120    && SSE_REGNO_P (REGNO (operands[1]))"
5121   [(set (match_dup 0)
5122         (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK))
5123    (set (match_dup 0)
5124         (zero_extend:SI (match_dup 2)))]
5125 {
5126   /* Generate SSE pmovmskb and zero-extend from QImode to SImode.  */
5127   operands[1] = lowpart_subreg (V16QImode, operands[1],
5128                                 GET_MODE (operands[1]));
5129   operands[2] = lowpart_subreg (QImode, operands[0],
5130                                 GET_MODE (operands[0]));
5131 }
5132   [(set_attr "mmx_isa" "native,sse")
5133    (set_attr "type" "mmxcvt,ssemov")
5134    (set_attr "mode" "DI,TI")])
5135
5136 (define_expand "mmx_maskmovq"
5137   [(set (match_operand:V8QI 0 "memory_operand")
5138         (unspec:V8QI [(match_operand:V8QI 1 "register_operand")
5139                       (match_operand:V8QI 2 "register_operand")
5140                       (match_dup 0)]
5141                      UNSPEC_MASKMOV))]
5142   "TARGET_SSE || TARGET_3DNOW_A")
5143
5144 (define_insn "*mmx_maskmovq"
5145   [(set (mem:V8QI (match_operand:P 0 "register_operand" "D"))
5146         (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "y")
5147                       (match_operand:V8QI 2 "register_operand" "y")
5148                       (mem:V8QI (match_dup 0))]
5149                      UNSPEC_MASKMOV))]
5150   "TARGET_SSE || TARGET_3DNOW_A"
5151   ;; @@@ check ordering of operands in intel/nonintel syntax
5152   "maskmovq\t{%2, %1|%1, %2}"
5153   [(set_attr "type" "mmxcvt")
5154    (set_attr "znver1_decode" "vector")
5155    (set_attr "mode" "DI")])
5156
5157 (define_int_iterator EMMS
5158   [(UNSPECV_EMMS "TARGET_MMX")
5159    (UNSPECV_FEMMS "TARGET_3DNOW")])
5160
5161 (define_int_attr emms
5162   [(UNSPECV_EMMS "emms")
5163    (UNSPECV_FEMMS "femms")])
5164
5165 (define_expand "mmx_<emms>"
5166   [(parallel
5167     [(unspec_volatile [(const_int 0)] EMMS)
5168       (clobber (reg:XF ST0_REG))
5169       (clobber (reg:XF ST1_REG))
5170       (clobber (reg:XF ST2_REG))
5171       (clobber (reg:XF ST3_REG))
5172       (clobber (reg:XF ST4_REG))
5173       (clobber (reg:XF ST5_REG))
5174       (clobber (reg:XF ST6_REG))
5175       (clobber (reg:XF ST7_REG))
5176       (clobber (reg:DI MM0_REG))
5177       (clobber (reg:DI MM1_REG))
5178       (clobber (reg:DI MM2_REG))
5179       (clobber (reg:DI MM3_REG))
5180       (clobber (reg:DI MM4_REG))
5181       (clobber (reg:DI MM5_REG))
5182       (clobber (reg:DI MM6_REG))
5183       (clobber (reg:DI MM7_REG))])]
5184   "TARGET_MMX || TARGET_MMX_WITH_SSE"
5185 {
5186    if (!TARGET_MMX)
5187      {
5188        emit_insn (gen_nop ());
5189        DONE;
5190      }
5191 })
5192
5193 (define_insn "*mmx_<emms>"
5194   [(unspec_volatile [(const_int 0)] EMMS)
5195    (clobber (reg:XF ST0_REG))
5196    (clobber (reg:XF ST1_REG))
5197    (clobber (reg:XF ST2_REG))
5198    (clobber (reg:XF ST3_REG))
5199    (clobber (reg:XF ST4_REG))
5200    (clobber (reg:XF ST5_REG))
5201    (clobber (reg:XF ST6_REG))
5202    (clobber (reg:XF ST7_REG))
5203    (clobber (reg:DI MM0_REG))
5204    (clobber (reg:DI MM1_REG))
5205    (clobber (reg:DI MM2_REG))
5206    (clobber (reg:DI MM3_REG))
5207    (clobber (reg:DI MM4_REG))
5208    (clobber (reg:DI MM5_REG))
5209    (clobber (reg:DI MM6_REG))
5210    (clobber (reg:DI MM7_REG))]
5211   ""
5212   "<emms>"
5213   [(set_attr "type" "mmx")
5214    (set_attr "modrm" "0")
5215    (set_attr "memory" "none")])