gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2016 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V15_REGNUM         47)
  61     (V31_REGNUM         63)
  62     (SFP_REGNUM         64)
  63     (AP_REGNUM          65)
  64     (CC_REGNUM          66)
  65   ]
  66 )
  67
  68 (define_c_enum "unspec" [
  69     UNSPEC_CASESI
  70     UNSPEC_CRC32B
  71     UNSPEC_CRC32CB
  72     UNSPEC_CRC32CH
  73     UNSPEC_CRC32CW
  74     UNSPEC_CRC32CX
  75     UNSPEC_CRC32H
  76     UNSPEC_CRC32W
  77     UNSPEC_CRC32X
  78     UNSPEC_URECPE
  79     UNSPEC_FRECPE
  80     UNSPEC_FRECPS
  81     UNSPEC_FRECPX
  82     UNSPEC_FRINTA
  83     UNSPEC_FRINTI
  84     UNSPEC_FRINTM
  85     UNSPEC_FRINTN
  86     UNSPEC_FRINTP
  87     UNSPEC_FRINTX
  88     UNSPEC_FRINTZ
  89     UNSPEC_GOTSMALLPIC
  90     UNSPEC_GOTSMALLPIC28K
  91     UNSPEC_GOTSMALLTLS
  92     UNSPEC_GOTTINYPIC
  93     UNSPEC_GOTTINYTLS
  94     UNSPEC_LD1
  95     UNSPEC_LD2
  96     UNSPEC_LD2_DUP
  97     UNSPEC_LD3
  98     UNSPEC_LD3_DUP
  99     UNSPEC_LD4
 100     UNSPEC_LD4_DUP
 101     UNSPEC_LD2_LANE
 102     UNSPEC_LD3_LANE
 103     UNSPEC_LD4_LANE
 104     UNSPEC_MB
 105     UNSPEC_NOP
 106     UNSPEC_PRLG_STK
 107     UNSPEC_RBIT
 108     UNSPEC_SISD_NEG
 109     UNSPEC_SISD_SSHL
 110     UNSPEC_SISD_USHL
 111     UNSPEC_SSHL_2S
 112     UNSPEC_ST1
 113     UNSPEC_ST2
 114     UNSPEC_ST3
 115     UNSPEC_ST4
 116     UNSPEC_ST2_LANE
 117     UNSPEC_ST3_LANE
 118     UNSPEC_ST4_LANE
 119     UNSPEC_TLS
 120     UNSPEC_TLSDESC
 121     UNSPEC_TLSLE12
 122     UNSPEC_TLSLE24
 123     UNSPEC_TLSLE32
 124     UNSPEC_TLSLE48
 125     UNSPEC_USHL_2S
 126     UNSPEC_VSTRUCTDUMMY
 127     UNSPEC_SP_SET
 128     UNSPEC_SP_TEST
 129     UNSPEC_RSQRT
 130     UNSPEC_RSQRTE
 131     UNSPEC_RSQRTS
 132 ])
 133
 134 (define_c_enum "unspecv" [
 135     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 136     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 137     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 138     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 139     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 140     UNSPECV_BLOCKAGE            ; Represent a blockage
 141     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 142   ]
 143 )
 144
 145 ;; If further include files are added the defintion of MD_INCLUDES
 146 ;; must be updated.
 147
 148 (include "constraints.md")
 149 (include "predicates.md")
 150 (include "iterators.md")
 151
 152 ;; -------------------------------------------------------------------
 153 ;; Instruction types and attributes
 154 ;; -------------------------------------------------------------------
 155
 156 ; The "type" attribute is included here from AArch32 backend to be able
 157 ; to share pipeline descriptions.
 158 (include "../arm/types.md")
 159
 160 ;; It is important to set the fp or simd attributes to yes when a pattern
 161 ;; alternative uses the FP or SIMD register files, usually signified by use of
 162 ;; the 'w' constraint.  This will ensure that the alternative will be
 163 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 164 ;; architecture extensions.  If all the alternatives in a pattern use the
 165 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 166 ;; or TARGET_SIMD.
 167
 168 ;; Attribute that specifies whether or not the instruction touches fp
 169 ;; registers.  When this is set to yes for an alternative, that alternative
 170 ;; will be disabled when !TARGET_FLOAT.
 171 (define_attr "fp" "no,yes" (const_string "no"))
 172
 173 ;; Attribute that specifies whether or not the instruction touches simd
 174 ;; registers.  When this is set to yes for an alternative, that alternative
 175 ;; will be disabled when !TARGET_SIMD.
 176 (define_attr "simd" "no,yes" (const_string "no"))
 177
 178 (define_attr "length" ""
 179   (const_int 4))
 180
 181 ;; Attribute that controls whether an alternative is enabled or not.
 182 ;; Currently it is only used to disable alternatives which touch fp or simd
 183 ;; registers when -mgeneral-regs-only is specified.
 184 (define_attr "enabled" "no,yes"
 185   (cond [(ior
 186         (and (eq_attr "fp" "yes")
 187              (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 188         (and (eq_attr "simd" "yes")
 189              (eq (symbol_ref "TARGET_SIMD") (const_int 0))))
 190              (const_string "no")
 191         ] (const_string "yes")))
 192
 193 ;; Attribute that specifies whether we are dealing with a branch to a
 194 ;; label that is far away, i.e. further away than the maximum/minimum
 195 ;; representable in a signed 21-bits number.
 196 ;; 0 :=: no
 197 ;; 1 :=: yes
 198 (define_attr "far_branch" "" (const_int 0))
 199
 200 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 201 ;; no predicated insns.
 202 (define_attr "predicated" "yes,no" (const_string "no"))
 203
 204 ;; -------------------------------------------------------------------
 205 ;; Pipeline descriptions and scheduling
 206 ;; -------------------------------------------------------------------
 207
 208 ;; Processor types.
 209 (include "aarch64-tune.md")
 210
 211 ;; Scheduling
 212 (include "../arm/cortex-a53.md")
 213 (include "../arm/cortex-a57.md")
 214 (include "../arm/exynos-m1.md")
 215 (include "thunderx.md")
 216 (include "../arm/xgene1.md")
 217
 218 ;; -------------------------------------------------------------------
 219 ;; Jumps and other miscellaneous insns
 220 ;; -------------------------------------------------------------------
 221
 222 (define_insn "indirect_jump"
 223   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 224   ""
 225   "br\\t%0"
 226   [(set_attr "type" "branch")]
 227 )
 228
 229 (define_insn "jump"
 230   [(set (pc) (label_ref (match_operand 0 "" "")))]
 231   ""
 232   "b\\t%l0"
 233   [(set_attr "type" "branch")]
 234 )
 235
 236 (define_expand "cbranch<mode>4"
 237   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 238                             [(match_operand:GPI 1 "register_operand" "")
 239                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 240                            (label_ref (match_operand 3 "" ""))
 241                            (pc)))]
 242   ""
 243   "
 244   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 245                                          operands[2]);
 246   operands[2] = const0_rtx;
 247   "
 248 )
 249
 250 (define_expand "cbranch<mode>4"
 251   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 252                             [(match_operand:GPF 1 "register_operand" "")
 253                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 254                            (label_ref (match_operand 3 "" ""))
 255                            (pc)))]
 256   ""
 257   "
 258   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 259                                          operands[2]);
 260   operands[2] = const0_rtx;
 261   "
 262 )
 263
 264 (define_expand "cbranchcc4"
 265   [(set (pc) (if_then_else
 266               (match_operator 0 "aarch64_comparison_operator"
 267                [(match_operand 1 "cc_register" "")
 268                 (match_operand 2 "const0_operand")])
 269               (label_ref (match_operand 3 "" ""))
 270               (pc)))]
 271   ""
 272   "")
 273
 274 (define_insn "ccmp_and<mode>"
 275   [(set (match_operand 1 "ccmp_cc_register" "")
 276         (compare
 277          (and:SI
 278           (match_operator 4 "aarch64_comparison_operator"
 279            [(match_operand 0 "ccmp_cc_register" "")
 280             (const_int 0)])
 281           (match_operator 5 "aarch64_comparison_operator"
 282            [(match_operand:GPI 2 "register_operand" "r,r,r")
 283             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 284          (const_int 0)))]
 285   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 286   "@
 287    ccmp\\t%<w>2, %<w>3, %k5, %m4
 288    ccmp\\t%<w>2, %<w>3, %k5, %m4
 289    ccmn\\t%<w>2, #%n3, %k5, %m4"
 290   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 291 )
 292
 293 (define_insn "ccmp_ior<mode>"
 294   [(set (match_operand 1 "ccmp_cc_register" "")
 295         (compare
 296          (ior:SI
 297           (match_operator 4 "aarch64_comparison_operator"
 298            [(match_operand 0 "ccmp_cc_register" "")
 299             (const_int 0)])
 300           (match_operator 5 "aarch64_comparison_operator"
 301            [(match_operand:GPI 2 "register_operand" "r,r,r")
 302             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 303          (const_int 0)))]
 304   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 305   "@
 306    ccmp\\t%<w>2, %<w>3, %K5, %M4
 307    ccmp\\t%<w>2, %<w>3, %K5, %M4
 308    ccmn\\t%<w>2, #%n3, %K5, %M4"
 309   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 310 )
 311
 312 (define_expand "cmp<mode>"
 313   [(set (match_operand 0 "cc_register" "")
 314         (match_operator:CC 1 "aarch64_comparison_operator"
 315          [(match_operand:GPI 2 "register_operand" "")
 316           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
 317   ""
 318   {
 319     operands[1] = gen_rtx_fmt_ee (COMPARE,
 320                                   SELECT_CC_MODE (GET_CODE (operands[1]),
 321                                                   operands[2], operands[3]),
 322                                   operands[2], operands[3]);
 323   }
 324 )
 325
 326 ;; Expansion of signed mod by a power of 2 using CSNEG.
 327 ;; For x0 % n where n is a power of 2 produce:
 328 ;; negs   x1, x0
 329 ;; and    x0, x0, #(n - 1)
 330 ;; and    x1, x1, #(n - 1)
 331 ;; csneg  x0, x0, x1, mi
 332
 333 (define_expand "mod<mode>3"
 334   [(match_operand:GPI 0 "register_operand" "")
 335    (match_operand:GPI 1 "register_operand" "")
 336    (match_operand:GPI 2 "const_int_operand" "")]
 337   ""
 338   {
 339     HOST_WIDE_INT val = INTVAL (operands[2]);
 340
 341     if (val <= 0
 342        || exact_log2 (val) <= 0
 343        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 344       FAIL;
 345
 346     rtx mask = GEN_INT (val - 1);
 347
 348     /* In the special case of x0 % 2 we can do the even shorter:
 349         cmp    x0, xzr
 350         and    x0, x0, 1
 351         cneg   x0, x0, lt.  */
 352     if (val == 2)
 353       {
 354         rtx masked = gen_reg_rtx (<MODE>mode);
 355         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 356         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 357         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 358         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 359         DONE;
 360       }
 361
 362     rtx neg_op = gen_reg_rtx (<MODE>mode);
 363     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 364
 365     /* Extract the condition register and mode.  */
 366     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 367     rtx cc_reg = SET_DEST (cmp);
 368     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 369
 370     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 371     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 372
 373     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 374     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 375
 376     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 377                                        masked_neg, masked_pos));
 378     DONE;
 379   }
 380 )
 381
 382 (define_insn "condjump"
 383   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 384                             [(match_operand 1 "cc_register" "") (const_int 0)])
 385                            (label_ref (match_operand 2 "" ""))
 386                            (pc)))]
 387   ""
 388   {
 389     if (get_attr_length (insn) == 8)
 390       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 391     else
 392       return  "b%m0\\t%l2";
 393   }
 394   [(set_attr "type" "branch")
 395    (set (attr "length")
 396         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 397                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 398                       (const_int 4)
 399                       (const_int 8)))
 400    (set (attr "far_branch")
 401         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 402                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 403                       (const_int 0)
 404                       (const_int 1)))]
 405 )
 406
 407 ;; For a 24-bit immediate CST we can optimize the compare for equality
 408 ;; and branch sequence from:
 409 ;;      mov     x0, #imm1
 410 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 411 ;;      cmp     x1, x0
 412 ;;      b<ne,eq> .Label
 413 ;; into the shorter:
 414 ;;      sub     x0, x1, #(CST & 0xfff000)
 415 ;;      subs    x0, x0, #(CST & 0x000fff)
 416 ;;      b<ne,eq> .Label
 417 (define_insn_and_split "*compare_condjump<mode>"
 418   [(set (pc) (if_then_else (EQL
 419                               (match_operand:GPI 0 "register_operand" "r")
 420                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 421                            (label_ref:P (match_operand 2 "" ""))
 422                            (pc)))]
 423   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 424    && !aarch64_plus_operand (operands[1], <MODE>mode)
 425    && !reload_completed"
 426   "#"
 427   "&& true"
 428   [(const_int 0)]
 429   {
 430     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 431     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 432     rtx tmp = gen_reg_rtx (<MODE>mode);
 433     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 434     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 435     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 436     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 437     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 438     DONE;
 439   }
 440 )
 441
 442 (define_expand "casesi"
 443   [(match_operand:SI 0 "register_operand" "")   ; Index
 444    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 445    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 446    (match_operand:DI 3 "" "")                   ; Table label
 447    (match_operand:DI 4 "" "")]                  ; Out of range label
 448   ""
 449   {
 450     if (operands[1] != const0_rtx)
 451       {
 452         rtx reg = gen_reg_rtx (SImode);
 453
 454         /* Canonical RTL says that if you have:
 455
 456            (minus (X) (CONST))
 457
 458            then this should be emitted as:
 459
 460            (plus (X) (-CONST))
 461
 462            The use of trunc_int_for_mode ensures that the resulting
 463            constant can be represented in SImode, this is important
 464            for the corner case where operand[1] is INT_MIN.  */
 465
 466         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 467
 468         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 469               (operands[1], SImode))
 470           operands[1] = force_reg (SImode, operands[1]);
 471         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 472         operands[0] = reg;
 473       }
 474
 475     if (!aarch64_plus_operand (operands[2], SImode))
 476       operands[2] = force_reg (SImode, operands[2]);
 477     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 478                                                  const0_rtx),
 479                                     operands[0], operands[2], operands[4]));
 480
 481     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (VOIDmode, operands[3]));
 482     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 483                                          operands[3]));
 484     DONE;
 485   }
 486 )
 487
 488 (define_insn "casesi_dispatch"
 489   [(parallel
 490     [(set (pc)
 491           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 492                            (match_operand:SI 1 "register_operand" "r")]
 493                         UNSPEC_CASESI)))
 494      (clobber (reg:CC CC_REGNUM))
 495      (clobber (match_scratch:DI 3 "=r"))
 496      (clobber (match_scratch:DI 4 "=r"))
 497      (use (label_ref (match_operand 2 "" "")))])]
 498   ""
 499   "*
 500   return aarch64_output_casesi (operands);
 501   "
 502   [(set_attr "length" "16")
 503    (set_attr "type" "branch")]
 504 )
 505
 506 (define_insn "nop"
 507   [(unspec[(const_int 0)] UNSPEC_NOP)]
 508   ""
 509   "nop"
 510   [(set_attr "type" "no_insn")]
 511 )
 512
 513 (define_insn "prefetch"
 514   [(prefetch (match_operand:DI 0 "register_operand" "r")
 515             (match_operand:QI 1 "const_int_operand" "")
 516             (match_operand:QI 2 "const_int_operand" ""))]
 517   ""
 518   {
 519     const char * pftype[2][4] =
 520     {
 521       {"prfm\\tPLDL1STRM, %a0",
 522        "prfm\\tPLDL3KEEP, %a0",
 523        "prfm\\tPLDL2KEEP, %a0",
 524        "prfm\\tPLDL1KEEP, %a0"},
 525       {"prfm\\tPSTL1STRM, %a0",
 526        "prfm\\tPSTL3KEEP, %a0",
 527        "prfm\\tPSTL2KEEP, %a0",
 528        "prfm\\tPSTL1KEEP, %a0"},
 529     };
 530
 531     int locality = INTVAL (operands[2]);
 532
 533     gcc_assert (IN_RANGE (locality, 0, 3));
 534
 535     return pftype[INTVAL(operands[1])][locality];
 536   }
 537   [(set_attr "type" "load1")]
 538 )
 539
 540 (define_insn "trap"
 541   [(trap_if (const_int 1) (const_int 8))]
 542   ""
 543   "brk #1000"
 544   [(set_attr "type" "trap")])
 545
 546 (define_expand "prologue"
 547   [(clobber (const_int 0))]
 548   ""
 549   "
 550   aarch64_expand_prologue ();
 551   DONE;
 552   "
 553 )
 554
 555 (define_expand "epilogue"
 556   [(clobber (const_int 0))]
 557   ""
 558   "
 559   aarch64_expand_epilogue (false);
 560   DONE;
 561   "
 562 )
 563
 564 (define_expand "sibcall_epilogue"
 565   [(clobber (const_int 0))]
 566   ""
 567   "
 568   aarch64_expand_epilogue (true);
 569   DONE;
 570   "
 571 )
 572
 573 (define_insn "*do_return"
 574   [(return)]
 575   ""
 576   "ret"
 577   [(set_attr "type" "branch")]
 578 )
 579
 580 (define_expand "return"
 581   [(simple_return)]
 582   "aarch64_use_return_insn_p ()"
 583   ""
 584 )
 585
 586 (define_insn "simple_return"
 587   [(simple_return)]
 588   ""
 589   "ret"
 590   [(set_attr "type" "branch")]
 591 )
 592
 593 (define_insn "eh_return"
 594   [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")]
 595     UNSPECV_EH_RETURN)]
 596   ""
 597   "#"
 598   [(set_attr "type" "branch")]
 599
 600 )
 601
 602 (define_split
 603   [(unspec_volatile [(match_operand:DI 0 "register_operand" "")]
 604     UNSPECV_EH_RETURN)]
 605   "reload_completed"
 606   [(set (match_dup 1) (match_dup 0))]
 607   {
 608     operands[1] = aarch64_final_eh_return_addr ();
 609   }
 610 )
 611
 612 (define_insn "*cb<optab><mode>1"
 613   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 614                                 (const_int 0))
 615                            (label_ref (match_operand 1 "" ""))
 616                            (pc)))]
 617   ""
 618   {
 619     if (get_attr_length (insn) == 8)
 620       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 621     else
 622       return "<cbz>\\t%<w>0, %l1";
 623   }
 624   [(set_attr "type" "branch")
 625    (set (attr "length")
 626         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 627                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 628                       (const_int 4)
 629                       (const_int 8)))
 630    (set (attr "far_branch")
 631         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 632                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 633                       (const_int 0)
 634                       (const_int 1)))]
 635 )
 636
 637 (define_insn "*tb<optab><mode>1"
 638   [(set (pc) (if_then_else
 639               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 640                                     (const_int 1)
 641                                     (match_operand 1 "const_int_operand" "n"))
 642                    (const_int 0))
 643              (label_ref (match_operand 2 "" ""))
 644              (pc)))
 645    (clobber (reg:CC CC_REGNUM))]
 646   ""
 647   {
 648     if (get_attr_length (insn) == 8)
 649       {
 650         if (get_attr_far_branch (insn) == 1)
 651           return aarch64_gen_far_branch (operands, 2, "Ltb",
 652                                          "<inv_tb>\\t%<w>0, %1, ");
 653         else
 654           {
 655             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 656             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 657           }
 658       }
 659     else
 660       return "<tbz>\t%<w>0, %1, %l2";
 661   }
 662   [(set_attr "type" "branch")
 663    (set (attr "length")
 664         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 665                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 666                       (const_int 4)
 667                       (const_int 8)))
 668    (set (attr "far_branch")
 669         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 670                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 671                       (const_int 0)
 672                       (const_int 1)))]
 673
 674 )
 675
 676 (define_insn "*cb<optab><mode>1"
 677   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 678                                  (const_int 0))
 679                            (label_ref (match_operand 1 "" ""))
 680                            (pc)))
 681    (clobber (reg:CC CC_REGNUM))]
 682   ""
 683   {
 684     if (get_attr_length (insn) == 8)
 685       {
 686         if (get_attr_far_branch (insn) == 1)
 687           return aarch64_gen_far_branch (operands, 1, "Ltb",
 688                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 689         else
 690           {
 691             char buf[64];
 692             uint64_t val = ((uint64_t) 1)
 693                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 694             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 695             output_asm_insn (buf, operands);
 696             return "<bcond>\t%l1";
 697           }
 698       }
 699     else
 700       return "<tbz>\t%<w>0, <sizem1>, %l1";
 701   }
 702   [(set_attr "type" "branch")
 703    (set (attr "length")
 704         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 705                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 706                       (const_int 4)
 707                       (const_int 8)))
 708    (set (attr "far_branch")
 709         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 710                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 711                       (const_int 0)
 712                       (const_int 1)))]
 713 )
 714
 715 ;; -------------------------------------------------------------------
 716 ;; Subroutine calls and sibcalls
 717 ;; -------------------------------------------------------------------
 718
 719 (define_expand "call_internal"
 720   [(parallel [(call (match_operand 0 "memory_operand" "")
 721                     (match_operand 1 "general_operand" ""))
 722               (use (match_operand 2 "" ""))
 723               (clobber (reg:DI LR_REGNUM))])])
 724
 725 (define_expand "call"
 726   [(parallel [(call (match_operand 0 "memory_operand" "")
 727                     (match_operand 1 "general_operand" ""))
 728               (use (match_operand 2 "" ""))
 729               (clobber (reg:DI LR_REGNUM))])]
 730   ""
 731   "
 732   {
 733     rtx callee, pat;
 734
 735     /* In an untyped call, we can get NULL for operand 2.  */
 736     if (operands[2] == NULL)
 737       operands[2] = const0_rtx;
 738
 739     /* Decide if we should generate indirect calls by loading the
 740        64-bit address of the callee into a register before performing
 741        the branch-and-link.  */
 742     callee = XEXP (operands[0], 0);
 743     if (GET_CODE (callee) == SYMBOL_REF
 744         ? (aarch64_is_long_call_p (callee)
 745            || aarch64_is_noplt_call_p (callee))
 746         : !REG_P (callee))
 747       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 748
 749     pat = gen_call_internal (operands[0], operands[1], operands[2]);
 750     aarch64_emit_call_insn (pat);
 751     DONE;
 752   }"
 753 )
 754
 755 (define_insn "*call_reg"
 756   [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
 757          (match_operand 1 "" ""))
 758    (use (match_operand 2 "" ""))
 759    (clobber (reg:DI LR_REGNUM))]
 760   ""
 761   "blr\\t%0"
 762   [(set_attr "type" "call")]
 763 )
 764
 765 (define_insn "*call_symbol"
 766   [(call (mem:DI (match_operand:DI 0 "" ""))
 767          (match_operand 1 "" ""))
 768    (use (match_operand 2 "" ""))
 769    (clobber (reg:DI LR_REGNUM))]
 770   "GET_CODE (operands[0]) == SYMBOL_REF
 771    && !aarch64_is_long_call_p (operands[0])
 772    && !aarch64_is_noplt_call_p (operands[0])"
 773   "bl\\t%a0"
 774   [(set_attr "type" "call")]
 775 )
 776
 777 (define_expand "call_value_internal"
 778   [(parallel [(set (match_operand 0 "" "")
 779                    (call (match_operand 1 "memory_operand" "")
 780                          (match_operand 2 "general_operand" "")))
 781               (use (match_operand 3 "" ""))
 782               (clobber (reg:DI LR_REGNUM))])])
 783
 784 (define_expand "call_value"
 785   [(parallel [(set (match_operand 0 "" "")
 786                    (call (match_operand 1 "memory_operand" "")
 787                          (match_operand 2 "general_operand" "")))
 788               (use (match_operand 3 "" ""))
 789               (clobber (reg:DI LR_REGNUM))])]
 790   ""
 791   "
 792   {
 793     rtx callee, pat;
 794
 795     /* In an untyped call, we can get NULL for operand 3.  */
 796     if (operands[3] == NULL)
 797       operands[3] = const0_rtx;
 798
 799     /* Decide if we should generate indirect calls by loading the
 800        64-bit address of the callee into a register before performing
 801        the branch-and-link.  */
 802     callee = XEXP (operands[1], 0);
 803     if (GET_CODE (callee) == SYMBOL_REF
 804         ? (aarch64_is_long_call_p (callee)
 805            || aarch64_is_noplt_call_p (callee))
 806         : !REG_P (callee))
 807       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 808
 809     pat = gen_call_value_internal (operands[0], operands[1], operands[2],
 810                                    operands[3]);
 811     aarch64_emit_call_insn (pat);
 812     DONE;
 813   }"
 814 )
 815
 816 (define_insn "*call_value_reg"
 817   [(set (match_operand 0 "" "")
 818         (call (mem:DI (match_operand:DI 1 "register_operand" "r"))
 819                       (match_operand 2 "" "")))
 820    (use (match_operand 3 "" ""))
 821    (clobber (reg:DI LR_REGNUM))]
 822   ""
 823   "blr\\t%1"
 824   [(set_attr "type" "call")]
 825
 826 )
 827
 828 (define_insn "*call_value_symbol"
 829   [(set (match_operand 0 "" "")
 830         (call (mem:DI (match_operand:DI 1 "" ""))
 831               (match_operand 2 "" "")))
 832    (use (match_operand 3 "" ""))
 833    (clobber (reg:DI LR_REGNUM))]
 834   "GET_CODE (operands[1]) == SYMBOL_REF
 835    && !aarch64_is_long_call_p (operands[1])
 836    && !aarch64_is_noplt_call_p (operands[1])"
 837   "bl\\t%a1"
 838   [(set_attr "type" "call")]
 839 )
 840
 841 (define_expand "sibcall_internal"
 842   [(parallel [(call (match_operand 0 "memory_operand" "")
 843                     (match_operand 1 "general_operand" ""))
 844               (return)
 845               (use (match_operand 2 "" ""))])])
 846
 847 (define_expand "sibcall"
 848   [(parallel [(call (match_operand 0 "memory_operand" "")
 849                     (match_operand 1 "general_operand" ""))
 850               (return)
 851               (use (match_operand 2 "" ""))])]
 852   ""
 853   {
 854     rtx pat;
 855     rtx callee = XEXP (operands[0], 0);
 856     if (!REG_P (callee)
 857        && ((GET_CODE (callee) != SYMBOL_REF)
 858            || aarch64_is_noplt_call_p (callee)))
 859       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 860
 861     if (operands[2] == NULL_RTX)
 862       operands[2] = const0_rtx;
 863
 864     pat = gen_sibcall_internal (operands[0], operands[1], operands[2]);
 865     aarch64_emit_call_insn (pat);
 866     DONE;
 867   }
 868 )
 869
 870 (define_expand "sibcall_value_internal"
 871   [(parallel [(set (match_operand 0 "" "")
 872                    (call (match_operand 1 "memory_operand" "")
 873                          (match_operand 2 "general_operand" "")))
 874               (return)
 875               (use (match_operand 3 "" ""))])])
 876
 877 (define_expand "sibcall_value"
 878   [(parallel [(set (match_operand 0 "" "")
 879                    (call (match_operand 1 "memory_operand" "")
 880                          (match_operand 2 "general_operand" "")))
 881               (return)
 882               (use (match_operand 3 "" ""))])]
 883   ""
 884   {
 885     rtx pat;
 886     rtx callee = XEXP (operands[1], 0);
 887     if (!REG_P (callee)
 888        && ((GET_CODE (callee) != SYMBOL_REF)
 889            || aarch64_is_noplt_call_p (callee)))
 890       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 891
 892     if (operands[3] == NULL_RTX)
 893       operands[3] = const0_rtx;
 894
 895     pat = gen_sibcall_value_internal (operands[0], operands[1], operands[2],
 896                                       operands[3]);
 897     aarch64_emit_call_insn (pat);
 898     DONE;
 899   }
 900 )
 901
 902 (define_insn "*sibcall_insn"
 903   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 904          (match_operand 1 "" ""))
 905    (return)
 906    (use (match_operand 2 "" ""))]
 907   "SIBLING_CALL_P (insn)"
 908   "@
 909    br\\t%0
 910    b\\t%a0"
 911   [(set_attr "type" "branch, branch")]
 912 )
 913
 914 (define_insn "*sibcall_value_insn"
 915   [(set (match_operand 0 "" "")
 916         (call (mem:DI
 917                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 918               (match_operand 2 "" "")))
 919    (return)
 920    (use (match_operand 3 "" ""))]
 921   "SIBLING_CALL_P (insn)"
 922   "@
 923    br\\t%1
 924    b\\t%a1"
 925   [(set_attr "type" "branch, branch")]
 926 )
 927
 928 ;; Call subroutine returning any type.
 929
 930 (define_expand "untyped_call"
 931   [(parallel [(call (match_operand 0 "")
 932                     (const_int 0))
 933               (match_operand 1 "")
 934               (match_operand 2 "")])]
 935   ""
 936 {
 937   int i;
 938
 939   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 940
 941   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 942     {
 943       rtx set = XVECEXP (operands[2], 0, i);
 944       emit_move_insn (SET_DEST (set), SET_SRC (set));
 945     }
 946
 947   /* The optimizer does not know that the call sets the function value
 948      registers we stored in the result block.  We avoid problems by
 949      claiming that all hard registers are used and clobbered at this
 950      point.  */
 951   emit_insn (gen_blockage ());
 952   DONE;
 953 })
 954
 955 ;; -------------------------------------------------------------------
 956 ;; Moves
 957 ;; -------------------------------------------------------------------
 958
 959 (define_expand "mov<mode>"
 960   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 961         (match_operand:SHORT 1 "general_operand" ""))]
 962   ""
 963   "
 964     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 965       operands[1] = force_reg (<MODE>mode, operands[1]);
 966   "
 967 )
 968
 969 (define_insn "*mov<mode>_aarch64"
 970   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,   *w,r,*w, m, m, r,*w,*w")
 971         (match_operand:SHORT 1 "general_operand"      " r,M,D<hq>,m, m,rZ,*w,*w, r,*w"))]
 972   "(register_operand (operands[0], <MODE>mode)
 973     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 974 {
 975    switch (which_alternative)
 976      {
 977      case 0:
 978        return "mov\t%w0, %w1";
 979      case 1:
 980        return "mov\t%w0, %1";
 981      case 2:
 982        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 983                                                         <MODE>mode);
 984      case 3:
 985        return "ldr<size>\t%w0, %1";
 986      case 4:
 987        return "ldr\t%<size>0, %1";
 988      case 5:
 989        return "str<size>\t%w1, %0";
 990      case 6:
 991        return "str\t%<size>1, %0";
 992      case 7:
 993        return "umov\t%w0, %1.<v>[0]";
 994      case 8:
 995        return "dup\t%0.<Vallxd>, %w1";
 996      case 9:
 997        return "dup\t%<Vetype>0, %1.<v>[0]";
 998      default:
 999        gcc_unreachable ();
1000      }
1001 }
1002   [(set_attr "type" "mov_reg,mov_imm,neon_move,load1,load1,store1,store1,\
1003                      neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1004    (set_attr "simd" "*,*,yes,*,*,*,*,yes,yes,yes")]
1005 )
1006
1007 (define_expand "mov<mode>"
1008   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1009         (match_operand:GPI 1 "general_operand" ""))]
1010   ""
1011   "
1012     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1013       operands[1] = force_reg (<MODE>mode, operands[1]);
1014
1015     /* FIXME: RR we still need to fix up what we are doing with
1016        symbol_refs and other types of constants.  */
1017     if (CONSTANT_P (operands[1])
1018         && !CONST_INT_P (operands[1]))
1019      {
1020        aarch64_expand_mov_immediate (operands[0], operands[1]);
1021        DONE;
1022      }
1023   "
1024 )
1025
1026 (define_insn_and_split "*movsi_aarch64"
1027   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r  ,*w, r,*w")
1028         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,m, m,rZ,*w,S,Ush,rZ,*w,*w"))]
1029   "(register_operand (operands[0], SImode)
1030     || aarch64_reg_or_zero (operands[1], SImode))"
1031   "@
1032    mov\\t%w0, %w1
1033    mov\\t%w0, %w1
1034    mov\\t%w0, %w1
1035    mov\\t%w0, %1
1036    #
1037    ldr\\t%w0, %1
1038    ldr\\t%s0, %1
1039    str\\t%w1, %0
1040    str\\t%s1, %0
1041    adr\\t%x0, %a1
1042    adrp\\t%x0, %A1
1043    fmov\\t%s0, %w1
1044    fmov\\t%w0, %s1
1045    fmov\\t%s0, %s1"
1046    "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1047     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1048    [(const_int 0)]
1049    "{
1050        aarch64_expand_mov_immediate (operands[0], operands[1]);
1051        DONE;
1052     }"
1053   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1054                      adr,adr,f_mcr,f_mrc,fmov")
1055    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes")]
1056 )
1057
1058 (define_insn_and_split "*movdi_aarch64"
1059   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r,  *w, r,*w,w")
1060         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,n,m, m,rZ,*w,S,Ush,rZ,*w,*w,Dd"))]
1061   "(register_operand (operands[0], DImode)
1062     || aarch64_reg_or_zero (operands[1], DImode))"
1063   "@
1064    mov\\t%x0, %x1
1065    mov\\t%0, %x1
1066    mov\\t%x0, %1
1067    mov\\t%x0, %1
1068    #
1069    ldr\\t%x0, %1
1070    ldr\\t%d0, %1
1071    str\\t%x1, %0
1072    str\\t%d1, %0
1073    adr\\t%x0, %a1
1074    adrp\\t%x0, %A1
1075    fmov\\t%d0, %x1
1076    fmov\\t%x0, %d1
1077    fmov\\t%d0, %d1
1078    movi\\t%d0, %1"
1079    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1080     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1081    [(const_int 0)]
1082    "{
1083        aarch64_expand_mov_immediate (operands[0], operands[1]);
1084        DONE;
1085     }"
1086   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1087                      adr,adr,f_mcr,f_mrc,fmov,neon_move")
1088    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1089    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")]
1090 )
1091
1092 (define_insn "insv_imm<mode>"
1093   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1094                           (const_int 16)
1095                           (match_operand:GPI 1 "const_int_operand" "n"))
1096         (match_operand:GPI 2 "const_int_operand" "n"))]
1097   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1098    && UINTVAL (operands[1]) % 16 == 0"
1099   "movk\\t%<w>0, %X2, lsl %1"
1100   [(set_attr "type" "mov_imm")]
1101 )
1102
1103 (define_expand "movti"
1104   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1105         (match_operand:TI 1 "general_operand" ""))]
1106   ""
1107   "
1108     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1109       operands[1] = force_reg (TImode, operands[1]);
1110   "
1111 )
1112
1113 (define_insn "*movti_aarch64"
1114   [(set (match_operand:TI 0
1115          "nonimmediate_operand"  "=r, *w,r ,*w,r  ,Ump,Ump,*w,m")
1116         (match_operand:TI 1
1117          "aarch64_movti_operand" " rn,r ,*w,*w,Ump,r  ,Z  , m,*w"))]
1118   "(register_operand (operands[0], TImode)
1119     || aarch64_reg_or_zero (operands[1], TImode))"
1120   "@
1121    #
1122    #
1123    #
1124    orr\\t%0.16b, %1.16b, %1.16b
1125    ldp\\t%0, %H0, %1
1126    stp\\t%1, %H1, %0
1127    stp\\txzr, xzr, %0
1128    ldr\\t%q0, %1
1129    str\\t%q1, %0"
1130   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1131                              load2,store2,store2,f_loadd,f_stored")
1132    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1133    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1134    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1135 )
1136
1137 ;; Split a TImode register-register or register-immediate move into
1138 ;; its component DImode pieces, taking care to handle overlapping
1139 ;; source and dest registers.
1140 (define_split
1141    [(set (match_operand:TI 0 "register_operand" "")
1142          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1143   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1144   [(const_int 0)]
1145 {
1146   aarch64_split_128bit_move (operands[0], operands[1]);
1147   DONE;
1148 })
1149
1150 (define_expand "mov<mode>"
1151   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1152         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1153   ""
1154   {
1155     if (!TARGET_FLOAT)
1156       {
1157         aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1158         FAIL;
1159       }
1160
1161     if (GET_CODE (operands[0]) == MEM
1162         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1163               && aarch64_float_const_zero_rtx_p (operands[1])))
1164       operands[1] = force_reg (<MODE>mode, operands[1]);
1165   }
1166 )
1167
1168 (define_insn "*movhf_aarch64"
1169   [(set (match_operand:HF 0 "nonimmediate_operand" "=w, ?r,w,w,m,r,m ,r")
1170         (match_operand:HF 1 "general_operand"      "?rY, w,w,m,w,m,rY,r"))]
1171   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1172     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1173   "@
1174    mov\\t%0.h[0], %w1
1175    umov\\t%w0, %1.h[0]
1176    mov\\t%0.h[0], %1.h[0]
1177    ldr\\t%h0, %1
1178    str\\t%h1, %0
1179    ldrh\\t%w0, %1
1180    strh\\t%w1, %0
1181    mov\\t%w0, %w1"
1182   [(set_attr "type" "neon_from_gp,neon_to_gp,neon_move,\
1183                      f_loads,f_stores,load1,store1,mov_reg")
1184    (set_attr "simd" "yes,yes,yes,*,*,*,*,*")
1185    (set_attr "fp"   "*,*,*,yes,yes,*,*,*")]
1186 )
1187
1188 (define_insn "*movsf_aarch64"
1189   [(set (match_operand:SF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
1190         (match_operand:SF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
1191   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1192     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1193   "@
1194    fmov\\t%s0, %w1
1195    fmov\\t%w0, %s1
1196    fmov\\t%s0, %s1
1197    fmov\\t%s0, %1
1198    ldr\\t%s0, %1
1199    str\\t%s1, %0
1200    ldr\\t%w0, %1
1201    str\\t%w1, %0
1202    mov\\t%w0, %w1"
1203   [(set_attr "type" "f_mcr,f_mrc,fmov,fconsts,\
1204                      f_loads,f_stores,load1,store1,mov_reg")]
1205 )
1206
1207 (define_insn "*movdf_aarch64"
1208   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
1209         (match_operand:DF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
1210   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1211     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1212   "@
1213    fmov\\t%d0, %x1
1214    fmov\\t%x0, %d1
1215    fmov\\t%d0, %d1
1216    fmov\\t%d0, %1
1217    ldr\\t%d0, %1
1218    str\\t%d1, %0
1219    ldr\\t%x0, %1
1220    str\\t%x1, %0
1221    mov\\t%x0, %x1"
1222   [(set_attr "type" "f_mcr,f_mrc,fmov,fconstd,\
1223                      f_loadd,f_stored,load1,store1,mov_reg")]
1224 )
1225
1226 (define_insn "*movtf_aarch64"
1227   [(set (match_operand:TF 0
1228          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r ,Ump,Ump")
1229         (match_operand:TF 1
1230          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,Ump,?r ,Y"))]
1231   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1232     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1233   "@
1234    orr\\t%0.16b, %1.16b, %1.16b
1235    #
1236    #
1237    #
1238    movi\\t%0.2d, #0
1239    fmov\\t%s0, wzr
1240    ldr\\t%q0, %1
1241    str\\t%q1, %0
1242    ldp\\t%0, %H0, %1
1243    stp\\t%1, %H1, %0
1244    stp\\txzr, xzr, %0"
1245   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1246                      f_loadd,f_stored,load2,store2,store2")
1247    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1248    (set_attr "fp" "*,*,yes,yes,*,yes,yes,yes,*,*,*")
1249    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1250 )
1251
1252 (define_split
1253    [(set (match_operand:TF 0 "register_operand" "")
1254          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1255   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1256   [(const_int 0)]
1257   {
1258     aarch64_split_128bit_move (operands[0], operands[1]);
1259     DONE;
1260   }
1261 )
1262
1263 ;; 0 is dst
1264 ;; 1 is src
1265 ;; 2 is size of move in bytes
1266 ;; 3 is alignment
1267
1268 (define_expand "movmemdi"
1269   [(match_operand:BLK 0 "memory_operand")
1270    (match_operand:BLK 1 "memory_operand")
1271    (match_operand:DI 2 "immediate_operand")
1272    (match_operand:DI 3 "immediate_operand")]
1273    "!STRICT_ALIGNMENT"
1274 {
1275   if (aarch64_expand_movmem (operands))
1276     DONE;
1277   FAIL;
1278 }
1279 )
1280
1281 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1282 ;; fairly lax checking on the second memory operation.
1283 (define_insn "load_pairsi"
1284   [(set (match_operand:SI 0 "register_operand" "=r,*w")
1285         (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1286    (set (match_operand:SI 2 "register_operand" "=r,*w")
1287         (match_operand:SI 3 "memory_operand" "m,m"))]
1288   "rtx_equal_p (XEXP (operands[3], 0),
1289                 plus_constant (Pmode,
1290                                XEXP (operands[1], 0),
1291                                GET_MODE_SIZE (SImode)))"
1292   "@
1293    ldp\\t%w0, %w2, %1
1294    ldp\\t%s0, %s2, %1"
1295   [(set_attr "type" "load2,neon_load1_2reg")
1296    (set_attr "fp" "*,yes")]
1297 )
1298
1299 (define_insn "load_pairdi"
1300   [(set (match_operand:DI 0 "register_operand" "=r,*w")
1301         (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1302    (set (match_operand:DI 2 "register_operand" "=r,*w")
1303         (match_operand:DI 3 "memory_operand" "m,m"))]
1304   "rtx_equal_p (XEXP (operands[3], 0),
1305                 plus_constant (Pmode,
1306                                XEXP (operands[1], 0),
1307                                GET_MODE_SIZE (DImode)))"
1308   "@
1309    ldp\\t%x0, %x2, %1
1310    ldp\\t%d0, %d2, %1"
1311   [(set_attr "type" "load2,neon_load1_2reg")
1312    (set_attr "fp" "*,yes")]
1313 )
1314
1315
1316 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1317 ;; fairly lax checking on the second memory operation.
1318 (define_insn "store_pairsi"
1319   [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1320         (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1321    (set (match_operand:SI 2 "memory_operand" "=m,m")
1322         (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1323   "rtx_equal_p (XEXP (operands[2], 0),
1324                 plus_constant (Pmode,
1325                                XEXP (operands[0], 0),
1326                                GET_MODE_SIZE (SImode)))"
1327   "@
1328    stp\\t%w1, %w3, %0
1329    stp\\t%s1, %s3, %0"
1330   [(set_attr "type" "store2,neon_store1_2reg")
1331    (set_attr "fp" "*,yes")]
1332 )
1333
1334 (define_insn "store_pairdi"
1335   [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1336         (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1337    (set (match_operand:DI 2 "memory_operand" "=m,m")
1338         (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1339   "rtx_equal_p (XEXP (operands[2], 0),
1340                 plus_constant (Pmode,
1341                                XEXP (operands[0], 0),
1342                                GET_MODE_SIZE (DImode)))"
1343   "@
1344    stp\\t%x1, %x3, %0
1345    stp\\t%d1, %d3, %0"
1346   [(set_attr "type" "store2,neon_store1_2reg")
1347    (set_attr "fp" "*,yes")]
1348 )
1349
1350 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1351 ;; fairly lax checking on the second memory operation.
1352 (define_insn "load_pairsf"
1353   [(set (match_operand:SF 0 "register_operand" "=w,*r")
1354         (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1355    (set (match_operand:SF 2 "register_operand" "=w,*r")
1356         (match_operand:SF 3 "memory_operand" "m,m"))]
1357   "rtx_equal_p (XEXP (operands[3], 0),
1358                 plus_constant (Pmode,
1359                                XEXP (operands[1], 0),
1360                                GET_MODE_SIZE (SFmode)))"
1361   "@
1362    ldp\\t%s0, %s2, %1
1363    ldp\\t%w0, %w2, %1"
1364   [(set_attr "type" "neon_load1_2reg,load2")
1365    (set_attr "fp" "yes,*")]
1366 )
1367
1368 (define_insn "load_pairdf"
1369   [(set (match_operand:DF 0 "register_operand" "=w,*r")
1370         (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1371    (set (match_operand:DF 2 "register_operand" "=w,*r")
1372         (match_operand:DF 3 "memory_operand" "m,m"))]
1373   "rtx_equal_p (XEXP (operands[3], 0),
1374                 plus_constant (Pmode,
1375                                XEXP (operands[1], 0),
1376                                GET_MODE_SIZE (DFmode)))"
1377   "@
1378    ldp\\t%d0, %d2, %1
1379    ldp\\t%x0, %x2, %1"
1380   [(set_attr "type" "neon_load1_2reg,load2")
1381    (set_attr "fp" "yes,*")]
1382 )
1383
1384 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1385 ;; fairly lax checking on the second memory operation.
1386 (define_insn "store_pairsf"
1387   [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1388         (match_operand:SF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1389    (set (match_operand:SF 2 "memory_operand" "=m,m")
1390         (match_operand:SF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1391   "rtx_equal_p (XEXP (operands[2], 0),
1392                 plus_constant (Pmode,
1393                                XEXP (operands[0], 0),
1394                                GET_MODE_SIZE (SFmode)))"
1395   "@
1396    stp\\t%s1, %s3, %0
1397    stp\\t%w1, %w3, %0"
1398   [(set_attr "type" "neon_store1_2reg,store2")
1399    (set_attr "fp" "yes,*")]
1400 )
1401
1402 (define_insn "store_pairdf"
1403   [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1404         (match_operand:DF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1405    (set (match_operand:DF 2 "memory_operand" "=m,m")
1406         (match_operand:DF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1407   "rtx_equal_p (XEXP (operands[2], 0),
1408                 plus_constant (Pmode,
1409                                XEXP (operands[0], 0),
1410                                GET_MODE_SIZE (DFmode)))"
1411   "@
1412    stp\\t%d1, %d3, %0
1413    stp\\t%x1, %x3, %0"
1414   [(set_attr "type" "neon_store1_2reg,store2")
1415    (set_attr "fp" "yes,*")]
1416 )
1417
1418 ;; Load pair with post-index writeback.  This is primarily used in function
1419 ;; epilogues.
1420 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1421   [(parallel
1422     [(set (match_operand:P 0 "register_operand" "=k")
1423           (plus:P (match_operand:P 1 "register_operand" "0")
1424                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1425      (set (match_operand:GPI 2 "register_operand" "=r")
1426           (mem:GPI (match_dup 1)))
1427      (set (match_operand:GPI 3 "register_operand" "=r")
1428           (mem:GPI (plus:P (match_dup 1)
1429                    (match_operand:P 5 "const_int_operand" "n"))))])]
1430   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1431   "ldp\\t%<w>2, %<w>3, [%1], %4"
1432   [(set_attr "type" "load2")]
1433 )
1434
1435 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1436   [(parallel
1437     [(set (match_operand:P 0 "register_operand" "=k")
1438           (plus:P (match_operand:P 1 "register_operand" "0")
1439                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1440      (set (match_operand:GPF 2 "register_operand" "=w")
1441           (mem:GPF (match_dup 1)))
1442      (set (match_operand:GPF 3 "register_operand" "=w")
1443           (mem:GPF (plus:P (match_dup 1)
1444                    (match_operand:P 5 "const_int_operand" "n"))))])]
1445   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1446   "ldp\\t%<w>2, %<w>3, [%1], %4"
1447   [(set_attr "type" "neon_load1_2reg")]
1448 )
1449
1450 ;; Store pair with pre-index writeback.  This is primarily used in function
1451 ;; prologues.
1452 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1453   [(parallel
1454     [(set (match_operand:P 0 "register_operand" "=&k")
1455           (plus:P (match_operand:P 1 "register_operand" "0")
1456                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1457      (set (mem:GPI (plus:P (match_dup 0)
1458                    (match_dup 4)))
1459           (match_operand:GPI 2 "register_operand" "r"))
1460      (set (mem:GPI (plus:P (match_dup 0)
1461                    (match_operand:P 5 "const_int_operand" "n")))
1462           (match_operand:GPI 3 "register_operand" "r"))])]
1463   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1464   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1465   [(set_attr "type" "store2")]
1466 )
1467
1468 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1469   [(parallel
1470     [(set (match_operand:P 0 "register_operand" "=&k")
1471           (plus:P (match_operand:P 1 "register_operand" "0")
1472                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1473      (set (mem:GPF (plus:P (match_dup 0)
1474                    (match_dup 4)))
1475           (match_operand:GPF 2 "register_operand" "w"))
1476      (set (mem:GPF (plus:P (match_dup 0)
1477                    (match_operand:P 5 "const_int_operand" "n")))
1478           (match_operand:GPF 3 "register_operand" "w"))])]
1479   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1480   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1481   [(set_attr "type" "neon_store1_2reg<q>")]
1482 )
1483
1484 ;; -------------------------------------------------------------------
1485 ;; Sign/Zero extension
1486 ;; -------------------------------------------------------------------
1487
1488 (define_expand "<optab>sidi2"
1489   [(set (match_operand:DI 0 "register_operand")
1490         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1491   ""
1492 )
1493
1494 (define_insn "*extendsidi2_aarch64"
1495   [(set (match_operand:DI 0 "register_operand" "=r,r")
1496         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1497   ""
1498   "@
1499    sxtw\t%0, %w1
1500    ldrsw\t%0, %1"
1501   [(set_attr "type" "extend,load1")]
1502 )
1503
1504 (define_insn "*load_pair_extendsidi2_aarch64"
1505   [(set (match_operand:DI 0 "register_operand" "=r")
1506         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1507    (set (match_operand:DI 2 "register_operand" "=r")
1508         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1509   "rtx_equal_p (XEXP (operands[3], 0),
1510                 plus_constant (Pmode,
1511                                XEXP (operands[1], 0),
1512                                GET_MODE_SIZE (SImode)))"
1513   "ldpsw\\t%0, %2, %1"
1514   [(set_attr "type" "load2")]
1515 )
1516
1517 (define_insn "*zero_extendsidi2_aarch64"
1518   [(set (match_operand:DI 0 "register_operand" "=r,r")
1519         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1520   ""
1521   "@
1522    uxtw\t%0, %w1
1523    ldr\t%w0, %1"
1524   [(set_attr "type" "extend,load1")]
1525 )
1526
1527 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1528   [(set (match_operand:DI 0 "register_operand" "=r")
1529         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1530    (set (match_operand:DI 2 "register_operand" "=r")
1531         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1532   "rtx_equal_p (XEXP (operands[3], 0),
1533                 plus_constant (Pmode,
1534                                XEXP (operands[1], 0),
1535                                GET_MODE_SIZE (SImode)))"
1536   "ldp\\t%w0, %w2, %1"
1537   [(set_attr "type" "load2")]
1538 )
1539
1540 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1541   [(set (match_operand:GPI 0 "register_operand")
1542         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1543   ""
1544 )
1545
1546 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1547   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1548         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1549   ""
1550   "@
1551    sxt<SHORT:size>\t%<GPI:w>0, %w1
1552    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1553   [(set_attr "type" "extend,load1")]
1554 )
1555
1556 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1557   [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1558         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1559   ""
1560   "@
1561    uxt<SHORT:size>\t%<GPI:w>0, %w1
1562    ldr<SHORT:size>\t%w0, %1
1563    ldr\t%<SHORT:size>0, %1"
1564   [(set_attr "type" "extend,load1,load1")]
1565 )
1566
1567 (define_expand "<optab>qihi2"
1568   [(set (match_operand:HI 0 "register_operand")
1569         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1570   ""
1571 )
1572
1573 (define_insn "*<optab>qihi2_aarch64"
1574   [(set (match_operand:HI 0 "register_operand" "=r,r")
1575         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1576   ""
1577   "@
1578    <su>xtb\t%w0, %w1
1579    <ldrxt>b\t%w0, %1"
1580   [(set_attr "type" "extend,load1")]
1581 )
1582
1583 ;; -------------------------------------------------------------------
1584 ;; Simple arithmetic
1585 ;; -------------------------------------------------------------------
1586
1587 (define_expand "add<mode>3"
1588   [(set
1589     (match_operand:GPI 0 "register_operand" "")
1590     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1591               (match_operand:GPI 2 "aarch64_pluslong_operand" "")))]
1592   ""
1593   "
1594   if (!aarch64_plus_operand (operands[2], VOIDmode))
1595     {
1596       if (can_create_pseudo_p ())
1597         {
1598           rtx tmp = gen_reg_rtx (<MODE>mode);
1599           emit_move_insn (tmp, operands[2]);
1600           operands[2] = tmp;
1601         }
1602       else
1603         {
1604           HOST_WIDE_INT imm = INTVAL (operands[2]);
1605           imm = imm >= 0 ? imm & 0xfff : -(-imm & 0xfff);
1606           emit_insn (gen_add<mode>3 (operands[0], operands[1],
1607                                      GEN_INT (INTVAL (operands[2]) - imm)));
1608           operands[1] = operands[0];
1609           operands[2] = GEN_INT (imm);
1610         }
1611     }
1612   "
1613 )
1614
1615 ;; Find add with a 2-instruction immediate and merge into 2 add instructions.
1616
1617 (define_insn_and_split "*add<mode>3_pluslong"
1618   [(set
1619     (match_operand:GPI 0 "register_operand" "=r")
1620     (plus:GPI (match_operand:GPI 1 "register_operand" "r")
1621               (match_operand:GPI 2 "aarch64_pluslong_immediate" "i")))]
1622   "!aarch64_plus_operand (operands[2], VOIDmode)
1623    && !aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1624   "#"
1625   "&& true"
1626   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1627    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1628   "
1629     {
1630       HOST_WIDE_INT imm = INTVAL (operands[2]);
1631       imm = imm >= 0 ? imm & 0xfff : -(-imm & 0xfff);
1632       operands[3] = GEN_INT (INTVAL (operands[2]) - imm);
1633       operands[4] = GEN_INT (imm);
1634     }
1635   "
1636 )
1637
1638 (define_insn "*addsi3_aarch64"
1639   [(set
1640     (match_operand:SI 0 "register_operand" "=rk,rk,w,rk")
1641     (plus:SI
1642      (match_operand:SI 1 "register_operand" "%rk,rk,w,rk")
1643      (match_operand:SI 2 "aarch64_plus_operand" "I,r,w,J")))]
1644   ""
1645   "@
1646   add\\t%w0, %w1, %2
1647   add\\t%w0, %w1, %w2
1648   add\\t%0.2s, %1.2s, %2.2s
1649   sub\\t%w0, %w1, #%n2"
1650   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm")
1651    (set_attr "simd" "*,*,yes,*")]
1652 )
1653
1654 ;; zero_extend version of above
1655 (define_insn "*addsi3_aarch64_uxtw"
1656   [(set
1657     (match_operand:DI 0 "register_operand" "=rk,rk,rk")
1658     (zero_extend:DI
1659      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
1660               (match_operand:SI 2 "aarch64_plus_operand" "I,r,J"))))]
1661   ""
1662   "@
1663   add\\t%w0, %w1, %2
1664   add\\t%w0, %w1, %w2
1665   sub\\t%w0, %w1, #%n2"
1666   [(set_attr "type" "alu_imm,alu_sreg,alu_imm")]
1667 )
1668
1669 (define_insn "*adddi3_aarch64"
1670   [(set
1671     (match_operand:DI 0 "register_operand" "=rk,rk,rk,w")
1672     (plus:DI
1673      (match_operand:DI 1 "register_operand" "%rk,rk,rk,w")
1674      (match_operand:DI 2 "aarch64_plus_operand" "I,r,J,w")))]
1675   ""
1676   "@
1677   add\\t%x0, %x1, %2
1678   add\\t%x0, %x1, %x2
1679   sub\\t%x0, %x1, #%n2
1680   add\\t%d0, %d1, %d2"
1681   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,neon_add")
1682    (set_attr "simd" "*,*,*,yes")]
1683 )
1684
1685 (define_expand "addti3"
1686   [(set (match_operand:TI 0 "register_operand" "")
1687         (plus:TI (match_operand:TI 1 "register_operand" "")
1688                  (match_operand:TI 2 "register_operand" "")))]
1689   ""
1690 {
1691   rtx low = gen_reg_rtx (DImode);
1692   emit_insn (gen_adddi3_compare0 (low, gen_lowpart (DImode, operands[1]),
1693                                   gen_lowpart (DImode, operands[2])));
1694
1695   rtx high = gen_reg_rtx (DImode);
1696   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1697                                  gen_highpart (DImode, operands[2])));
1698
1699   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1700   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1701   DONE;
1702 })
1703
1704 (define_insn "add<mode>3_compare0"
1705   [(set (reg:CC_NZ CC_REGNUM)
1706         (compare:CC_NZ
1707          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1708                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1709          (const_int 0)))
1710    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1711         (plus:GPI (match_dup 1) (match_dup 2)))]
1712   ""
1713   "@
1714   adds\\t%<w>0, %<w>1, %<w>2
1715   adds\\t%<w>0, %<w>1, %<w>2
1716   subs\\t%<w>0, %<w>1, #%n2"
1717   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1718 )
1719
1720 ;; zero_extend version of above
1721 (define_insn "*addsi3_compare0_uxtw"
1722   [(set (reg:CC_NZ CC_REGNUM)
1723         (compare:CC_NZ
1724          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1725                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1726          (const_int 0)))
1727    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1728         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1729   ""
1730   "@
1731   adds\\t%w0, %w1, %w2
1732   adds\\t%w0, %w1, %w2
1733   subs\\t%w0, %w1, #%n2"
1734   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1735 )
1736
1737 (define_insn "*adds_shift_imm_<mode>"
1738   [(set (reg:CC_NZ CC_REGNUM)
1739         (compare:CC_NZ
1740          (plus:GPI (ASHIFT:GPI
1741                     (match_operand:GPI 1 "register_operand" "r")
1742                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1743                    (match_operand:GPI 3 "register_operand" "r"))
1744          (const_int 0)))
1745    (set (match_operand:GPI 0 "register_operand" "=r")
1746         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1747                   (match_dup 3)))]
1748   ""
1749   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1750   [(set_attr "type" "alus_shift_imm")]
1751 )
1752
1753 (define_insn "*subs_shift_imm_<mode>"
1754   [(set (reg:CC_NZ CC_REGNUM)
1755         (compare:CC_NZ
1756          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1757                     (ASHIFT:GPI
1758                      (match_operand:GPI 2 "register_operand" "r")
1759                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1760          (const_int 0)))
1761    (set (match_operand:GPI 0 "register_operand" "=r")
1762         (minus:GPI (match_dup 1)
1763                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1764   ""
1765   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1766   [(set_attr "type" "alus_shift_imm")]
1767 )
1768
1769 (define_insn "*adds_mul_imm_<mode>"
1770   [(set (reg:CC_NZ CC_REGNUM)
1771         (compare:CC_NZ
1772          (plus:GPI (mult:GPI
1773                     (match_operand:GPI 1 "register_operand" "r")
1774                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1775                    (match_operand:GPI 3 "register_operand" "r"))
1776          (const_int 0)))
1777    (set (match_operand:GPI 0 "register_operand" "=r")
1778         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1779                   (match_dup 3)))]
1780   ""
1781   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1782   [(set_attr "type" "alus_shift_imm")]
1783 )
1784
1785 (define_insn "*subs_mul_imm_<mode>"
1786   [(set (reg:CC_NZ CC_REGNUM)
1787         (compare:CC_NZ
1788          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1789                     (mult:GPI
1790                      (match_operand:GPI 2 "register_operand" "r")
1791                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1792          (const_int 0)))
1793    (set (match_operand:GPI 0 "register_operand" "=r")
1794         (minus:GPI (match_dup 1)
1795                    (mult:GPI (match_dup 2) (match_dup 3))))]
1796   ""
1797   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1798   [(set_attr "type" "alus_shift_imm")]
1799 )
1800
1801 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1802   [(set (reg:CC_NZ CC_REGNUM)
1803         (compare:CC_NZ
1804          (plus:GPI
1805           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1806           (match_operand:GPI 2 "register_operand" "r"))
1807         (const_int 0)))
1808    (set (match_operand:GPI 0 "register_operand" "=r")
1809         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1810   ""
1811   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1812   [(set_attr "type" "alus_ext")]
1813 )
1814
1815 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1816   [(set (reg:CC_NZ CC_REGNUM)
1817         (compare:CC_NZ
1818          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1819                     (ANY_EXTEND:GPI
1820                      (match_operand:ALLX 2 "register_operand" "r")))
1821         (const_int 0)))
1822    (set (match_operand:GPI 0 "register_operand" "=r")
1823         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1824   ""
1825   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1826   [(set_attr "type" "alus_ext")]
1827 )
1828
1829 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1830   [(set (reg:CC_NZ CC_REGNUM)
1831         (compare:CC_NZ
1832          (plus:GPI (ashift:GPI
1833                     (ANY_EXTEND:GPI
1834                      (match_operand:ALLX 1 "register_operand" "r"))
1835                     (match_operand 2 "aarch64_imm3" "Ui3"))
1836                    (match_operand:GPI 3 "register_operand" "r"))
1837          (const_int 0)))
1838    (set (match_operand:GPI 0 "register_operand" "=rk")
1839         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1840                               (match_dup 2))
1841                   (match_dup 3)))]
1842   ""
1843   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1844   [(set_attr "type" "alus_ext")]
1845 )
1846
1847 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1848   [(set (reg:CC_NZ CC_REGNUM)
1849         (compare:CC_NZ
1850          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1851                     (ashift:GPI
1852                      (ANY_EXTEND:GPI
1853                       (match_operand:ALLX 2 "register_operand" "r"))
1854                      (match_operand 3 "aarch64_imm3" "Ui3")))
1855          (const_int 0)))
1856    (set (match_operand:GPI 0 "register_operand" "=rk")
1857         (minus:GPI (match_dup 1)
1858                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
1859                                (match_dup 3))))]
1860   ""
1861   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
1862   [(set_attr "type" "alus_ext")]
1863 )
1864
1865 (define_insn "*adds_<optab><mode>_multp2"
1866   [(set (reg:CC_NZ CC_REGNUM)
1867         (compare:CC_NZ
1868          (plus:GPI (ANY_EXTRACT:GPI
1869                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1870                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1871                     (match_operand 3 "const_int_operand" "n")
1872                     (const_int 0))
1873                    (match_operand:GPI 4 "register_operand" "r"))
1874         (const_int 0)))
1875    (set (match_operand:GPI 0 "register_operand" "=r")
1876         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
1877                                    (match_dup 3)
1878                                    (const_int 0))
1879                   (match_dup 4)))]
1880   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1881   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1882   [(set_attr "type" "alus_ext")]
1883 )
1884
1885 (define_insn "*subs_<optab><mode>_multp2"
1886   [(set (reg:CC_NZ CC_REGNUM)
1887         (compare:CC_NZ
1888          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
1889                     (ANY_EXTRACT:GPI
1890                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1891                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1892                      (match_operand 3 "const_int_operand" "n")
1893                      (const_int 0)))
1894         (const_int 0)))
1895    (set (match_operand:GPI 0 "register_operand" "=r")
1896         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
1897                                   (mult:GPI (match_dup 1) (match_dup 2))
1898                                   (match_dup 3)
1899                                   (const_int 0))))]
1900   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1901   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1902   [(set_attr "type" "alus_ext")]
1903 )
1904
1905 (define_insn "*add<mode>3nr_compare0"
1906   [(set (reg:CC_NZ CC_REGNUM)
1907         (compare:CC_NZ
1908          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
1909                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
1910          (const_int 0)))]
1911   ""
1912   "@
1913   cmn\\t%<w>0, %<w>1
1914   cmn\\t%<w>0, %<w>1
1915   cmp\\t%<w>0, #%n1"
1916   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1917 )
1918
1919 (define_insn "*compare_neg<mode>"
1920   [(set (reg:CC_Z CC_REGNUM)
1921         (compare:CC_Z
1922          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
1923          (match_operand:GPI 1 "register_operand" "r")))]
1924   ""
1925   "cmn\\t%<w>1, %<w>0"
1926   [(set_attr "type" "alus_sreg")]
1927 )
1928
1929 (define_insn "*add_<shift>_<mode>"
1930   [(set (match_operand:GPI 0 "register_operand" "=r")
1931         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
1932                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1933                   (match_operand:GPI 3 "register_operand" "r")))]
1934   ""
1935   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1936   [(set_attr "type" "alu_shift_imm")]
1937 )
1938
1939 ;; zero_extend version of above
1940 (define_insn "*add_<shift>_si_uxtw"
1941   [(set (match_operand:DI 0 "register_operand" "=r")
1942         (zero_extend:DI
1943          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
1944                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
1945                   (match_operand:SI 3 "register_operand" "r"))))]
1946   ""
1947   "add\\t%w0, %w3, %w1, <shift> %2"
1948   [(set_attr "type" "alu_shift_imm")]
1949 )
1950
1951 (define_insn "*add_mul_imm_<mode>"
1952   [(set (match_operand:GPI 0 "register_operand" "=r")
1953         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1954                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1955                   (match_operand:GPI 3 "register_operand" "r")))]
1956   ""
1957   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1958   [(set_attr "type" "alu_shift_imm")]
1959 )
1960
1961 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
1962   [(set (match_operand:GPI 0 "register_operand" "=rk")
1963         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1964                   (match_operand:GPI 2 "register_operand" "r")))]
1965   ""
1966   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1967   [(set_attr "type" "alu_ext")]
1968 )
1969
1970 ;; zero_extend version of above
1971 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
1972   [(set (match_operand:DI 0 "register_operand" "=rk")
1973         (zero_extend:DI
1974          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
1975                   (match_operand:GPI 2 "register_operand" "r"))))]
1976   ""
1977   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
1978   [(set_attr "type" "alu_ext")]
1979 )
1980
1981 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
1982   [(set (match_operand:GPI 0 "register_operand" "=rk")
1983         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
1984                                (match_operand:ALLX 1 "register_operand" "r"))
1985                               (match_operand 2 "aarch64_imm3" "Ui3"))
1986                   (match_operand:GPI 3 "register_operand" "r")))]
1987   ""
1988   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1989   [(set_attr "type" "alu_ext")]
1990 )
1991
1992 ;; zero_extend version of above
1993 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
1994   [(set (match_operand:DI 0 "register_operand" "=rk")
1995         (zero_extend:DI
1996          (plus:SI (ashift:SI (ANY_EXTEND:SI
1997                               (match_operand:SHORT 1 "register_operand" "r"))
1998                              (match_operand 2 "aarch64_imm3" "Ui3"))
1999                   (match_operand:SI 3 "register_operand" "r"))))]
2000   ""
2001   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2002   [(set_attr "type" "alu_ext")]
2003 )
2004
2005 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2006   [(set (match_operand:GPI 0 "register_operand" "=rk")
2007         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2008                              (match_operand:ALLX 1 "register_operand" "r"))
2009                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2010                   (match_operand:GPI 3 "register_operand" "r")))]
2011   ""
2012   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2013   [(set_attr "type" "alu_ext")]
2014 )
2015
2016 ;; zero_extend version of above
2017 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2018   [(set (match_operand:DI 0 "register_operand" "=rk")
2019         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2020                              (match_operand:SHORT 1 "register_operand" "r"))
2021                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2022                   (match_operand:SI 3 "register_operand" "r"))))]
2023   ""
2024   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2025   [(set_attr "type" "alu_ext")]
2026 )
2027
2028 (define_insn "*add_<optab><mode>_multp2"
2029   [(set (match_operand:GPI 0 "register_operand" "=rk")
2030         (plus:GPI (ANY_EXTRACT:GPI
2031                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2032                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2033                    (match_operand 3 "const_int_operand" "n")
2034                    (const_int 0))
2035                   (match_operand:GPI 4 "register_operand" "r")))]
2036   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2037   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2038   [(set_attr "type" "alu_ext")]
2039 )
2040
2041 ;; zero_extend version of above
2042 (define_insn "*add_<optab>si_multp2_uxtw"
2043   [(set (match_operand:DI 0 "register_operand" "=rk")
2044         (zero_extend:DI
2045          (plus:SI (ANY_EXTRACT:SI
2046                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2047                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2048                    (match_operand 3 "const_int_operand" "n")
2049                    (const_int 0))
2050                   (match_operand:SI 4 "register_operand" "r"))))]
2051   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2052   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2053   [(set_attr "type" "alu_ext")]
2054 )
2055
2056 (define_insn "add<mode>3_carryin"
2057   [(set
2058     (match_operand:GPI 0 "register_operand" "=r")
2059     (plus:GPI (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
2060               (plus:GPI
2061                 (match_operand:GPI 1 "register_operand" "r")
2062                 (match_operand:GPI 2 "register_operand" "r"))))]
2063    ""
2064    "adc\\t%<w>0, %<w>1, %<w>2"
2065   [(set_attr "type" "adc_reg")]
2066 )
2067
2068 ;; zero_extend version of above
2069 (define_insn "*addsi3_carryin_uxtw"
2070   [(set
2071     (match_operand:DI 0 "register_operand" "=r")
2072     (zero_extend:DI
2073      (plus:SI (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2074               (plus:SI
2075                (match_operand:SI 1 "register_operand" "r")
2076                (match_operand:SI 2 "register_operand" "r")))))]
2077    ""
2078    "adc\\t%w0, %w1, %w2"
2079   [(set_attr "type" "adc_reg")]
2080 )
2081
2082 (define_insn "*add<mode>3_carryin_alt1"
2083   [(set
2084     (match_operand:GPI 0 "register_operand" "=r")
2085     (plus:GPI (plus:GPI
2086                 (match_operand:GPI 1 "register_operand" "r")
2087                 (match_operand:GPI 2 "register_operand" "r"))
2088               (geu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2089    ""
2090    "adc\\t%<w>0, %<w>1, %<w>2"
2091   [(set_attr "type" "adc_reg")]
2092 )
2093
2094 ;; zero_extend version of above
2095 (define_insn "*addsi3_carryin_alt1_uxtw"
2096   [(set
2097     (match_operand:DI 0 "register_operand" "=r")
2098     (zero_extend:DI
2099      (plus:SI (plus:SI
2100                (match_operand:SI 1 "register_operand" "r")
2101                (match_operand:SI 2 "register_operand" "r"))
2102               (geu:SI (reg:CC CC_REGNUM) (const_int 0)))))]
2103    ""
2104    "adc\\t%w0, %w1, %w2"
2105   [(set_attr "type" "adc_reg")]
2106 )
2107
2108 (define_insn "*add<mode>3_carryin_alt2"
2109   [(set
2110     (match_operand:GPI 0 "register_operand" "=r")
2111     (plus:GPI (plus:GPI
2112                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
2113                 (match_operand:GPI 1 "register_operand" "r"))
2114               (match_operand:GPI 2 "register_operand" "r")))]
2115    ""
2116    "adc\\t%<w>0, %<w>1, %<w>2"
2117   [(set_attr "type" "adc_reg")]
2118 )
2119
2120 ;; zero_extend version of above
2121 (define_insn "*addsi3_carryin_alt2_uxtw"
2122   [(set
2123     (match_operand:DI 0 "register_operand" "=r")
2124     (zero_extend:DI
2125      (plus:SI (plus:SI
2126                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2127                (match_operand:SI 1 "register_operand" "r"))
2128               (match_operand:SI 2 "register_operand" "r"))))]
2129    ""
2130    "adc\\t%w0, %w1, %w2"
2131   [(set_attr "type" "adc_reg")]
2132 )
2133
2134 (define_insn "*add<mode>3_carryin_alt3"
2135   [(set
2136     (match_operand:GPI 0 "register_operand" "=r")
2137     (plus:GPI (plus:GPI
2138                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
2139                 (match_operand:GPI 2 "register_operand" "r"))
2140               (match_operand:GPI 1 "register_operand" "r")))]
2141    ""
2142    "adc\\t%<w>0, %<w>1, %<w>2"
2143   [(set_attr "type" "adc_reg")]
2144 )
2145
2146 ;; zero_extend version of above
2147 (define_insn "*addsi3_carryin_alt3_uxtw"
2148   [(set
2149     (match_operand:DI 0 "register_operand" "=r")
2150     (zero_extend:DI
2151      (plus:SI (plus:SI
2152                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2153                (match_operand:SI 2 "register_operand" "r"))
2154               (match_operand:SI 1 "register_operand" "r"))))]
2155    ""
2156    "adc\\t%w0, %w1, %w2"
2157   [(set_attr "type" "adc_reg")]
2158 )
2159
2160 (define_insn "*add_uxt<mode>_shift2"
2161   [(set (match_operand:GPI 0 "register_operand" "=rk")
2162         (plus:GPI (and:GPI
2163                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2164                                (match_operand 2 "aarch64_imm3" "Ui3"))
2165                    (match_operand 3 "const_int_operand" "n"))
2166                   (match_operand:GPI 4 "register_operand" "r")))]
2167   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2168   "*
2169   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2170                                            INTVAL (operands[3])));
2171   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2172   [(set_attr "type" "alu_ext")]
2173 )
2174
2175 ;; zero_extend version of above
2176 (define_insn "*add_uxtsi_shift2_uxtw"
2177   [(set (match_operand:DI 0 "register_operand" "=rk")
2178         (zero_extend:DI
2179          (plus:SI (and:SI
2180                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2181                               (match_operand 2 "aarch64_imm3" "Ui3"))
2182                    (match_operand 3 "const_int_operand" "n"))
2183                   (match_operand:SI 4 "register_operand" "r"))))]
2184   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2185   "*
2186   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2187                                            INTVAL (operands[3])));
2188   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2189   [(set_attr "type" "alu_ext")]
2190 )
2191
2192 (define_insn "*add_uxt<mode>_multp2"
2193   [(set (match_operand:GPI 0 "register_operand" "=rk")
2194         (plus:GPI (and:GPI
2195                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2196                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2197                    (match_operand 3 "const_int_operand" "n"))
2198                   (match_operand:GPI 4 "register_operand" "r")))]
2199   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2200   "*
2201   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2202                                            INTVAL (operands[3])));
2203   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2204   [(set_attr "type" "alu_ext")]
2205 )
2206
2207 ;; zero_extend version of above
2208 (define_insn "*add_uxtsi_multp2_uxtw"
2209   [(set (match_operand:DI 0 "register_operand" "=rk")
2210         (zero_extend:DI
2211          (plus:SI (and:SI
2212                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2213                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2214                    (match_operand 3 "const_int_operand" "n"))
2215                   (match_operand:SI 4 "register_operand" "r"))))]
2216   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2217   "*
2218   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2219                                            INTVAL (operands[3])));
2220   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2221   [(set_attr "type" "alu_ext")]
2222 )
2223
2224 (define_insn "subsi3"
2225   [(set (match_operand:SI 0 "register_operand" "=rk")
2226         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2227                   (match_operand:SI 2 "register_operand" "r")))]
2228   ""
2229   "sub\\t%w0, %w1, %w2"
2230   [(set_attr "type" "alu_sreg")]
2231 )
2232
2233 ;; zero_extend version of above
2234 (define_insn "*subsi3_uxtw"
2235   [(set (match_operand:DI 0 "register_operand" "=rk")
2236         (zero_extend:DI
2237          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2238                    (match_operand:SI 2 "register_operand" "r"))))]
2239   ""
2240   "sub\\t%w0, %w1, %w2"
2241   [(set_attr "type" "alu_sreg")]
2242 )
2243
2244 (define_insn "subdi3"
2245   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2246         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2247                   (match_operand:DI 2 "register_operand" "r,w")))]
2248   ""
2249   "@
2250    sub\\t%x0, %x1, %x2
2251    sub\\t%d0, %d1, %d2"
2252   [(set_attr "type" "alu_sreg, neon_sub")
2253    (set_attr "simd" "*,yes")]
2254 )
2255
2256 (define_expand "subti3"
2257   [(set (match_operand:TI 0 "register_operand" "")
2258         (minus:TI (match_operand:TI 1 "register_operand" "")
2259                   (match_operand:TI 2 "register_operand" "")))]
2260   ""
2261 {
2262   rtx low = gen_reg_rtx (DImode);
2263   emit_insn (gen_subdi3_compare0 (low, gen_lowpart (DImode, operands[1]),
2264                                   gen_lowpart (DImode, operands[2])));
2265
2266   rtx high = gen_reg_rtx (DImode);
2267   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2268                                  gen_highpart (DImode, operands[2])));
2269
2270   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2271   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2272   DONE;
2273 })
2274
2275 (define_insn "sub<mode>3_compare0"
2276   [(set (reg:CC_NZ CC_REGNUM)
2277         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2278                                   (match_operand:GPI 2 "register_operand" "r"))
2279                        (const_int 0)))
2280    (set (match_operand:GPI 0 "register_operand" "=r")
2281         (minus:GPI (match_dup 1) (match_dup 2)))]
2282   ""
2283   "subs\\t%<w>0, %<w>1, %<w>2"
2284   [(set_attr "type" "alus_sreg")]
2285 )
2286
2287 ;; zero_extend version of above
2288 (define_insn "*subsi3_compare0_uxtw"
2289   [(set (reg:CC_NZ CC_REGNUM)
2290         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2291                                  (match_operand:SI 2 "register_operand" "r"))
2292                        (const_int 0)))
2293    (set (match_operand:DI 0 "register_operand" "=r")
2294         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2295   ""
2296   "subs\\t%w0, %w1, %w2"
2297   [(set_attr "type" "alus_sreg")]
2298 )
2299
2300 (define_insn "*sub_<shift>_<mode>"
2301   [(set (match_operand:GPI 0 "register_operand" "=r")
2302         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2303                    (ASHIFT:GPI
2304                     (match_operand:GPI 1 "register_operand" "r")
2305                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2306   ""
2307   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2308   [(set_attr "type" "alu_shift_imm")]
2309 )
2310
2311 ;; zero_extend version of above
2312 (define_insn "*sub_<shift>_si_uxtw"
2313   [(set (match_operand:DI 0 "register_operand" "=r")
2314         (zero_extend:DI
2315          (minus:SI (match_operand:SI 3 "register_operand" "r")
2316                    (ASHIFT:SI
2317                     (match_operand:SI 1 "register_operand" "r")
2318                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2319   ""
2320   "sub\\t%w0, %w3, %w1, <shift> %2"
2321   [(set_attr "type" "alu_shift_imm")]
2322 )
2323
2324 (define_insn "*sub_mul_imm_<mode>"
2325   [(set (match_operand:GPI 0 "register_operand" "=r")
2326         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2327                    (mult:GPI
2328                     (match_operand:GPI 1 "register_operand" "r")
2329                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2330   ""
2331   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2332   [(set_attr "type" "alu_shift_imm")]
2333 )
2334
2335 ;; zero_extend version of above
2336 (define_insn "*sub_mul_imm_si_uxtw"
2337   [(set (match_operand:DI 0 "register_operand" "=r")
2338         (zero_extend:DI
2339          (minus:SI (match_operand:SI 3 "register_operand" "r")
2340                    (mult:SI
2341                     (match_operand:SI 1 "register_operand" "r")
2342                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2343   ""
2344   "sub\\t%w0, %w3, %w1, lsl %p2"
2345   [(set_attr "type" "alu_shift_imm")]
2346 )
2347
2348 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2349   [(set (match_operand:GPI 0 "register_operand" "=rk")
2350         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2351                    (ANY_EXTEND:GPI
2352                     (match_operand:ALLX 2 "register_operand" "r"))))]
2353   ""
2354   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2355   [(set_attr "type" "alu_ext")]
2356 )
2357
2358 ;; zero_extend version of above
2359 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2360   [(set (match_operand:DI 0 "register_operand" "=rk")
2361         (zero_extend:DI
2362          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2363                    (ANY_EXTEND:SI
2364                     (match_operand:SHORT 2 "register_operand" "r")))))]
2365   ""
2366   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2367   [(set_attr "type" "alu_ext")]
2368 )
2369
2370 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2371   [(set (match_operand:GPI 0 "register_operand" "=rk")
2372         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2373                    (ashift:GPI (ANY_EXTEND:GPI
2374                                 (match_operand:ALLX 2 "register_operand" "r"))
2375                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2376   ""
2377   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2378   [(set_attr "type" "alu_ext")]
2379 )
2380
2381 ;; zero_extend version of above
2382 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2383   [(set (match_operand:DI 0 "register_operand" "=rk")
2384         (zero_extend:DI
2385          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2386                    (ashift:SI (ANY_EXTEND:SI
2387                                (match_operand:SHORT 2 "register_operand" "r"))
2388                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2389   ""
2390   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2391   [(set_attr "type" "alu_ext")]
2392 )
2393
2394 (define_insn "*sub_<optab><mode>_multp2"
2395   [(set (match_operand:GPI 0 "register_operand" "=rk")
2396         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2397                    (ANY_EXTRACT:GPI
2398                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2399                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2400                     (match_operand 3 "const_int_operand" "n")
2401                     (const_int 0))))]
2402   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2403   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2404   [(set_attr "type" "alu_ext")]
2405 )
2406
2407 ;; zero_extend version of above
2408 (define_insn "*sub_<optab>si_multp2_uxtw"
2409   [(set (match_operand:DI 0 "register_operand" "=rk")
2410         (zero_extend:DI
2411          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2412                    (ANY_EXTRACT:SI
2413                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2414                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2415                     (match_operand 3 "const_int_operand" "n")
2416                     (const_int 0)))))]
2417   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2418   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2419   [(set_attr "type" "alu_ext")]
2420 )
2421
2422 (define_insn "sub<mode>3_carryin"
2423   [(set
2424     (match_operand:GPI 0 "register_operand" "=r")
2425     (minus:GPI (minus:GPI
2426                 (match_operand:GPI 1 "register_operand" "r")
2427                 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2428                (match_operand:GPI 2 "register_operand" "r")))]
2429    ""
2430    "sbc\\t%<w>0, %<w>1, %<w>2"
2431   [(set_attr "type" "adc_reg")]
2432 )
2433
2434 ;; zero_extend version of the above
2435 (define_insn "*subsi3_carryin_uxtw"
2436   [(set
2437     (match_operand:DI 0 "register_operand" "=r")
2438     (zero_extend:DI
2439      (minus:SI (minus:SI
2440                 (match_operand:SI 1 "register_operand" "r")
2441                 (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2442                (match_operand:SI 2 "register_operand" "r"))))]
2443    ""
2444    "sbc\\t%w0, %w1, %w2"
2445   [(set_attr "type" "adc_reg")]
2446 )
2447
2448 (define_insn "*sub_uxt<mode>_shift2"
2449   [(set (match_operand:GPI 0 "register_operand" "=rk")
2450         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2451                    (and:GPI
2452                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2453                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2454                     (match_operand 3 "const_int_operand" "n"))))]
2455   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2456   "*
2457   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2458                                            INTVAL (operands[3])));
2459   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2460   [(set_attr "type" "alu_ext")]
2461 )
2462
2463 ;; zero_extend version of above
2464 (define_insn "*sub_uxtsi_shift2_uxtw"
2465   [(set (match_operand:DI 0 "register_operand" "=rk")
2466         (zero_extend:DI
2467          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2468                    (and:SI
2469                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2470                                (match_operand 2 "aarch64_imm3" "Ui3"))
2471                     (match_operand 3 "const_int_operand" "n")))))]
2472   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2473   "*
2474   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2475                                            INTVAL (operands[3])));
2476   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2477   [(set_attr "type" "alu_ext")]
2478 )
2479
2480 (define_insn "*sub_uxt<mode>_multp2"
2481   [(set (match_operand:GPI 0 "register_operand" "=rk")
2482         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2483                    (and:GPI
2484                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2485                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2486                     (match_operand 3 "const_int_operand" "n"))))]
2487   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2488   "*
2489   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2490                                            INTVAL (operands[3])));
2491   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2492   [(set_attr "type" "alu_ext")]
2493 )
2494
2495 ;; zero_extend version of above
2496 (define_insn "*sub_uxtsi_multp2_uxtw"
2497   [(set (match_operand:DI 0 "register_operand" "=rk")
2498         (zero_extend:DI
2499          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2500                    (and:SI
2501                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2502                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2503                     (match_operand 3 "const_int_operand" "n")))))]
2504   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2505   "*
2506   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2507                                            INTVAL (operands[3])));
2508   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2509   [(set_attr "type" "alu_ext")]
2510 )
2511
2512 (define_expand "abs<mode>2"
2513   [(match_operand:GPI 0 "register_operand" "")
2514    (match_operand:GPI 1 "register_operand" "")]
2515   ""
2516   {
2517     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2518     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2519     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2520     DONE;
2521   }
2522 )
2523
2524 (define_insn "neg<mode>2"
2525   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2526         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2527   ""
2528   "@
2529    neg\\t%<w>0, %<w>1
2530    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2531   [(set_attr "type" "alu_sreg, neon_neg<q>")
2532    (set_attr "simd" "*,yes")]
2533 )
2534
2535 ;; zero_extend version of above
2536 (define_insn "*negsi2_uxtw"
2537   [(set (match_operand:DI 0 "register_operand" "=r")
2538         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2539   ""
2540   "neg\\t%w0, %w1"
2541   [(set_attr "type" "alu_sreg")]
2542 )
2543
2544 (define_insn "*ngc<mode>"
2545   [(set (match_operand:GPI 0 "register_operand" "=r")
2546         (minus:GPI (neg:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2547                    (match_operand:GPI 1 "register_operand" "r")))]
2548   ""
2549   "ngc\\t%<w>0, %<w>1"
2550   [(set_attr "type" "adc_reg")]
2551 )
2552
2553 (define_insn "*ngcsi_uxtw"
2554   [(set (match_operand:DI 0 "register_operand" "=r")
2555         (zero_extend:DI
2556          (minus:SI (neg:SI (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2557                    (match_operand:SI 1 "register_operand" "r"))))]
2558   ""
2559   "ngc\\t%w0, %w1"
2560   [(set_attr "type" "adc_reg")]
2561 )
2562
2563 (define_insn "neg<mode>2_compare0"
2564   [(set (reg:CC_NZ CC_REGNUM)
2565         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2566                        (const_int 0)))
2567    (set (match_operand:GPI 0 "register_operand" "=r")
2568         (neg:GPI (match_dup 1)))]
2569   ""
2570   "negs\\t%<w>0, %<w>1"
2571   [(set_attr "type" "alus_sreg")]
2572 )
2573
2574 ;; zero_extend version of above
2575 (define_insn "*negsi2_compare0_uxtw"
2576   [(set (reg:CC_NZ CC_REGNUM)
2577         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2578                        (const_int 0)))
2579    (set (match_operand:DI 0 "register_operand" "=r")
2580         (zero_extend:DI (neg:SI (match_dup 1))))]
2581   ""
2582   "negs\\t%w0, %w1"
2583   [(set_attr "type" "alus_sreg")]
2584 )
2585
2586 (define_insn "*neg_<shift><mode>3_compare0"
2587   [(set (reg:CC_NZ CC_REGNUM)
2588         (compare:CC_NZ
2589          (neg:GPI (ASHIFT:GPI
2590                    (match_operand:GPI 1 "register_operand" "r")
2591                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2592          (const_int 0)))
2593    (set (match_operand:GPI 0 "register_operand" "=r")
2594         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2595   ""
2596   "negs\\t%<w>0, %<w>1, <shift> %2"
2597   [(set_attr "type" "alus_shift_imm")]
2598 )
2599
2600 (define_insn "*neg_<shift>_<mode>2"
2601   [(set (match_operand:GPI 0 "register_operand" "=r")
2602         (neg:GPI (ASHIFT:GPI
2603                   (match_operand:GPI 1 "register_operand" "r")
2604                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2605   ""
2606   "neg\\t%<w>0, %<w>1, <shift> %2"
2607   [(set_attr "type" "alu_shift_imm")]
2608 )
2609
2610 ;; zero_extend version of above
2611 (define_insn "*neg_<shift>_si2_uxtw"
2612   [(set (match_operand:DI 0 "register_operand" "=r")
2613         (zero_extend:DI
2614          (neg:SI (ASHIFT:SI
2615                   (match_operand:SI 1 "register_operand" "r")
2616                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2617   ""
2618   "neg\\t%w0, %w1, <shift> %2"
2619   [(set_attr "type" "alu_shift_imm")]
2620 )
2621
2622 (define_insn "*neg_mul_imm_<mode>2"
2623   [(set (match_operand:GPI 0 "register_operand" "=r")
2624         (neg:GPI (mult:GPI
2625                   (match_operand:GPI 1 "register_operand" "r")
2626                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2627   ""
2628   "neg\\t%<w>0, %<w>1, lsl %p2"
2629   [(set_attr "type" "alu_shift_imm")]
2630 )
2631
2632 ;; zero_extend version of above
2633 (define_insn "*neg_mul_imm_si2_uxtw"
2634   [(set (match_operand:DI 0 "register_operand" "=r")
2635         (zero_extend:DI
2636          (neg:SI (mult:SI
2637                   (match_operand:SI 1 "register_operand" "r")
2638                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2639   ""
2640   "neg\\t%w0, %w1, lsl %p2"
2641   [(set_attr "type" "alu_shift_imm")]
2642 )
2643
2644 (define_insn "mul<mode>3"
2645   [(set (match_operand:GPI 0 "register_operand" "=r")
2646         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2647                   (match_operand:GPI 2 "register_operand" "r")))]
2648   ""
2649   "mul\\t%<w>0, %<w>1, %<w>2"
2650   [(set_attr "type" "mul")]
2651 )
2652
2653 ;; zero_extend version of above
2654 (define_insn "*mulsi3_uxtw"
2655   [(set (match_operand:DI 0 "register_operand" "=r")
2656         (zero_extend:DI
2657          (mult:SI (match_operand:SI 1 "register_operand" "r")
2658                   (match_operand:SI 2 "register_operand" "r"))))]
2659   ""
2660   "mul\\t%w0, %w1, %w2"
2661   [(set_attr "type" "mul")]
2662 )
2663
2664 (define_insn "madd<mode>"
2665   [(set (match_operand:GPI 0 "register_operand" "=r")
2666         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2667                             (match_operand:GPI 2 "register_operand" "r"))
2668                   (match_operand:GPI 3 "register_operand" "r")))]
2669   ""
2670   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2671   [(set_attr "type" "mla")]
2672 )
2673
2674 ;; zero_extend version of above
2675 (define_insn "*maddsi_uxtw"
2676   [(set (match_operand:DI 0 "register_operand" "=r")
2677         (zero_extend:DI
2678          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2679                            (match_operand:SI 2 "register_operand" "r"))
2680                   (match_operand:SI 3 "register_operand" "r"))))]
2681   ""
2682   "madd\\t%w0, %w1, %w2, %w3"
2683   [(set_attr "type" "mla")]
2684 )
2685
2686 (define_insn "*msub<mode>"
2687   [(set (match_operand:GPI 0 "register_operand" "=r")
2688         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2689                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2690                              (match_operand:GPI 2 "register_operand" "r"))))]
2691
2692   ""
2693   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2694   [(set_attr "type" "mla")]
2695 )
2696
2697 ;; zero_extend version of above
2698 (define_insn "*msubsi_uxtw"
2699   [(set (match_operand:DI 0 "register_operand" "=r")
2700         (zero_extend:DI
2701          (minus:SI (match_operand:SI 3 "register_operand" "r")
2702                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2703                             (match_operand:SI 2 "register_operand" "r")))))]
2704
2705   ""
2706   "msub\\t%w0, %w1, %w2, %w3"
2707   [(set_attr "type" "mla")]
2708 )
2709
2710 (define_insn "*mul<mode>_neg"
2711   [(set (match_operand:GPI 0 "register_operand" "=r")
2712         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2713                   (match_operand:GPI 2 "register_operand" "r")))]
2714
2715   ""
2716   "mneg\\t%<w>0, %<w>1, %<w>2"
2717   [(set_attr "type" "mul")]
2718 )
2719
2720 ;; zero_extend version of above
2721 (define_insn "*mulsi_neg_uxtw"
2722   [(set (match_operand:DI 0 "register_operand" "=r")
2723         (zero_extend:DI
2724          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2725                   (match_operand:SI 2 "register_operand" "r"))))]
2726
2727   ""
2728   "mneg\\t%w0, %w1, %w2"
2729   [(set_attr "type" "mul")]
2730 )
2731
2732 (define_insn "<su_optab>mulsidi3"
2733   [(set (match_operand:DI 0 "register_operand" "=r")
2734         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2735                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2736   ""
2737   "<su>mull\\t%0, %w1, %w2"
2738   [(set_attr "type" "<su>mull")]
2739 )
2740
2741 (define_insn "<su_optab>maddsidi4"
2742   [(set (match_operand:DI 0 "register_operand" "=r")
2743         (plus:DI (mult:DI
2744                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2745                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
2746                  (match_operand:DI 3 "register_operand" "r")))]
2747   ""
2748   "<su>maddl\\t%0, %w1, %w2, %3"
2749   [(set_attr "type" "<su>mlal")]
2750 )
2751
2752 (define_insn "<su_optab>msubsidi4"
2753   [(set (match_operand:DI 0 "register_operand" "=r")
2754         (minus:DI
2755          (match_operand:DI 3 "register_operand" "r")
2756          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2757                   (ANY_EXTEND:DI
2758                    (match_operand:SI 2 "register_operand" "r")))))]
2759   ""
2760   "<su>msubl\\t%0, %w1, %w2, %3"
2761   [(set_attr "type" "<su>mlal")]
2762 )
2763
2764 (define_insn "*<su_optab>mulsidi_neg"
2765   [(set (match_operand:DI 0 "register_operand" "=r")
2766         (mult:DI (neg:DI
2767                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
2768                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2769   ""
2770   "<su>mnegl\\t%0, %w1, %w2"
2771   [(set_attr "type" "<su>mull")]
2772 )
2773
2774 (define_expand "<su_optab>mulditi3"
2775   [(set (match_operand:TI 0 "register_operand")
2776         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
2777                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
2778   ""
2779 {
2780   rtx low = gen_reg_rtx (DImode);
2781   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
2782
2783   rtx high = gen_reg_rtx (DImode);
2784   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
2785
2786   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2787   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2788   DONE;
2789 })
2790
2791 ;; The default expansion of multi3 using umuldi3_highpart will perform
2792 ;; the additions in an order that fails to combine into two madd insns.
2793 (define_expand "multi3"
2794   [(set (match_operand:TI 0 "register_operand")
2795         (mult:TI (match_operand:TI 1 "register_operand")
2796                  (match_operand:TI 2 "register_operand")))]
2797   ""
2798 {
2799   rtx l0 = gen_reg_rtx (DImode);
2800   rtx l1 = gen_lowpart (DImode, operands[1]);
2801   rtx l2 = gen_lowpart (DImode, operands[2]);
2802   rtx h0 = gen_reg_rtx (DImode);
2803   rtx h1 = gen_highpart (DImode, operands[1]);
2804   rtx h2 = gen_highpart (DImode, operands[2]);
2805
2806   emit_insn (gen_muldi3 (l0, l1, l2));
2807   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
2808   emit_insn (gen_madddi (h0, h1, l2, h0));
2809   emit_insn (gen_madddi (h0, l1, h2, h0));
2810
2811   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
2812   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
2813   DONE;
2814 })
2815
2816 (define_insn "<su>muldi3_highpart"
2817   [(set (match_operand:DI 0 "register_operand" "=r")
2818         (truncate:DI
2819          (lshiftrt:TI
2820           (mult:TI
2821            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
2822            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
2823           (const_int 64))))]
2824   ""
2825   "<su>mulh\\t%0, %1, %2"
2826   [(set_attr "type" "<su>mull")]
2827 )
2828
2829 (define_insn "<su_optab>div<mode>3"
2830   [(set (match_operand:GPI 0 "register_operand" "=r")
2831         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
2832                      (match_operand:GPI 2 "register_operand" "r")))]
2833   ""
2834   "<su>div\\t%<w>0, %<w>1, %<w>2"
2835   [(set_attr "type" "<su>div")]
2836 )
2837
2838 ;; zero_extend version of above
2839 (define_insn "*<su_optab>divsi3_uxtw"
2840   [(set (match_operand:DI 0 "register_operand" "=r")
2841         (zero_extend:DI
2842          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
2843                      (match_operand:SI 2 "register_operand" "r"))))]
2844   ""
2845   "<su>div\\t%w0, %w1, %w2"
2846   [(set_attr "type" "<su>div")]
2847 )
2848
2849 ;; -------------------------------------------------------------------
2850 ;; Comparison insns
2851 ;; -------------------------------------------------------------------
2852
2853 (define_insn "*cmp<mode>"
2854   [(set (reg:CC CC_REGNUM)
2855         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
2856                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
2857   ""
2858   "@
2859    cmp\\t%<w>0, %<w>1
2860    cmp\\t%<w>0, %<w>1
2861    cmn\\t%<w>0, #%n1"
2862   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2863 )
2864
2865 (define_insn "*cmp<mode>"
2866   [(set (reg:CCFP CC_REGNUM)
2867         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
2868                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2869    "TARGET_FLOAT"
2870    "@
2871     fcmp\\t%<s>0, #0.0
2872     fcmp\\t%<s>0, %<s>1"
2873   [(set_attr "type" "fcmp<s>")]
2874 )
2875
2876 (define_insn "*cmpe<mode>"
2877   [(set (reg:CCFPE CC_REGNUM)
2878         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
2879                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2880    "TARGET_FLOAT"
2881    "@
2882     fcmpe\\t%<s>0, #0.0
2883     fcmpe\\t%<s>0, %<s>1"
2884   [(set_attr "type" "fcmp<s>")]
2885 )
2886
2887 (define_insn "*cmp_swp_<shift>_reg<mode>"
2888   [(set (reg:CC_SWP CC_REGNUM)
2889         (compare:CC_SWP (ASHIFT:GPI
2890                          (match_operand:GPI 0 "register_operand" "r")
2891                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
2892                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
2893   ""
2894   "cmp\\t%<w>2, %<w>0, <shift> %1"
2895   [(set_attr "type" "alus_shift_imm")]
2896 )
2897
2898 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
2899   [(set (reg:CC_SWP CC_REGNUM)
2900         (compare:CC_SWP (ANY_EXTEND:GPI
2901                          (match_operand:ALLX 0 "register_operand" "r"))
2902                         (match_operand:GPI 1 "register_operand" "r")))]
2903   ""
2904   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
2905   [(set_attr "type" "alus_ext")]
2906 )
2907
2908 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
2909   [(set (reg:CC_SWP CC_REGNUM)
2910         (compare:CC_SWP (ashift:GPI
2911                          (ANY_EXTEND:GPI
2912                           (match_operand:ALLX 0 "register_operand" "r"))
2913                          (match_operand 1 "aarch64_imm3" "Ui3"))
2914         (match_operand:GPI 2 "register_operand" "r")))]
2915   ""
2916   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
2917   [(set_attr "type" "alus_ext")]
2918 )
2919
2920 ;; -------------------------------------------------------------------
2921 ;; Store-flag and conditional select insns
2922 ;; -------------------------------------------------------------------
2923
2924 (define_expand "cstore<mode>4"
2925   [(set (match_operand:SI 0 "register_operand" "")
2926         (match_operator:SI 1 "aarch64_comparison_operator"
2927          [(match_operand:GPI 2 "register_operand" "")
2928           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
2929   ""
2930   "
2931   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2932                                       operands[3]);
2933   operands[3] = const0_rtx;
2934   "
2935 )
2936
2937 (define_expand "cstorecc4"
2938   [(set (match_operand:SI 0 "register_operand")
2939        (match_operator 1 "aarch64_comparison_operator"
2940         [(match_operand 2 "ccmp_cc_register")
2941          (match_operand 3 "const0_operand")]))]
2942   ""
2943 "{
2944   emit_insn (gen_rtx_SET (operands[0], operands[1]));
2945   DONE;
2946 }")
2947
2948
2949 (define_expand "cstore<mode>4"
2950   [(set (match_operand:SI 0 "register_operand" "")
2951         (match_operator:SI 1 "aarch64_comparison_operator"
2952          [(match_operand:GPF 2 "register_operand" "")
2953           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
2954   ""
2955   "
2956   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2957                                       operands[3]);
2958   operands[3] = const0_rtx;
2959   "
2960 )
2961
2962 (define_insn "aarch64_cstore<mode>"
2963   [(set (match_operand:ALLI 0 "register_operand" "=r")
2964         (match_operator:ALLI 1 "aarch64_comparison_operator"
2965          [(match_operand 2 "cc_register" "") (const_int 0)]))]
2966   ""
2967   "cset\\t%<w>0, %m1"
2968   [(set_attr "type" "csel")]
2969 )
2970
2971 ;; For a 24-bit immediate CST we can optimize the compare for equality
2972 ;; and branch sequence from:
2973 ;;      mov     x0, #imm1
2974 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
2975 ;;      cmp     x1, x0
2976 ;;      cset    x2, <ne,eq>
2977 ;; into the shorter:
2978 ;;      sub     x0, x1, #(CST & 0xfff000)
2979 ;;      subs    x0, x0, #(CST & 0x000fff)
2980 ;;      cset x2, <ne, eq>.
2981 (define_insn_and_split "*compare_cstore<mode>_insn"
2982   [(set (match_operand:GPI 0 "register_operand" "=r")
2983          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
2984                   (match_operand:GPI 2 "aarch64_imm24" "n")))]
2985   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
2986    && !aarch64_plus_operand (operands[2], <MODE>mode)
2987    && !reload_completed"
2988   "#"
2989   "&& true"
2990   [(const_int 0)]
2991   {
2992     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
2993     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
2994     rtx tmp = gen_reg_rtx (<MODE>mode);
2995     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
2996     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
2997     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
2998     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
2999     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3000     DONE;
3001   }
3002   [(set_attr "type" "csel")]
3003 )
3004
3005 ;; zero_extend version of the above
3006 (define_insn "*cstoresi_insn_uxtw"
3007   [(set (match_operand:DI 0 "register_operand" "=r")
3008         (zero_extend:DI
3009          (match_operator:SI 1 "aarch64_comparison_operator"
3010           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3011   ""
3012   "cset\\t%w0, %m1"
3013   [(set_attr "type" "csel")]
3014 )
3015
3016 (define_insn "cstore<mode>_neg"
3017   [(set (match_operand:ALLI 0 "register_operand" "=r")
3018         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator"
3019                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3020   ""
3021   "csetm\\t%<w>0, %m1"
3022   [(set_attr "type" "csel")]
3023 )
3024
3025 ;; zero_extend version of the above
3026 (define_insn "*cstoresi_neg_uxtw"
3027   [(set (match_operand:DI 0 "register_operand" "=r")
3028         (zero_extend:DI
3029          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator"
3030                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3031   ""
3032   "csetm\\t%w0, %m1"
3033   [(set_attr "type" "csel")]
3034 )
3035
3036 (define_expand "cmov<mode>6"
3037   [(set (match_operand:GPI 0 "register_operand" "")
3038         (if_then_else:GPI
3039          (match_operator 1 "aarch64_comparison_operator"
3040           [(match_operand:GPI 2 "register_operand" "")
3041            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3042          (match_operand:GPI 4 "register_operand" "")
3043          (match_operand:GPI 5 "register_operand" "")))]
3044   ""
3045   "
3046   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3047                                       operands[3]);
3048   operands[3] = const0_rtx;
3049   "
3050 )
3051
3052 (define_expand "cmov<mode>6"
3053   [(set (match_operand:GPF 0 "register_operand" "")
3054         (if_then_else:GPF
3055          (match_operator 1 "aarch64_comparison_operator"
3056           [(match_operand:GPF 2 "register_operand" "")
3057            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3058          (match_operand:GPF 4 "register_operand" "")
3059          (match_operand:GPF 5 "register_operand" "")))]
3060   ""
3061   "
3062   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3063                                       operands[3]);
3064   operands[3] = const0_rtx;
3065   "
3066 )
3067
3068 (define_insn "*cmov<mode>_insn"
3069   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3070         (if_then_else:ALLI
3071          (match_operator 1 "aarch64_comparison_operator"
3072           [(match_operand 2 "cc_register" "") (const_int 0)])
3073          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3074          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3075   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3076      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3077   ;; Final two alternatives should be unreachable, but included for completeness
3078   "@
3079    csel\\t%<w>0, %<w>3, %<w>4, %m1
3080    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3081    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3082    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3083    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3084    mov\\t%<w>0, -1
3085    mov\\t%<w>0, 1"
3086   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3087 )
3088
3089 ;; zero_extend version of above
3090 (define_insn "*cmovsi_insn_uxtw"
3091   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3092         (zero_extend:DI
3093          (if_then_else:SI
3094           (match_operator 1 "aarch64_comparison_operator"
3095            [(match_operand 2 "cc_register" "") (const_int 0)])
3096           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3097           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3098   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3099      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3100   ;; Final two alternatives should be unreachable, but included for completeness
3101   "@
3102    csel\\t%w0, %w3, %w4, %m1
3103    csinv\\t%w0, %w3, wzr, %m1
3104    csinv\\t%w0, %w4, wzr, %M1
3105    csinc\\t%w0, %w3, wzr, %m1
3106    csinc\\t%w0, %w4, wzr, %M1
3107    mov\\t%w0, -1
3108    mov\\t%w0, 1"
3109   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3110 )
3111
3112 (define_insn "*cmovdi_insn_uxtw"
3113   [(set (match_operand:DI 0 "register_operand" "=r")
3114         (if_then_else:DI
3115          (match_operator 1 "aarch64_comparison_operator"
3116           [(match_operand 2 "cc_register" "") (const_int 0)])
3117          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3118          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3119   ""
3120   "csel\\t%w0, %w3, %w4, %m1"
3121   [(set_attr "type" "csel")]
3122 )
3123
3124 (define_insn "*cmov<mode>_insn"
3125   [(set (match_operand:GPF 0 "register_operand" "=w")
3126         (if_then_else:GPF
3127          (match_operator 1 "aarch64_comparison_operator"
3128           [(match_operand 2 "cc_register" "") (const_int 0)])
3129          (match_operand:GPF 3 "register_operand" "w")
3130          (match_operand:GPF 4 "register_operand" "w")))]
3131   "TARGET_FLOAT"
3132   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3133   [(set_attr "type" "fcsel")]
3134 )
3135
3136 (define_expand "mov<mode>cc"
3137   [(set (match_operand:ALLI 0 "register_operand" "")
3138         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3139                            (match_operand:ALLI 2 "register_operand" "")
3140                            (match_operand:ALLI 3 "register_operand" "")))]
3141   ""
3142   {
3143     enum rtx_code code = GET_CODE (operands[1]);
3144
3145     if (code == UNEQ || code == LTGT)
3146       FAIL;
3147
3148     if (!ccmp_cc_register (XEXP (operands[1], 0),
3149                            GET_MODE (XEXP (operands[1], 0))))
3150       {
3151         rtx ccreg;
3152         ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3153                                          XEXP (operands[1], 1));
3154         operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3155       }
3156   }
3157 )
3158
3159 (define_expand "mov<GPF:mode><GPI:mode>cc"
3160   [(set (match_operand:GPI 0 "register_operand" "")
3161         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3162                           (match_operand:GPF 2 "register_operand" "")
3163                           (match_operand:GPF 3 "register_operand" "")))]
3164   ""
3165   {
3166     rtx ccreg;
3167     enum rtx_code code = GET_CODE (operands[1]);
3168
3169     if (code == UNEQ || code == LTGT)
3170       FAIL;
3171
3172     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3173                                   XEXP (operands[1], 1));
3174     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3175   }
3176 )
3177
3178 (define_expand "mov<mode>cc"
3179   [(set (match_operand:GPF 0 "register_operand" "")
3180         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3181                           (match_operand:GPF 2 "register_operand" "")
3182                           (match_operand:GPF 3 "register_operand" "")))]
3183   ""
3184   {
3185     rtx ccreg;
3186     enum rtx_code code = GET_CODE (operands[1]);
3187
3188     if (code == UNEQ || code == LTGT)
3189       FAIL;
3190
3191     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3192                                   XEXP (operands[1], 1));
3193     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3194   }
3195 )
3196
3197 (define_expand "<neg_not_op><mode>cc"
3198   [(set (match_operand:GPI 0 "register_operand" "")
3199         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3200                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3201                           (match_operand:GPI 3 "register_operand" "")))]
3202   ""
3203   {
3204     rtx ccreg;
3205     enum rtx_code code = GET_CODE (operands[1]);
3206
3207     if (code == UNEQ || code == LTGT)
3208       FAIL;
3209
3210     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3211                                       XEXP (operands[1], 1));
3212     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3213   }
3214 )
3215
3216 ;; CRC32 instructions.
3217 (define_insn "aarch64_<crc_variant>"
3218   [(set (match_operand:SI 0 "register_operand" "=r")
3219         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3220                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3221          CRC))]
3222   "TARGET_CRC32"
3223   {
3224     if (GET_MODE_BITSIZE (GET_MODE (operands[2])) >= 64)
3225       return "<crc_variant>\\t%w0, %w1, %x2";
3226     else
3227       return "<crc_variant>\\t%w0, %w1, %w2";
3228   }
3229   [(set_attr "type" "crc")]
3230 )
3231
3232 (define_insn "*csinc2<mode>_insn"
3233   [(set (match_operand:GPI 0 "register_operand" "=r")
3234         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3235                   (match_operand:GPI 1 "register_operand" "r")))]
3236   ""
3237   "cinc\\t%<w>0, %<w>1, %m2"
3238   [(set_attr "type" "csel")]
3239 )
3240
3241 (define_insn "csinc3<mode>_insn"
3242   [(set (match_operand:GPI 0 "register_operand" "=r")
3243         (if_then_else:GPI
3244           (match_operand 1 "aarch64_comparison_operation" "")
3245           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3246                     (const_int 1))
3247           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3248   ""
3249   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3250   [(set_attr "type" "csel")]
3251 )
3252
3253 (define_insn "*csinv3<mode>_insn"
3254   [(set (match_operand:GPI 0 "register_operand" "=r")
3255         (if_then_else:GPI
3256           (match_operand 1 "aarch64_comparison_operation" "")
3257           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3258           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3259   ""
3260   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3261   [(set_attr "type" "csel")]
3262 )
3263
3264 (define_insn "csneg3_uxtw_insn"
3265   [(set (match_operand:DI 0 "register_operand" "=r")
3266         (zero_extend:DI
3267           (if_then_else:SI
3268             (match_operand 1 "aarch64_comparison_operation" "")
3269             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3270             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3271   ""
3272   "csneg\\t%w0, %w3, %w2, %M1"
3273   [(set_attr "type" "csel")]
3274 )
3275
3276 (define_insn "csneg3<mode>_insn"
3277   [(set (match_operand:GPI 0 "register_operand" "=r")
3278         (if_then_else:GPI
3279           (match_operand 1 "aarch64_comparison_operation" "")
3280           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3281           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3282   ""
3283   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3284   [(set_attr "type" "csel")]
3285 )
3286
3287 ;; -------------------------------------------------------------------
3288 ;; Logical operations
3289 ;; -------------------------------------------------------------------
3290
3291 (define_insn "<optab><mode>3"
3292   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3293         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3294                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3295   ""
3296   "@
3297   <logical>\\t%<w>0, %<w>1, %<w>2
3298   <logical>\\t%<w>0, %<w>1, %<w>2
3299   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3300   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3301    (set_attr "simd" "*,*,yes")]
3302 )
3303
3304 ;; zero_extend version of above
3305 (define_insn "*<optab>si3_uxtw"
3306   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3307         (zero_extend:DI
3308          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3309                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3310   ""
3311   "<logical>\\t%w0, %w1, %w2"
3312   [(set_attr "type" "logic_reg,logic_imm")]
3313 )
3314
3315 (define_insn "*and<mode>3_compare0"
3316   [(set (reg:CC_NZ CC_REGNUM)
3317         (compare:CC_NZ
3318          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3319                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3320          (const_int 0)))
3321    (set (match_operand:GPI 0 "register_operand" "=r,r")
3322         (and:GPI (match_dup 1) (match_dup 2)))]
3323   ""
3324   "ands\\t%<w>0, %<w>1, %<w>2"
3325   [(set_attr "type" "logics_reg,logics_imm")]
3326 )
3327
3328 ;; zero_extend version of above
3329 (define_insn "*andsi3_compare0_uxtw"
3330   [(set (reg:CC_NZ CC_REGNUM)
3331         (compare:CC_NZ
3332          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3333                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3334          (const_int 0)))
3335    (set (match_operand:DI 0 "register_operand" "=r,r")
3336         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3337   ""
3338   "ands\\t%w0, %w1, %w2"
3339   [(set_attr "type" "logics_reg,logics_imm")]
3340 )
3341
3342 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3343   [(set (reg:CC_NZ CC_REGNUM)
3344         (compare:CC_NZ
3345          (and:GPI (SHIFT:GPI
3346                    (match_operand:GPI 1 "register_operand" "r")
3347                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3348                   (match_operand:GPI 3 "register_operand" "r"))
3349          (const_int 0)))
3350    (set (match_operand:GPI 0 "register_operand" "=r")
3351         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3352   ""
3353   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3354   [(set_attr "type" "logics_shift_imm")]
3355 )
3356
3357 ;; zero_extend version of above
3358 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3359   [(set (reg:CC_NZ CC_REGNUM)
3360         (compare:CC_NZ
3361          (and:SI (SHIFT:SI
3362                   (match_operand:SI 1 "register_operand" "r")
3363                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3364                  (match_operand:SI 3 "register_operand" "r"))
3365          (const_int 0)))
3366    (set (match_operand:DI 0 "register_operand" "=r")
3367         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3368                                 (match_dup 3))))]
3369   ""
3370   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3371   [(set_attr "type" "logics_shift_imm")]
3372 )
3373
3374 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3375   [(set (match_operand:GPI 0 "register_operand" "=r")
3376         (LOGICAL:GPI (SHIFT:GPI
3377                       (match_operand:GPI 1 "register_operand" "r")
3378                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3379                      (match_operand:GPI 3 "register_operand" "r")))]
3380   ""
3381   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3382   [(set_attr "type" "logic_shift_imm")]
3383 )
3384
3385 (define_insn "*<optab>_rol<mode>3"
3386   [(set (match_operand:GPI 0 "register_operand" "=r")
3387         (LOGICAL:GPI (rotate:GPI
3388                       (match_operand:GPI 1 "register_operand" "r")
3389                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3390                      (match_operand:GPI 3 "register_operand" "r")))]
3391   ""
3392   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3393   [(set_attr "type" "logic_shift_imm")]
3394 )
3395
3396 ;; zero_extend versions of above
3397 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3398   [(set (match_operand:DI 0 "register_operand" "=r")
3399         (zero_extend:DI
3400          (LOGICAL:SI (SHIFT:SI
3401                       (match_operand:SI 1 "register_operand" "r")
3402                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3403                      (match_operand:SI 3 "register_operand" "r"))))]
3404   ""
3405   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3406   [(set_attr "type" "logic_shift_imm")]
3407 )
3408
3409 (define_insn "*<optab>_rolsi3_uxtw"
3410   [(set (match_operand:DI 0 "register_operand" "=r")
3411         (zero_extend:DI
3412          (LOGICAL:SI (rotate:SI
3413                       (match_operand:SI 1 "register_operand" "r")
3414                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3415                      (match_operand:SI 3 "register_operand" "r"))))]
3416   ""
3417   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3418   [(set_attr "type" "logic_shift_imm")]
3419 )
3420
3421 (define_insn "one_cmpl<mode>2"
3422   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3423         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3424   ""
3425   "@
3426   mvn\\t%<w>0, %<w>1
3427   mvn\\t%0.8b, %1.8b"
3428   [(set_attr "type" "logic_reg,neon_logic")
3429    (set_attr "simd" "*,yes")]
3430 )
3431
3432 (define_insn "*one_cmpl_<optab><mode>2"
3433   [(set (match_operand:GPI 0 "register_operand" "=r")
3434         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3435                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3436   ""
3437   "mvn\\t%<w>0, %<w>1, <shift> %2"
3438   [(set_attr "type" "logic_shift_imm")]
3439 )
3440
3441 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3442
3443 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3444   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3445         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3446                      (match_operand:GPI 2 "register_operand" "r,w")))]
3447   ""
3448   "@
3449   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3450   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3451   [(set_attr "type" "logic_reg,neon_logic")
3452    (set_attr "simd" "*,yes")]
3453 )
3454
3455 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3456   [(set (match_operand:DI 0 "register_operand" "=r")
3457         (zero_extend:DI
3458           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3459                        (match_operand:SI 2 "register_operand" "r"))))]
3460   ""
3461   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3462   [(set_attr "type" "logic_reg")]
3463 )
3464
3465 (define_insn "*xor_one_cmplsidi3_ze"
3466   [(set (match_operand:DI 0 "register_operand" "=r")
3467         (zero_extend:DI
3468           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3469                           (match_operand:SI 2 "register_operand" "r")))))]
3470   ""
3471   "eon\\t%w0, %w1, %w2"
3472   [(set_attr "type" "logic_reg")]
3473 )
3474
3475 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3476 ;; eon does not operate on SIMD registers so the vector variant must be split.
3477 (define_insn_and_split "*xor_one_cmpl<mode>3"
3478   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3479         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3480                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3481   ""
3482   "@
3483   eon\\t%<w>0, %<w>1, %<w>2
3484   #"
3485   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3486   [(set (match_operand:GPI 0 "register_operand" "=w")
3487         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3488                  (match_operand:GPI 2 "register_operand" "w")))
3489    (set (match_dup 0) (not:GPI (match_dup 0)))]
3490   ""
3491   [(set_attr "type" "logic_reg,multiple")
3492    (set_attr "simd" "*,yes")]
3493 )
3494
3495 (define_insn "*and_one_cmpl<mode>3_compare0"
3496   [(set (reg:CC_NZ CC_REGNUM)
3497         (compare:CC_NZ
3498          (and:GPI (not:GPI
3499                    (match_operand:GPI 1 "register_operand" "r"))
3500                   (match_operand:GPI 2 "register_operand" "r"))
3501          (const_int 0)))
3502    (set (match_operand:GPI 0 "register_operand" "=r")
3503         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3504   ""
3505   "bics\\t%<w>0, %<w>2, %<w>1"
3506   [(set_attr "type" "logics_reg")]
3507 )
3508
3509 ;; zero_extend version of above
3510 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3511   [(set (reg:CC_NZ CC_REGNUM)
3512         (compare:CC_NZ
3513          (and:SI (not:SI
3514                   (match_operand:SI 1 "register_operand" "r"))
3515                  (match_operand:SI 2 "register_operand" "r"))
3516          (const_int 0)))
3517    (set (match_operand:DI 0 "register_operand" "=r")
3518         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3519   ""
3520   "bics\\t%w0, %w2, %w1"
3521   [(set_attr "type" "logics_reg")]
3522 )
3523
3524 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3525   [(set (reg:CC_NZ CC_REGNUM)
3526     (compare:CC_NZ
3527      (and:GPI (not:GPI
3528            (match_operand:GPI 0 "register_operand" "r"))
3529           (match_operand:GPI 1 "register_operand" "r"))
3530      (const_int 0)))]
3531   ""
3532   "bics\\t<w>zr, %<w>1, %<w>0"
3533   [(set_attr "type" "logics_reg")]
3534 )
3535
3536 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3537   [(set (match_operand:GPI 0 "register_operand" "=r")
3538         (LOGICAL:GPI (not:GPI
3539                       (SHIFT:GPI
3540                        (match_operand:GPI 1 "register_operand" "r")
3541                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3542                      (match_operand:GPI 3 "register_operand" "r")))]
3543   ""
3544   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3545   [(set_attr "type" "logic_shift_imm")]
3546 )
3547
3548 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3549   [(set (match_operand:GPI 0 "register_operand" "=r")
3550         (not:GPI (xor:GPI
3551                       (SHIFT:GPI
3552                        (match_operand:GPI 1 "register_operand" "r")
3553                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3554                      (match_operand:GPI 3 "register_operand" "r"))))]
3555   ""
3556   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3557   [(set_attr "type" "logic_shift_imm")]
3558 )
3559
3560 ;; Zero-extend version of the above.
3561 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3562   [(set (match_operand:DI 0 "register_operand" "=r")
3563         (zero_extend:DI
3564           (not:SI (xor:SI
3565                     (SHIFT:SI
3566                       (match_operand:SI 1 "register_operand" "r")
3567                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3568                     (match_operand:SI 3 "register_operand" "r")))))]
3569   ""
3570   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3571   [(set_attr "type" "logic_shift_imm")]
3572 )
3573
3574 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3575   [(set (reg:CC_NZ CC_REGNUM)
3576         (compare:CC_NZ
3577          (and:GPI (not:GPI
3578                    (SHIFT:GPI
3579                     (match_operand:GPI 1 "register_operand" "r")
3580                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3581                   (match_operand:GPI 3 "register_operand" "r"))
3582          (const_int 0)))
3583    (set (match_operand:GPI 0 "register_operand" "=r")
3584         (and:GPI (not:GPI
3585                   (SHIFT:GPI
3586                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3587   ""
3588   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3589   [(set_attr "type" "logics_shift_imm")]
3590 )
3591
3592 ;; zero_extend version of above
3593 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3594   [(set (reg:CC_NZ CC_REGNUM)
3595         (compare:CC_NZ
3596          (and:SI (not:SI
3597                   (SHIFT:SI
3598                    (match_operand:SI 1 "register_operand" "r")
3599                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3600                  (match_operand:SI 3 "register_operand" "r"))
3601          (const_int 0)))
3602    (set (match_operand:DI 0 "register_operand" "=r")
3603         (zero_extend:DI (and:SI
3604                          (not:SI
3605                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3606   ""
3607   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3608   [(set_attr "type" "logics_shift_imm")]
3609 )
3610
3611 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3612   [(set (reg:CC_NZ CC_REGNUM)
3613     (compare:CC_NZ
3614      (and:GPI (not:GPI
3615            (SHIFT:GPI
3616             (match_operand:GPI 0 "register_operand" "r")
3617             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3618           (match_operand:GPI 2 "register_operand" "r"))
3619      (const_int 0)))]
3620   ""
3621   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3622   [(set_attr "type" "logics_shift_imm")]
3623 )
3624
3625 (define_insn "clz<mode>2"
3626   [(set (match_operand:GPI 0 "register_operand" "=r")
3627         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3628   ""
3629   "clz\\t%<w>0, %<w>1"
3630   [(set_attr "type" "clz")]
3631 )
3632
3633 (define_expand "ffs<mode>2"
3634   [(match_operand:GPI 0 "register_operand")
3635    (match_operand:GPI 1 "register_operand")]
3636   ""
3637   {
3638     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3639     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3640
3641     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3642     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3643     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3644     DONE;
3645   }
3646 )
3647
3648 (define_insn "clrsb<mode>2"
3649   [(set (match_operand:GPI 0 "register_operand" "=r")
3650         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
3651   ""
3652   "cls\\t%<w>0, %<w>1"
3653   [(set_attr "type" "clz")]
3654 )
3655
3656 (define_insn "rbit<mode>2"
3657   [(set (match_operand:GPI 0 "register_operand" "=r")
3658         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
3659   ""
3660   "rbit\\t%<w>0, %<w>1"
3661   [(set_attr "type" "rbit")]
3662 )
3663
3664 (define_expand "ctz<mode>2"
3665   [(match_operand:GPI 0 "register_operand")
3666    (match_operand:GPI 1 "register_operand")]
3667   ""
3668   {
3669     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3670     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3671     DONE;
3672   }
3673 )
3674
3675 (define_insn "*and<mode>3nr_compare0"
3676   [(set (reg:CC_NZ CC_REGNUM)
3677         (compare:CC_NZ
3678          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
3679                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
3680          (const_int 0)))]
3681   ""
3682   "tst\\t%<w>0, %<w>1"
3683   [(set_attr "type" "logics_reg,logics_imm")]
3684 )
3685
3686 (define_insn "*and<mode>3nr_compare0_zextract"
3687   [(set (reg:CC_NZ CC_REGNUM)
3688         (compare:CC_NZ
3689          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
3690                   (match_operand:GPI 1 "const_int_operand" "n")
3691                   (match_operand:GPI 2 "const_int_operand" "n"))
3692          (const_int 0)))]
3693   "INTVAL (operands[1]) > 0
3694    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
3695         <= GET_MODE_BITSIZE (<MODE>mode))
3696    && aarch64_bitmask_imm (
3697         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
3698                                                  operands[2])),
3699         <MODE>mode)"
3700   {
3701     operands[1]
3702       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
3703     return "tst\\t%<w>0, %1";
3704   }
3705   [(set_attr "type" "logics_shift_imm")]
3706 )
3707
3708 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
3709   [(set (reg:CC_NZ CC_REGNUM)
3710         (compare:CC_NZ
3711          (and:GPI (SHIFT:GPI
3712                    (match_operand:GPI 0 "register_operand" "r")
3713                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3714                   (match_operand:GPI 2 "register_operand" "r"))
3715         (const_int 0)))]
3716   ""
3717   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
3718   [(set_attr "type" "logics_shift_imm")]
3719 )
3720
3721 ;; -------------------------------------------------------------------
3722 ;; Shifts
3723 ;; -------------------------------------------------------------------
3724
3725 (define_expand "<optab><mode>3"
3726   [(set (match_operand:GPI 0 "register_operand")
3727         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
3728                     (match_operand:QI 2 "nonmemory_operand")))]
3729   ""
3730   {
3731     if (CONST_INT_P (operands[2]))
3732       {
3733         operands[2] = GEN_INT (INTVAL (operands[2])
3734                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3735
3736         if (operands[2] == const0_rtx)
3737           {
3738             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3739             DONE;
3740           }
3741       }
3742   }
3743 )
3744
3745 (define_expand "ashl<mode>3"
3746   [(set (match_operand:SHORT 0 "register_operand")
3747         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
3748                       (match_operand:QI 2 "nonmemory_operand")))]
3749   ""
3750   {
3751     if (CONST_INT_P (operands[2]))
3752       {
3753         operands[2] = GEN_INT (INTVAL (operands[2])
3754                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3755
3756         if (operands[2] == const0_rtx)
3757           {
3758             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3759             DONE;
3760           }
3761       }
3762     else
3763       FAIL;
3764   }
3765 )
3766
3767 (define_expand "rotr<mode>3"
3768   [(set (match_operand:GPI 0 "register_operand")
3769         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3770                       (match_operand:QI 2 "nonmemory_operand")))]
3771   ""
3772   {
3773     if (CONST_INT_P (operands[2]))
3774       {
3775         operands[2] = GEN_INT (INTVAL (operands[2])
3776                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3777
3778         if (operands[2] == const0_rtx)
3779           {
3780             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3781             DONE;
3782           }
3783       }
3784   }
3785 )
3786
3787 (define_expand "rotl<mode>3"
3788   [(set (match_operand:GPI 0 "register_operand")
3789         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3790                       (match_operand:QI 2 "nonmemory_operand")))]
3791   ""
3792   {
3793     /* (SZ - cnt) % SZ == -cnt % SZ */
3794     if (CONST_INT_P (operands[2]))
3795       {
3796         operands[2] = GEN_INT ((-INTVAL (operands[2]))
3797                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3798         if (operands[2] == const0_rtx)
3799           {
3800             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3801             DONE;
3802           }
3803       }
3804     else
3805       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
3806                                         NULL_RTX, 1);
3807   }
3808 )
3809
3810 ;; Logical left shift using SISD or Integer instruction
3811 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3812   [(set (match_operand:GPI 0 "register_operand" "=r,w,w")
3813         (ashift:GPI
3814           (match_operand:GPI 1 "register_operand" "r,w,w")
3815           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "rUs<cmode>,Us<cmode>,w")))]
3816   ""
3817   "@
3818    lsl\t%<w>0, %<w>1, %<w>2
3819    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3820    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
3821   [(set_attr "simd" "no,yes,yes")
3822    (set_attr "type" "shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
3823 )
3824
3825 ;; Logical right shift using SISD or Integer instruction
3826 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3827   [(set (match_operand:GPI 0 "register_operand" "=r,w,&w,&w")
3828         (lshiftrt:GPI
3829           (match_operand:GPI 1 "register_operand" "r,w,w,w")
3830           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "rUs<cmode>,Us<cmode>,w,0")))]
3831   ""
3832   "@
3833    lsr\t%<w>0, %<w>1, %<w>2
3834    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3835    #
3836    #"
3837   [(set_attr "simd" "no,yes,yes,yes")
3838    (set_attr "type" "shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
3839 )
3840
3841 (define_split
3842   [(set (match_operand:DI 0 "aarch64_simd_register")
3843         (lshiftrt:DI
3844            (match_operand:DI 1 "aarch64_simd_register")
3845            (match_operand:QI 2 "aarch64_simd_register")))]
3846   "TARGET_SIMD && reload_completed"
3847   [(set (match_dup 3)
3848         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3849    (set (match_dup 0)
3850         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
3851   {
3852     operands[3] = gen_lowpart (QImode, operands[0]);
3853   }
3854 )
3855
3856 (define_split
3857   [(set (match_operand:SI 0 "aarch64_simd_register")
3858         (lshiftrt:SI
3859            (match_operand:SI 1 "aarch64_simd_register")
3860            (match_operand:QI 2 "aarch64_simd_register")))]
3861   "TARGET_SIMD && reload_completed"
3862   [(set (match_dup 3)
3863         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3864    (set (match_dup 0)
3865         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
3866   {
3867     operands[3] = gen_lowpart (QImode, operands[0]);
3868   }
3869 )
3870
3871 ;; Arithmetic right shift using SISD or Integer instruction
3872 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3873   [(set (match_operand:GPI 0 "register_operand" "=r,w,&w,&w")
3874         (ashiftrt:GPI
3875           (match_operand:GPI 1 "register_operand" "r,w,w,w")
3876           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "rUs<cmode>,Us<cmode>,w,0")))]
3877   ""
3878   "@
3879    asr\t%<w>0, %<w>1, %<w>2
3880    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3881    #
3882    #"
3883   [(set_attr "simd" "no,yes,yes,yes")
3884    (set_attr "type" "shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
3885 )
3886
3887 (define_split
3888   [(set (match_operand:DI 0 "aarch64_simd_register")
3889         (ashiftrt:DI
3890            (match_operand:DI 1 "aarch64_simd_register")
3891            (match_operand:QI 2 "aarch64_simd_register")))]
3892   "TARGET_SIMD && reload_completed"
3893   [(set (match_dup 3)
3894         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3895    (set (match_dup 0)
3896         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
3897 {
3898   operands[3] = gen_lowpart (QImode, operands[0]);
3899 }
3900 )
3901
3902 (define_split
3903   [(set (match_operand:SI 0 "aarch64_simd_register")
3904         (ashiftrt:SI
3905            (match_operand:SI 1 "aarch64_simd_register")
3906            (match_operand:QI 2 "aarch64_simd_register")))]
3907   "TARGET_SIMD && reload_completed"
3908   [(set (match_dup 3)
3909         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3910    (set (match_dup 0)
3911         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
3912 {
3913   operands[3] = gen_lowpart (QImode, operands[0]);
3914 }
3915 )
3916
3917 (define_insn "*aarch64_sisd_ushl"
3918   [(set (match_operand:DI 0 "register_operand" "=w")
3919         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3920                     (match_operand:QI 2 "register_operand" "w")]
3921                    UNSPEC_SISD_USHL))]
3922   "TARGET_SIMD"
3923   "ushl\t%d0, %d1, %d2"
3924   [(set_attr "simd" "yes")
3925    (set_attr "type" "neon_shift_reg")]
3926 )
3927
3928 (define_insn "*aarch64_ushl_2s"
3929   [(set (match_operand:SI 0 "register_operand" "=w")
3930         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3931                     (match_operand:QI 2 "register_operand" "w")]
3932                    UNSPEC_USHL_2S))]
3933   "TARGET_SIMD"
3934   "ushl\t%0.2s, %1.2s, %2.2s"
3935   [(set_attr "simd" "yes")
3936    (set_attr "type" "neon_shift_reg")]
3937 )
3938
3939 (define_insn "*aarch64_sisd_sshl"
3940   [(set (match_operand:DI 0 "register_operand" "=w")
3941         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3942                     (match_operand:QI 2 "register_operand" "w")]
3943                    UNSPEC_SISD_SSHL))]
3944   "TARGET_SIMD"
3945   "sshl\t%d0, %d1, %d2"
3946   [(set_attr "simd" "yes")
3947    (set_attr "type" "neon_shift_reg")]
3948 )
3949
3950 (define_insn "*aarch64_sshl_2s"
3951   [(set (match_operand:SI 0 "register_operand" "=w")
3952         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3953                     (match_operand:QI 2 "register_operand" "w")]
3954                    UNSPEC_SSHL_2S))]
3955   "TARGET_SIMD"
3956   "sshl\t%0.2s, %1.2s, %2.2s"
3957   [(set_attr "simd" "yes")
3958    (set_attr "type" "neon_shift_reg")]
3959 )
3960
3961 (define_insn "*aarch64_sisd_neg_qi"
3962   [(set (match_operand:QI 0 "register_operand" "=w")
3963         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
3964                    UNSPEC_SISD_NEG))]
3965   "TARGET_SIMD"
3966   "neg\t%d0, %d1"
3967   [(set_attr "simd" "yes")
3968    (set_attr "type" "neon_neg")]
3969 )
3970
3971 ;; Rotate right
3972 (define_insn "*ror<mode>3_insn"
3973   [(set (match_operand:GPI 0 "register_operand" "=r,r")
3974      (rotatert:GPI
3975        (match_operand:GPI 1 "register_operand" "r,r")
3976        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "r,Us<cmode>")))]
3977   ""
3978   "ror\\t%<w>0, %<w>1, %<w>2"
3979   [(set_attr "type" "shift_reg, rotate_imm")]
3980 )
3981
3982 ;; zero_extend version of above
3983 (define_insn "*<optab>si3_insn_uxtw"
3984   [(set (match_operand:DI 0 "register_operand" "=r")
3985         (zero_extend:DI (SHIFT:SI
3986          (match_operand:SI 1 "register_operand" "r")
3987          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "rUss"))))]
3988   ""
3989   "<shift>\\t%w0, %w1, %w2"
3990   [(set_attr "type" "shift_reg")]
3991 )
3992
3993 (define_insn "*<optab><mode>3_insn"
3994   [(set (match_operand:SHORT 0 "register_operand" "=r")
3995         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
3996                       (match_operand 2 "const_int_operand" "n")))]
3997   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
3998 {
3999   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4000   return "<bfshift>\t%w0, %w1, %2, %3";
4001 }
4002   [(set_attr "type" "bfm")]
4003 )
4004
4005 (define_insn "*extr<mode>5_insn"
4006   [(set (match_operand:GPI 0 "register_operand" "=r")
4007         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4008                              (match_operand 3 "const_int_operand" "n"))
4009                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4010                                (match_operand 4 "const_int_operand" "n"))))]
4011   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4012    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4013   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4014   [(set_attr "type" "shift_imm")]
4015 )
4016
4017 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4018 ;; so we have to match both orderings.
4019 (define_insn "*extr<mode>5_insn_alt"
4020   [(set (match_operand:GPI 0 "register_operand" "=r")
4021         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4022                                 (match_operand 4 "const_int_operand" "n"))
4023                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4024                               (match_operand 3 "const_int_operand" "n"))))]
4025   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4026    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4027        == GET_MODE_BITSIZE (<MODE>mode))"
4028   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4029   [(set_attr "type" "shift_imm")]
4030 )
4031
4032 ;; zero_extend version of the above
4033 (define_insn "*extrsi5_insn_uxtw"
4034   [(set (match_operand:DI 0 "register_operand" "=r")
4035         (zero_extend:DI
4036          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4037                             (match_operand 3 "const_int_operand" "n"))
4038                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4039                               (match_operand 4 "const_int_operand" "n")))))]
4040   "UINTVAL (operands[3]) < 32 &&
4041    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4042   "extr\\t%w0, %w1, %w2, %4"
4043   [(set_attr "type" "shift_imm")]
4044 )
4045
4046 (define_insn "*extrsi5_insn_uxtw_alt"
4047   [(set (match_operand:DI 0 "register_operand" "=r")
4048         (zero_extend:DI
4049          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4050                                (match_operand 4 "const_int_operand" "n"))
4051                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
4052                             (match_operand 3 "const_int_operand" "n")))))]
4053   "UINTVAL (operands[3]) < 32 &&
4054    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4055   "extr\\t%w0, %w1, %w2, %4"
4056   [(set_attr "type" "shift_imm")]
4057 )
4058
4059 (define_insn "*ror<mode>3_insn"
4060   [(set (match_operand:GPI 0 "register_operand" "=r")
4061         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4062                     (match_operand 2 "const_int_operand" "n")))]
4063   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4064 {
4065   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4066   return "ror\\t%<w>0, %<w>1, %3";
4067 }
4068   [(set_attr "type" "rotate_imm")]
4069 )
4070
4071 ;; zero_extend version of the above
4072 (define_insn "*rorsi3_insn_uxtw"
4073   [(set (match_operand:DI 0 "register_operand" "=r")
4074         (zero_extend:DI
4075          (rotate:SI (match_operand:SI 1 "register_operand" "r")
4076                     (match_operand 2 "const_int_operand" "n"))))]
4077   "UINTVAL (operands[2]) < 32"
4078 {
4079   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4080   return "ror\\t%w0, %w1, %3";
4081 }
4082   [(set_attr "type" "rotate_imm")]
4083 )
4084
4085 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4086   [(set (match_operand:GPI 0 "register_operand" "=r")
4087         (ANY_EXTEND:GPI
4088          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4089                        (match_operand 2 "const_int_operand" "n"))))]
4090   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4091 {
4092   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4093   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4094 }
4095   [(set_attr "type" "bfm")]
4096 )
4097
4098 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4099   [(set (match_operand:GPI 0 "register_operand" "=r")
4100         (zero_extend:GPI
4101          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4102                          (match_operand 2 "const_int_operand" "n"))))]
4103   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4104 {
4105   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4106   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4107 }
4108   [(set_attr "type" "bfm")]
4109 )
4110
4111 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4112   [(set (match_operand:GPI 0 "register_operand" "=r")
4113         (sign_extend:GPI
4114          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4115                          (match_operand 2 "const_int_operand" "n"))))]
4116   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4117 {
4118   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4119   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4120 }
4121   [(set_attr "type" "bfm")]
4122 )
4123
4124 ;; -------------------------------------------------------------------
4125 ;; Bitfields
4126 ;; -------------------------------------------------------------------
4127
4128 (define_expand "<optab>"
4129   [(set (match_operand:DI 0 "register_operand" "=r")
4130         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand" "r")
4131                         (match_operand 2 "const_int_operand" "n")
4132                         (match_operand 3 "const_int_operand" "n")))]
4133   ""
4134   ""
4135 )
4136
4137 (define_insn "*<optab><mode>"
4138   [(set (match_operand:GPI 0 "register_operand" "=r")
4139         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4140                          (match_operand 2 "const_int_operand" "n")
4141                          (match_operand 3 "const_int_operand" "n")))]
4142   ""
4143   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4144   [(set_attr "type" "bfm")]
4145 )
4146
4147 ;; Bitfield Insert (insv)
4148 (define_expand "insv<mode>"
4149   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4150                           (match_operand 1 "const_int_operand")
4151                           (match_operand 2 "const_int_operand"))
4152         (match_operand:GPI 3 "general_operand"))]
4153   ""
4154 {
4155   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4156   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4157   rtx value = operands[3];
4158
4159   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4160     FAIL;
4161
4162   if (CONST_INT_P (value))
4163     {
4164       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4165
4166       /* Prefer AND/OR for inserting all zeros or all ones.  */
4167       if ((UINTVAL (value) & mask) == 0
4168            || (UINTVAL (value) & mask) == mask)
4169         FAIL;
4170
4171       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
4172       if (width == 16 && (pos % 16) == 0)
4173         DONE;
4174     }
4175   operands[3] = force_reg (<MODE>mode, value);
4176 })
4177
4178 (define_insn "*insv_reg<mode>"
4179   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4180                           (match_operand 1 "const_int_operand" "n")
4181                           (match_operand 2 "const_int_operand" "n"))
4182         (match_operand:GPI 3 "register_operand" "r"))]
4183   "!(UINTVAL (operands[1]) == 0
4184      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4185          > GET_MODE_BITSIZE (<MODE>mode)))"
4186   "bfi\\t%<w>0, %<w>3, %2, %1"
4187   [(set_attr "type" "bfm")]
4188 )
4189
4190 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4191   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4192                           (match_operand 1 "const_int_operand" "n")
4193                           (match_operand 2 "const_int_operand" "n"))
4194         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
4195   "UINTVAL (operands[1]) <= <ALLX:sizen>"
4196   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4197   [(set_attr "type" "bfm")]
4198 )
4199
4200 (define_insn "*extr_insv_lower_reg<mode>"
4201   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4202                           (match_operand 1 "const_int_operand" "n")
4203                           (const_int 0))
4204         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4205                           (match_dup 1)
4206                           (match_operand 3 "const_int_operand" "n")))]
4207   "!(UINTVAL (operands[1]) == 0
4208      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4209          > GET_MODE_BITSIZE (<MODE>mode)))"
4210   "bfxil\\t%<w>0, %<w>2, %3, %1"
4211   [(set_attr "type" "bfm")]
4212 )
4213
4214 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4215   [(set (match_operand:GPI 0 "register_operand" "=r")
4216         (ashift:GPI (ANY_EXTEND:GPI
4217                      (match_operand:ALLX 1 "register_operand" "r"))
4218                     (match_operand 2 "const_int_operand" "n")))]
4219   "UINTVAL (operands[2]) < <GPI:sizen>"
4220 {
4221   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4222               ? GEN_INT (<ALLX:sizen>)
4223               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4224   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4225 }
4226   [(set_attr "type" "bfm")]
4227 )
4228
4229 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4230
4231 (define_insn "*andim_ashift<mode>_bfiz"
4232   [(set (match_operand:GPI 0 "register_operand" "=r")
4233         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4234                              (match_operand 2 "const_int_operand" "n"))
4235                  (match_operand 3 "const_int_operand" "n")))]
4236   "(INTVAL (operands[2]) < (<GPI:sizen>))
4237    && exact_log2 ((INTVAL (operands[3]) >> INTVAL (operands[2])) + 1) >= 0
4238    && (INTVAL (operands[3]) & ((1 << INTVAL (operands[2])) - 1)) == 0"
4239   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4240   [(set_attr "type" "bfm")]
4241 )
4242
4243 (define_insn "bswap<mode>2"
4244   [(set (match_operand:GPI 0 "register_operand" "=r")
4245         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4246   ""
4247   "rev\\t%<w>0, %<w>1"
4248   [(set_attr "type" "rev")]
4249 )
4250
4251 (define_insn "bswaphi2"
4252   [(set (match_operand:HI 0 "register_operand" "=r")
4253         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4254   ""
4255   "rev16\\t%w0, %w1"
4256   [(set_attr "type" "rev")]
4257 )
4258
4259 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4260 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4261 ;; each valid permutation.
4262
4263 (define_insn "rev16<mode>2"
4264   [(set (match_operand:GPI 0 "register_operand" "=r")
4265         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4266                                       (const_int 8))
4267                           (match_operand:GPI 3 "const_int_operand" "n"))
4268                  (and:GPI (lshiftrt:GPI (match_dup 1)
4269                                         (const_int 8))
4270                           (match_operand:GPI 2 "const_int_operand" "n"))))]
4271   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4272    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4273   "rev16\\t%<w>0, %<w>1"
4274   [(set_attr "type" "rev")]
4275 )
4276
4277 (define_insn "rev16<mode>2_alt"
4278   [(set (match_operand:GPI 0 "register_operand" "=r")
4279         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4280                                         (const_int 8))
4281                           (match_operand:GPI 2 "const_int_operand" "n"))
4282                  (and:GPI (ashift:GPI (match_dup 1)
4283                                       (const_int 8))
4284                           (match_operand:GPI 3 "const_int_operand" "n"))))]
4285   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4286    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4287   "rev16\\t%<w>0, %<w>1"
4288   [(set_attr "type" "rev")]
4289 )
4290
4291 ;; zero_extend version of above
4292 (define_insn "*bswapsi2_uxtw"
4293   [(set (match_operand:DI 0 "register_operand" "=r")
4294         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4295   ""
4296   "rev\\t%w0, %w1"
4297   [(set_attr "type" "rev")]
4298 )
4299
4300 ;; -------------------------------------------------------------------
4301 ;; Floating-point intrinsics
4302 ;; -------------------------------------------------------------------
4303
4304 ;; frint floating-point round to integral standard patterns.
4305 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4306
4307 (define_insn "<frint_pattern><mode>2"
4308   [(set (match_operand:GPF 0 "register_operand" "=w")
4309         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4310          FRINT))]
4311   "TARGET_FLOAT"
4312   "frint<frint_suffix>\\t%<s>0, %<s>1"
4313   [(set_attr "type" "f_rint<s>")]
4314 )
4315
4316 ;; frcvt floating-point round to integer and convert standard patterns.
4317 ;; Expands to lbtrunc, lceil, lfloor, lround.
4318 (define_insn "l<fcvt_pattern><su_optab><GPF:mode><GPI:mode>2"
4319   [(set (match_operand:GPI 0 "register_operand" "=r")
4320         (FIXUORS:GPI (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4321                       FCVT)))]
4322   "TARGET_FLOAT"
4323   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF:s>1"
4324   [(set_attr "type" "f_cvtf2i")]
4325 )
4326
4327 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4328   [(set (match_operand:GPI 0 "register_operand" "=r")
4329         (FIXUORS:GPI
4330           (mult:GPF
4331             (match_operand:GPF 1 "register_operand" "w")
4332             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4333   "TARGET_FLOAT
4334    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4335                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4336   {
4337     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4338     char buf[64];
4339     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4340     output_asm_insn (buf, operands);
4341     return "";
4342   }
4343   [(set_attr "type" "f_cvtf2i")]
4344 )
4345
4346 ;; fma - no throw
4347
4348 (define_insn "fma<mode>4"
4349   [(set (match_operand:GPF 0 "register_operand" "=w")
4350         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4351                  (match_operand:GPF 2 "register_operand" "w")
4352                  (match_operand:GPF 3 "register_operand" "w")))]
4353   "TARGET_FLOAT"
4354   "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4355   [(set_attr "type" "fmac<s>")]
4356 )
4357
4358 (define_insn "fnma<mode>4"
4359   [(set (match_operand:GPF 0 "register_operand" "=w")
4360         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4361                  (match_operand:GPF 2 "register_operand" "w")
4362                  (match_operand:GPF 3 "register_operand" "w")))]
4363   "TARGET_FLOAT"
4364   "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4365   [(set_attr "type" "fmac<s>")]
4366 )
4367
4368 (define_insn "fms<mode>4"
4369   [(set (match_operand:GPF 0 "register_operand" "=w")
4370         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4371                  (match_operand:GPF 2 "register_operand" "w")
4372                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4373   "TARGET_FLOAT"
4374   "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4375   [(set_attr "type" "fmac<s>")]
4376 )
4377
4378 (define_insn "fnms<mode>4"
4379   [(set (match_operand:GPF 0 "register_operand" "=w")
4380         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4381                  (match_operand:GPF 2 "register_operand" "w")
4382                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4383   "TARGET_FLOAT"
4384   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4385   [(set_attr "type" "fmac<s>")]
4386 )
4387
4388 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4389 (define_insn "*fnmadd<mode>4"
4390   [(set (match_operand:GPF 0 "register_operand" "=w")
4391         (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4392                           (match_operand:GPF 2 "register_operand" "w")
4393                           (match_operand:GPF 3 "register_operand" "w"))))]
4394   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
4395   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4396   [(set_attr "type" "fmac<s>")]
4397 )
4398
4399 ;; -------------------------------------------------------------------
4400 ;; Floating-point conversions
4401 ;; -------------------------------------------------------------------
4402
4403 (define_insn "extendsfdf2"
4404   [(set (match_operand:DF 0 "register_operand" "=w")
4405         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
4406   "TARGET_FLOAT"
4407   "fcvt\\t%d0, %s1"
4408   [(set_attr "type" "f_cvt")]
4409 )
4410
4411 (define_insn "extendhfsf2"
4412   [(set (match_operand:SF 0 "register_operand" "=w")
4413         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
4414   "TARGET_FLOAT"
4415   "fcvt\\t%s0, %h1"
4416   [(set_attr "type" "f_cvt")]
4417 )
4418
4419 (define_insn "extendhfdf2"
4420   [(set (match_operand:DF 0 "register_operand" "=w")
4421         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
4422   "TARGET_FLOAT"
4423   "fcvt\\t%d0, %h1"
4424   [(set_attr "type" "f_cvt")]
4425 )
4426
4427 (define_insn "truncdfsf2"
4428   [(set (match_operand:SF 0 "register_operand" "=w")
4429         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
4430   "TARGET_FLOAT"
4431   "fcvt\\t%s0, %d1"
4432   [(set_attr "type" "f_cvt")]
4433 )
4434
4435 (define_insn "truncsfhf2"
4436   [(set (match_operand:HF 0 "register_operand" "=w")
4437         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
4438   "TARGET_FLOAT"
4439   "fcvt\\t%h0, %s1"
4440   [(set_attr "type" "f_cvt")]
4441 )
4442
4443 (define_insn "truncdfhf2"
4444   [(set (match_operand:HF 0 "register_operand" "=w")
4445         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
4446   "TARGET_FLOAT"
4447   "fcvt\\t%h0, %d1"
4448   [(set_attr "type" "f_cvt")]
4449 )
4450
4451 (define_insn "fix_trunc<GPF:mode><GPI:mode>2"
4452   [(set (match_operand:GPI 0 "register_operand" "=r")
4453         (fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4454   "TARGET_FLOAT"
4455   "fcvtzs\\t%<GPI:w>0, %<GPF:s>1"
4456   [(set_attr "type" "f_cvtf2i")]
4457 )
4458
4459 (define_insn "fixuns_trunc<GPF:mode><GPI:mode>2"
4460   [(set (match_operand:GPI 0 "register_operand" "=r")
4461         (unsigned_fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4462   "TARGET_FLOAT"
4463   "fcvtzu\\t%<GPI:w>0, %<GPF:s>1"
4464   [(set_attr "type" "f_cvtf2i")]
4465 )
4466
4467 (define_insn "<optab><fcvt_target><GPF:mode>2"
4468   [(set (match_operand:GPF 0 "register_operand" "=w,w")
4469         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
4470   "TARGET_FLOAT"
4471   "@
4472    <su_optab>cvtf\t%<GPF:s>0, %<s>1
4473    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
4474   [(set_attr "simd" "yes,no")
4475    (set_attr "fp" "no,yes")
4476    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
4477 )
4478
4479 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
4480   [(set (match_operand:GPF 0 "register_operand" "=w")
4481         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
4482   "TARGET_FLOAT"
4483   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
4484   [(set_attr "type" "f_cvti2f")]
4485 )
4486
4487 ;; -------------------------------------------------------------------
4488 ;; Floating-point arithmetic
4489 ;; -------------------------------------------------------------------
4490
4491 (define_insn "add<mode>3"
4492   [(set (match_operand:GPF 0 "register_operand" "=w")
4493         (plus:GPF
4494          (match_operand:GPF 1 "register_operand" "w")
4495          (match_operand:GPF 2 "register_operand" "w")))]
4496   "TARGET_FLOAT"
4497   "fadd\\t%<s>0, %<s>1, %<s>2"
4498   [(set_attr "type" "fadd<s>")]
4499 )
4500
4501 (define_insn "sub<mode>3"
4502   [(set (match_operand:GPF 0 "register_operand" "=w")
4503         (minus:GPF
4504          (match_operand:GPF 1 "register_operand" "w")
4505          (match_operand:GPF 2 "register_operand" "w")))]
4506   "TARGET_FLOAT"
4507   "fsub\\t%<s>0, %<s>1, %<s>2"
4508   [(set_attr "type" "fadd<s>")]
4509 )
4510
4511 (define_insn "mul<mode>3"
4512   [(set (match_operand:GPF 0 "register_operand" "=w")
4513         (mult:GPF
4514          (match_operand:GPF 1 "register_operand" "w")
4515          (match_operand:GPF 2 "register_operand" "w")))]
4516   "TARGET_FLOAT"
4517   "fmul\\t%<s>0, %<s>1, %<s>2"
4518   [(set_attr "type" "fmul<s>")]
4519 )
4520
4521 (define_insn "*fnmul<mode>3"
4522   [(set (match_operand:GPF 0 "register_operand" "=w")
4523         (mult:GPF
4524                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4525                  (match_operand:GPF 2 "register_operand" "w")))]
4526   "TARGET_FLOAT && !flag_rounding_math"
4527   "fnmul\\t%<s>0, %<s>1, %<s>2"
4528   [(set_attr "type" "fmul<s>")]
4529 )
4530
4531 (define_insn "*fnmul<mode>3"
4532   [(set (match_operand:GPF 0 "register_operand" "=w")
4533         (neg:GPF (mult:GPF
4534                  (match_operand:GPF 1 "register_operand" "w")
4535                  (match_operand:GPF 2 "register_operand" "w"))))]
4536   "TARGET_FLOAT"
4537   "fnmul\\t%<s>0, %<s>1, %<s>2"
4538   [(set_attr "type" "fmul<s>")]
4539 )
4540
4541 (define_insn "div<mode>3"
4542   [(set (match_operand:GPF 0 "register_operand" "=w")
4543         (div:GPF
4544          (match_operand:GPF 1 "register_operand" "w")
4545          (match_operand:GPF 2 "register_operand" "w")))]
4546   "TARGET_FLOAT"
4547   "fdiv\\t%<s>0, %<s>1, %<s>2"
4548   [(set_attr "type" "fdiv<s>")]
4549 )
4550
4551 (define_insn "neg<mode>2"
4552   [(set (match_operand:GPF 0 "register_operand" "=w")
4553         (neg:GPF (match_operand:GPF 1 "register_operand" "w")))]
4554   "TARGET_FLOAT"
4555   "fneg\\t%<s>0, %<s>1"
4556   [(set_attr "type" "ffarith<s>")]
4557 )
4558
4559 (define_insn "sqrt<mode>2"
4560   [(set (match_operand:GPF 0 "register_operand" "=w")
4561         (sqrt:GPF (match_operand:GPF 1 "register_operand" "w")))]
4562   "TARGET_FLOAT"
4563   "fsqrt\\t%<s>0, %<s>1"
4564   [(set_attr "type" "fsqrt<s>")]
4565 )
4566
4567 (define_insn "abs<mode>2"
4568   [(set (match_operand:GPF 0 "register_operand" "=w")
4569         (abs:GPF (match_operand:GPF 1 "register_operand" "w")))]
4570   "TARGET_FLOAT"
4571   "fabs\\t%<s>0, %<s>1"
4572   [(set_attr "type" "ffarith<s>")]
4573 )
4574
4575 ;; Given that smax/smin do not specify the result when either input is NaN,
4576 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
4577 ;; for smin.
4578
4579 (define_insn "smax<mode>3"
4580   [(set (match_operand:GPF 0 "register_operand" "=w")
4581         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
4582                   (match_operand:GPF 2 "register_operand" "w")))]
4583   "TARGET_FLOAT"
4584   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
4585   [(set_attr "type" "f_minmax<s>")]
4586 )
4587
4588 (define_insn "smin<mode>3"
4589   [(set (match_operand:GPF 0 "register_operand" "=w")
4590         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
4591                   (match_operand:GPF 2 "register_operand" "w")))]
4592   "TARGET_FLOAT"
4593   "fminnm\\t%<s>0, %<s>1, %<s>2"
4594   [(set_attr "type" "f_minmax<s>")]
4595 )
4596
4597 ;; Scalar forms for the IEEE-754 fmax()/fmin() functions
4598 (define_insn "<fmaxmin><mode>3"
4599   [(set (match_operand:GPF 0 "register_operand" "=w")
4600         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")
4601                      (match_operand:GPF 2 "register_operand" "w")]
4602                      FMAXMIN))]
4603   "TARGET_FLOAT"
4604   "<fmaxmin_op>\\t%<s>0, %<s>1, %<s>2"
4605   [(set_attr "type" "f_minmax<s>")]
4606 )
4607
4608 ;; For copysign (x, y), we want to generate:
4609 ;;
4610 ;;   LDR d2, #(1 << 63)
4611 ;;   BSL v2.8b, [y], [x]
4612 ;;
4613 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
4614 ;; aarch64_simd_bsldf will select the best suited of these instructions
4615 ;; to generate based on register allocation, and knows how to partially
4616 ;; constant fold based on the values of X and Y, so expand through that.
4617
4618 (define_expand "copysigndf3"
4619   [(match_operand:DF 0 "register_operand")
4620    (match_operand:DF 1 "register_operand")
4621    (match_operand:DF 2 "register_operand")]
4622   "TARGET_FLOAT && TARGET_SIMD"
4623 {
4624   rtx mask = gen_reg_rtx (DImode);
4625   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
4626   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
4627                                      operands[2], operands[1]));
4628   DONE;
4629 }
4630 )
4631
4632 ;; As above, but we must first get to a 64-bit value if we wish to use
4633 ;; aarch64_simd_bslv2sf.
4634
4635 (define_expand "copysignsf3"
4636   [(match_operand:SF 0 "register_operand")
4637    (match_operand:SF 1 "register_operand")
4638    (match_operand:SF 2 "register_operand")]
4639   "TARGET_FLOAT && TARGET_SIMD"
4640 {
4641   rtx mask = gen_reg_rtx (DImode);
4642
4643   /* Juggle modes to get us in to a vector mode for BSL.  */
4644   rtx op1 = lowpart_subreg (V2SFmode, operands[1], SFmode);
4645   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
4646   rtx tmp = gen_reg_rtx (V2SFmode);
4647   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 31));
4648   emit_insn (gen_aarch64_simd_bslv2sf (tmp, mask, op2, op1));
4649   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
4650   DONE;
4651 }
4652 )
4653
4654 ;; -------------------------------------------------------------------
4655 ;; Reload support
4656 ;; -------------------------------------------------------------------
4657 ;; Reload Scalar Floating point modes from constant pool.
4658 ;; The AArch64 port doesn't have __int128 constant move support.
4659 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
4660  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
4661        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
4662   (clobber (match_operand:P 2 "register_operand" "=&r"))]
4663  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
4664  {
4665    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
4666    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
4667    DONE;
4668  }
4669 )
4670
4671 ;; Reload Vector modes from constant pool.
4672 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
4673  [(set (match_operand:VALL 0 "register_operand" "=w")
4674        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
4675   (clobber (match_operand:P 2 "register_operand" "=&r"))]
4676  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
4677  {
4678    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
4679    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
4680    DONE;
4681  }
4682 )
4683
4684 (define_expand "aarch64_reload_mov<mode>"
4685   [(set (match_operand:TX 0 "register_operand" "=w")
4686         (match_operand:TX 1 "register_operand" "w"))
4687    (clobber (match_operand:DI 2 "register_operand" "=&r"))
4688   ]
4689   "TARGET_FLOAT"
4690   {
4691     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
4692     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
4693     gen_aarch64_movtilow_tilow (op0, op1);
4694     gen_aarch64_movdi_tihigh (operands[2], op1);
4695     gen_aarch64_movtihigh_di (op0, operands[2]);
4696     DONE;
4697   }
4698 )
4699
4700 ;; The following secondary reload helpers patterns are invoked
4701 ;; after or during reload as we don't want these patterns to start
4702 ;; kicking in during the combiner.
4703
4704 (define_insn "aarch64_movdi_<mode>low"
4705   [(set (match_operand:DI 0 "register_operand" "=r")
4706         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
4707                          (const_int 64) (const_int 0)))]
4708   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4709   "fmov\\t%x0, %d1"
4710   [(set_attr "type" "f_mrc")
4711    (set_attr "length" "4")
4712   ])
4713
4714 (define_insn "aarch64_movdi_<mode>high"
4715   [(set (match_operand:DI 0 "register_operand" "=r")
4716         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
4717                          (const_int 64) (const_int 64)))]
4718   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4719   "fmov\\t%x0, %1.d[1]"
4720   [(set_attr "type" "f_mrc")
4721    (set_attr "length" "4")
4722   ])
4723
4724 (define_insn "aarch64_mov<mode>high_di"
4725   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
4726                          (const_int 64) (const_int 64))
4727         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4728   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4729   "fmov\\t%0.d[1], %x1"
4730   [(set_attr "type" "f_mcr")
4731    (set_attr "length" "4")
4732   ])
4733
4734 (define_insn "aarch64_mov<mode>low_di"
4735   [(set (match_operand:TX 0 "register_operand" "=w")
4736         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4737   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4738   "fmov\\t%d0, %x1"
4739   [(set_attr "type" "f_mcr")
4740    (set_attr "length" "4")
4741   ])
4742
4743 (define_insn "aarch64_movtilow_tilow"
4744   [(set (match_operand:TI 0 "register_operand" "=w")
4745         (zero_extend:TI
4746           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
4747   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4748   "fmov\\t%d0, %d1"
4749   [(set_attr "type" "fmov")
4750    (set_attr "length" "4")
4751   ])
4752
4753 ;; There is a deliberate reason why the parameters of high and lo_sum's
4754 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
4755 ;; and lo_sum's to be used with the labels defining the jump tables in
4756 ;; rodata section.
4757
4758 (define_expand "add_losym"
4759   [(set (match_operand 0 "register_operand" "=r")
4760         (lo_sum (match_operand 1 "register_operand" "r")
4761                 (match_operand 2 "aarch64_valid_symref" "S")))]
4762   ""
4763 {
4764   machine_mode mode = GET_MODE (operands[0]);
4765
4766   emit_insn ((mode == DImode
4767               ? gen_add_losym_di
4768               : gen_add_losym_si) (operands[0],
4769                                    operands[1],
4770                                    operands[2]));
4771   DONE;
4772 })
4773
4774 (define_insn "add_losym_<mode>"
4775   [(set (match_operand:P 0 "register_operand" "=r")
4776         (lo_sum:P (match_operand:P 1 "register_operand" "r")
4777                   (match_operand 2 "aarch64_valid_symref" "S")))]
4778   ""
4779   "add\\t%<w>0, %<w>1, :lo12:%a2"
4780   [(set_attr "type" "alu_imm")]
4781 )
4782
4783 (define_insn "ldr_got_small_<mode>"
4784   [(set (match_operand:PTR 0 "register_operand" "=r")
4785         (unspec:PTR [(mem:PTR (lo_sum:PTR
4786                               (match_operand:PTR 1 "register_operand" "r")
4787                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
4788                     UNSPEC_GOTSMALLPIC))]
4789   ""
4790   "ldr\\t%<w>0, [%1, #:got_lo12:%a2]"
4791   [(set_attr "type" "load1")]
4792 )
4793
4794 (define_insn "ldr_got_small_sidi"
4795   [(set (match_operand:DI 0 "register_operand" "=r")
4796         (zero_extend:DI
4797          (unspec:SI [(mem:SI (lo_sum:DI
4798                              (match_operand:DI 1 "register_operand" "r")
4799                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
4800                     UNSPEC_GOTSMALLPIC)))]
4801   "TARGET_ILP32"
4802   "ldr\\t%w0, [%1, #:got_lo12:%a2]"
4803   [(set_attr "type" "load1")]
4804 )
4805
4806 (define_insn "ldr_got_small_28k_<mode>"
4807   [(set (match_operand:PTR 0 "register_operand" "=r")
4808         (unspec:PTR [(mem:PTR (lo_sum:PTR
4809                               (match_operand:PTR 1 "register_operand" "r")
4810                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
4811                     UNSPEC_GOTSMALLPIC28K))]
4812   ""
4813   "ldr\\t%<w>0, [%1, #:<got_modifier>:%a2]"
4814   [(set_attr "type" "load1")]
4815 )
4816
4817 (define_insn "ldr_got_small_28k_sidi"
4818   [(set (match_operand:DI 0 "register_operand" "=r")
4819         (zero_extend:DI
4820          (unspec:SI [(mem:SI (lo_sum:DI
4821                              (match_operand:DI 1 "register_operand" "r")
4822                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
4823                     UNSPEC_GOTSMALLPIC28K)))]
4824   "TARGET_ILP32"
4825   "ldr\\t%w0, [%1, #:gotpage_lo14:%a2]"
4826   [(set_attr "type" "load1")]
4827 )
4828
4829 (define_insn "ldr_got_tiny"
4830   [(set (match_operand:DI 0 "register_operand" "=r")
4831         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
4832                    UNSPEC_GOTTINYPIC))]
4833   ""
4834   "ldr\\t%0, %L1"
4835   [(set_attr "type" "load1")]
4836 )
4837
4838 (define_insn "aarch64_load_tp_hard"
4839   [(set (match_operand:DI 0 "register_operand" "=r")
4840         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
4841   ""
4842   "mrs\\t%0, tpidr_el0"
4843   [(set_attr "type" "mrs")]
4844 )
4845
4846 ;; The TLS ABI specifically requires that the compiler does not schedule
4847 ;; instructions in the TLS stubs, in order to enable linker relaxation.
4848 ;; Therefore we treat the stubs as an atomic sequence.
4849 (define_expand "tlsgd_small"
4850  [(parallel [(set (match_operand 0 "register_operand" "")
4851                   (call (mem:DI (match_dup 2)) (const_int 1)))
4852              (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
4853              (clobber (reg:DI LR_REGNUM))])]
4854  ""
4855 {
4856   operands[2] = aarch64_tls_get_addr ();
4857 })
4858
4859 (define_insn "*tlsgd_small"
4860   [(set (match_operand 0 "register_operand" "")
4861         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
4862    (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
4863    (clobber (reg:DI LR_REGNUM))
4864   ]
4865   ""
4866   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
4867   [(set_attr "type" "call")
4868    (set_attr "length" "16")])
4869
4870 (define_insn "tlsie_small_<mode>"
4871   [(set (match_operand:PTR 0 "register_operand" "=r")
4872         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4873                    UNSPEC_GOTSMALLTLS))]
4874   ""
4875   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
4876   [(set_attr "type" "load1")
4877    (set_attr "length" "8")]
4878 )
4879
4880 (define_insn "tlsie_small_sidi"
4881   [(set (match_operand:DI 0 "register_operand" "=r")
4882         (zero_extend:DI
4883           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4884                       UNSPEC_GOTSMALLTLS)))]
4885   ""
4886   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
4887   [(set_attr "type" "load1")
4888    (set_attr "length" "8")]
4889 )
4890
4891 (define_insn "tlsie_tiny_<mode>"
4892   [(set (match_operand:PTR 0 "register_operand" "=&r")
4893         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
4894                      (match_operand:PTR 2 "register_operand" "r")]
4895                    UNSPEC_GOTTINYTLS))]
4896   ""
4897   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
4898   [(set_attr "type" "multiple")
4899    (set_attr "length" "8")]
4900 )
4901
4902 (define_insn "tlsie_tiny_sidi"
4903   [(set (match_operand:DI 0 "register_operand" "=&r")
4904         (zero_extend:DI
4905           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
4906                       (match_operand:DI 2 "register_operand" "r")
4907                       ]
4908                       UNSPEC_GOTTINYTLS)))]
4909   ""
4910   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
4911   [(set_attr "type" "multiple")
4912    (set_attr "length" "8")]
4913 )
4914
4915 (define_insn "tlsle12_<mode>"
4916   [(set (match_operand:P 0 "register_operand" "=r")
4917         (unspec:P [(match_operand:P 1 "register_operand" "r")
4918                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4919                    UNSPEC_TLSLE12))]
4920   ""
4921   "add\\t%<w>0, %<w>1, #%L2";
4922   [(set_attr "type" "alu_sreg")
4923    (set_attr "length" "4")]
4924 )
4925
4926 (define_insn "tlsle24_<mode>"
4927   [(set (match_operand:P 0 "register_operand" "=r")
4928         (unspec:P [(match_operand:P 1 "register_operand" "r")
4929                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4930                    UNSPEC_TLSLE24))]
4931   ""
4932   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
4933   [(set_attr "type" "multiple")
4934    (set_attr "length" "8")]
4935 )
4936
4937 (define_insn "tlsle32_<mode>"
4938   [(set (match_operand:P 0 "register_operand" "=r")
4939         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
4940                    UNSPEC_TLSLE32))]
4941   ""
4942   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
4943   [(set_attr "type" "multiple")
4944    (set_attr "length" "8")]
4945 )
4946
4947 (define_insn "tlsle48_<mode>"
4948   [(set (match_operand:P 0 "register_operand" "=r")
4949         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
4950                    UNSPEC_TLSLE48))]
4951   ""
4952   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
4953   [(set_attr "type" "multiple")
4954    (set_attr "length" "12")]
4955 )
4956
4957 (define_insn "tlsdesc_small_<mode>"
4958   [(set (reg:PTR R0_REGNUM)
4959         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
4960                    UNSPEC_TLSDESC))
4961    (clobber (reg:DI LR_REGNUM))
4962    (clobber (reg:CC CC_REGNUM))
4963    (clobber (match_scratch:DI 1 "=r"))]
4964   "TARGET_TLS_DESC"
4965   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
4966   [(set_attr "type" "call")
4967    (set_attr "length" "16")])
4968
4969 (define_insn "stack_tie"
4970   [(set (mem:BLK (scratch))
4971         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
4972                      (match_operand:DI 1 "register_operand" "rk")]
4973                     UNSPEC_PRLG_STK))]
4974   ""
4975   ""
4976   [(set_attr "length" "0")]
4977 )
4978
4979 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
4980 ;; all of memory.  This blocks insns from being moved across this point.
4981
4982 (define_insn "blockage"
4983   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
4984   ""
4985   ""
4986   [(set_attr "length" "0")
4987    (set_attr "type" "block")]
4988 )
4989
4990 (define_insn "probe_stack_range_<PTR:mode>"
4991   [(set (match_operand:PTR 0 "register_operand" "=r")
4992         (unspec_volatile:PTR [(match_operand:PTR 1 "register_operand" "0")
4993                               (match_operand:PTR 2 "register_operand" "r")]
4994                                UNSPECV_PROBE_STACK_RANGE))]
4995   ""
4996 {
4997   return aarch64_output_probe_stack_range (operands[0], operands[2]);
4998 }
4999   [(set_attr "length" "32")]
5000 )
5001
5002 ;; Named pattern for expanding thread pointer reference.
5003 (define_expand "get_thread_pointerdi"
5004   [(match_operand:DI 0 "register_operand" "=r")]
5005   ""
5006 {
5007   rtx tmp = aarch64_load_tp (operands[0]);
5008   if (tmp != operands[0])
5009     emit_move_insn (operands[0], tmp);
5010   DONE;
5011 })
5012
5013 ;; Named patterns for stack smashing protection.
5014 (define_expand "stack_protect_set"
5015   [(match_operand 0 "memory_operand")
5016    (match_operand 1 "memory_operand")]
5017   ""
5018 {
5019   machine_mode mode = GET_MODE (operands[0]);
5020
5021   emit_insn ((mode == DImode
5022               ? gen_stack_protect_set_di
5023               : gen_stack_protect_set_si) (operands[0], operands[1]));
5024   DONE;
5025 })
5026
5027 (define_insn "stack_protect_set_<mode>"
5028   [(set (match_operand:PTR 0 "memory_operand" "=m")
5029         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
5030          UNSPEC_SP_SET))
5031    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
5032   ""
5033   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
5034   [(set_attr "length" "12")
5035    (set_attr "type" "multiple")])
5036
5037 (define_expand "stack_protect_test"
5038   [(match_operand 0 "memory_operand")
5039    (match_operand 1 "memory_operand")
5040    (match_operand 2)]
5041   ""
5042 {
5043   rtx result;
5044   machine_mode mode = GET_MODE (operands[0]);
5045
5046   result = gen_reg_rtx(mode);
5047
5048   emit_insn ((mode == DImode
5049               ? gen_stack_protect_test_di
5050               : gen_stack_protect_test_si) (result,
5051                                             operands[0],
5052                                             operands[1]));
5053
5054   if (mode == DImode)
5055     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5056                                     result, const0_rtx, operands[2]));
5057   else
5058     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5059                                     result, const0_rtx, operands[2]));
5060   DONE;
5061 })
5062
5063 (define_insn "stack_protect_test_<mode>"
5064   [(set (match_operand:PTR 0 "register_operand" "=r")
5065         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
5066                      (match_operand:PTR 2 "memory_operand" "m")]
5067          UNSPEC_SP_TEST))
5068    (clobber (match_scratch:PTR 3 "=&r"))]
5069   ""
5070   "ldr\t%<w>3, %x1\;ldr\t%<w>0, %x2\;eor\t%<w>0, %<w>3, %<w>0"
5071   [(set_attr "length" "12")
5072    (set_attr "type" "multiple")])
5073
5074 ;; Write Floating-point Control Register.
5075 (define_insn "set_fpcr"
5076   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
5077   ""
5078   "msr\\tfpcr, %0"
5079   [(set_attr "type" "mrs")])
5080
5081 ;; Read Floating-point Control Register.
5082 (define_insn "get_fpcr"
5083   [(set (match_operand:SI 0 "register_operand" "=r")
5084         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
5085   ""
5086   "mrs\\t%0, fpcr"
5087   [(set_attr "type" "mrs")])
5088
5089 ;; Write Floating-point Status Register.
5090 (define_insn "set_fpsr"
5091   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
5092   ""
5093   "msr\\tfpsr, %0"
5094   [(set_attr "type" "mrs")])
5095
5096 ;; Read Floating-point Status Register.
5097 (define_insn "get_fpsr"
5098   [(set (match_operand:SI 0 "register_operand" "=r")
5099         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
5100   ""
5101   "mrs\\t%0, fpsr"
5102   [(set_attr "type" "mrs")])
5103
5104
5105 ;; Define the subtract-one-and-jump insns so loop.c
5106 ;; knows what to generate.
5107 (define_expand "doloop_end"
5108   [(use (match_operand 0 "" ""))      ; loop pseudo
5109    (use (match_operand 1 "" ""))]     ; label
5110   "optimize > 0 && flag_modulo_sched"
5111 {
5112   rtx s0;
5113   rtx bcomp;
5114   rtx loc_ref;
5115   rtx cc_reg;
5116   rtx insn;
5117   rtx cmp;
5118
5119   /* Currently SMS relies on the do-loop pattern to recognize loops
5120      where (1) the control part consists of all insns defining and/or
5121      using a certain 'count' register and (2) the loop count can be
5122      adjusted by modifying this register prior to the loop.
5123      ??? The possible introduction of a new block to initialize the
5124      new IV can potentially affect branch optimizations.  */
5125
5126   if (GET_MODE (operands[0]) != DImode)
5127     FAIL;
5128
5129   s0 = operands [0];
5130   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
5131
5132   cmp = XVECEXP (PATTERN (insn), 0, 0);
5133   cc_reg = SET_DEST (cmp);
5134   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
5135   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
5136   emit_jump_insn (gen_rtx_SET (pc_rtx,
5137                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
5138                                                      loc_ref, pc_rtx)));
5139   DONE;
5140 })
5141
5142 ;; AdvSIMD Stuff
5143 (include "aarch64-simd.md")
5144
5145 ;; Atomic Operations
5146 (include "atomics.md")
5147
5148 ;; ldp/stp peephole patterns
5149 (include "aarch64-ldpstp.md")