gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2016 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V15_REGNUM         47)
  61     (V31_REGNUM         63)
  62     (LAST_SAVED_REGNUM  63)
  63     (SFP_REGNUM         64)
  64     (AP_REGNUM          65)
  65     (CC_REGNUM          66)
  66   ]
  67 )
  68
  69 (define_c_enum "unspec" [
  70     UNSPEC_CASESI
  71     UNSPEC_CRC32B
  72     UNSPEC_CRC32CB
  73     UNSPEC_CRC32CH
  74     UNSPEC_CRC32CW
  75     UNSPEC_CRC32CX
  76     UNSPEC_CRC32H
  77     UNSPEC_CRC32W
  78     UNSPEC_CRC32X
  79     UNSPEC_FCVTZS
  80     UNSPEC_FCVTZU
  81     UNSPEC_URECPE
  82     UNSPEC_FRECPE
  83     UNSPEC_FRECPS
  84     UNSPEC_FRECPX
  85     UNSPEC_FRINTA
  86     UNSPEC_FRINTI
  87     UNSPEC_FRINTM
  88     UNSPEC_FRINTN
  89     UNSPEC_FRINTP
  90     UNSPEC_FRINTX
  91     UNSPEC_FRINTZ
  92     UNSPEC_GOTSMALLPIC
  93     UNSPEC_GOTSMALLPIC28K
  94     UNSPEC_GOTSMALLTLS
  95     UNSPEC_GOTTINYPIC
  96     UNSPEC_GOTTINYTLS
  97     UNSPEC_LD1
  98     UNSPEC_LD2
  99     UNSPEC_LD2_DUP
 100     UNSPEC_LD3
 101     UNSPEC_LD3_DUP
 102     UNSPEC_LD4
 103     UNSPEC_LD4_DUP
 104     UNSPEC_LD2_LANE
 105     UNSPEC_LD3_LANE
 106     UNSPEC_LD4_LANE
 107     UNSPEC_MB
 108     UNSPEC_NOP
 109     UNSPEC_PRLG_STK
 110     UNSPEC_RBIT
 111     UNSPEC_SCVTF
 112     UNSPEC_SISD_NEG
 113     UNSPEC_SISD_SSHL
 114     UNSPEC_SISD_USHL
 115     UNSPEC_SSHL_2S
 116     UNSPEC_ST1
 117     UNSPEC_ST2
 118     UNSPEC_ST3
 119     UNSPEC_ST4
 120     UNSPEC_ST2_LANE
 121     UNSPEC_ST3_LANE
 122     UNSPEC_ST4_LANE
 123     UNSPEC_TLS
 124     UNSPEC_TLSDESC
 125     UNSPEC_TLSLE12
 126     UNSPEC_TLSLE24
 127     UNSPEC_TLSLE32
 128     UNSPEC_TLSLE48
 129     UNSPEC_UCVTF
 130     UNSPEC_USHL_2S
 131     UNSPEC_VSTRUCTDUMMY
 132     UNSPEC_SP_SET
 133     UNSPEC_SP_TEST
 134     UNSPEC_RSQRT
 135     UNSPEC_RSQRTE
 136     UNSPEC_RSQRTS
 137     UNSPEC_NZCV
 138 ])
 139
 140 (define_c_enum "unspecv" [
 141     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 142     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 143     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 144     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 145     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 146     UNSPECV_BLOCKAGE            ; Represent a blockage
 147     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 148   ]
 149 )
 150
 151 ;; If further include files are added the defintion of MD_INCLUDES
 152 ;; must be updated.
 153
 154 (include "constraints.md")
 155 (include "predicates.md")
 156 (include "iterators.md")
 157
 158 ;; -------------------------------------------------------------------
 159 ;; Instruction types and attributes
 160 ;; -------------------------------------------------------------------
 161
 162 ; The "type" attribute is included here from AArch32 backend to be able
 163 ; to share pipeline descriptions.
 164 (include "../arm/types.md")
 165
 166 ;; It is important to set the fp or simd attributes to yes when a pattern
 167 ;; alternative uses the FP or SIMD register files, usually signified by use of
 168 ;; the 'w' constraint.  This will ensure that the alternative will be
 169 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 170 ;; architecture extensions.  If all the alternatives in a pattern use the
 171 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 172 ;; or TARGET_SIMD.
 173
 174 ;; Attribute that specifies whether or not the instruction touches fp
 175 ;; registers.  When this is set to yes for an alternative, that alternative
 176 ;; will be disabled when !TARGET_FLOAT.
 177 (define_attr "fp" "no,yes" (const_string "no"))
 178
 179 ;; Attribute that specifies whether or not the instruction touches simd
 180 ;; registers.  When this is set to yes for an alternative, that alternative
 181 ;; will be disabled when !TARGET_SIMD.
 182 (define_attr "simd" "no,yes" (const_string "no"))
 183
 184 (define_attr "length" ""
 185   (const_int 4))
 186
 187 ;; Attribute that controls whether an alternative is enabled or not.
 188 ;; Currently it is only used to disable alternatives which touch fp or simd
 189 ;; registers when -mgeneral-regs-only is specified.
 190 (define_attr "enabled" "no,yes"
 191   (cond [(ior
 192         (and (eq_attr "fp" "yes")
 193              (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 194         (and (eq_attr "simd" "yes")
 195              (eq (symbol_ref "TARGET_SIMD") (const_int 0))))
 196              (const_string "no")
 197         ] (const_string "yes")))
 198
 199 ;; Attribute that specifies whether we are dealing with a branch to a
 200 ;; label that is far away, i.e. further away than the maximum/minimum
 201 ;; representable in a signed 21-bits number.
 202 ;; 0 :=: no
 203 ;; 1 :=: yes
 204 (define_attr "far_branch" "" (const_int 0))
 205
 206 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 207 ;; no predicated insns.
 208 (define_attr "predicated" "yes,no" (const_string "no"))
 209
 210 ;; -------------------------------------------------------------------
 211 ;; Pipeline descriptions and scheduling
 212 ;; -------------------------------------------------------------------
 213
 214 ;; Processor types.
 215 (include "aarch64-tune.md")
 216
 217 ;; Scheduling
 218 (include "../arm/cortex-a53.md")
 219 (include "../arm/cortex-a57.md")
 220 (include "../arm/exynos-m1.md")
 221 (include "thunderx.md")
 222 (include "../arm/xgene1.md")
 223
 224 ;; -------------------------------------------------------------------
 225 ;; Jumps and other miscellaneous insns
 226 ;; -------------------------------------------------------------------
 227
 228 (define_insn "indirect_jump"
 229   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 230   ""
 231   "br\\t%0"
 232   [(set_attr "type" "branch")]
 233 )
 234
 235 (define_insn "jump"
 236   [(set (pc) (label_ref (match_operand 0 "" "")))]
 237   ""
 238   "b\\t%l0"
 239   [(set_attr "type" "branch")]
 240 )
 241
 242 (define_expand "cbranch<mode>4"
 243   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 244                             [(match_operand:GPI 1 "register_operand" "")
 245                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 246                            (label_ref (match_operand 3 "" ""))
 247                            (pc)))]
 248   ""
 249   "
 250   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 251                                          operands[2]);
 252   operands[2] = const0_rtx;
 253   "
 254 )
 255
 256 (define_expand "cbranch<mode>4"
 257   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 258                             [(match_operand:GPF 1 "register_operand" "")
 259                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 260                            (label_ref (match_operand 3 "" ""))
 261                            (pc)))]
 262   ""
 263   "
 264   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 265                                          operands[2]);
 266   operands[2] = const0_rtx;
 267   "
 268 )
 269
 270 (define_expand "cbranchcc4"
 271   [(set (pc) (if_then_else
 272               (match_operator 0 "aarch64_comparison_operator"
 273                [(match_operand 1 "cc_register" "")
 274                 (match_operand 2 "const0_operand")])
 275               (label_ref (match_operand 3 "" ""))
 276               (pc)))]
 277   ""
 278   "")
 279
 280 (define_insn "ccmp<mode>"
 281   [(set (match_operand:CC 1 "cc_register" "")
 282         (if_then_else:CC
 283           (match_operator 4 "aarch64_comparison_operator"
 284            [(match_operand 0 "cc_register" "")
 285             (const_int 0)])
 286           (compare:CC
 287             (match_operand:GPI 2 "register_operand" "r,r,r")
 288             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 289           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 290   ""
 291   "@
 292    ccmp\\t%<w>2, %<w>3, %k5, %m4
 293    ccmp\\t%<w>2, %3, %k5, %m4
 294    ccmn\\t%<w>2, #%n3, %k5, %m4"
 295   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 296 )
 297
 298 (define_insn "fccmp<mode>"
 299   [(set (match_operand:CCFP 1 "cc_register" "")
 300         (if_then_else:CCFP
 301           (match_operator 4 "aarch64_comparison_operator"
 302            [(match_operand 0 "cc_register" "")
 303             (const_int 0)])
 304           (compare:CCFP
 305             (match_operand:GPF 2 "register_operand" "w")
 306             (match_operand:GPF 3 "register_operand" "w"))
 307           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 308   "TARGET_FLOAT"
 309   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 310   [(set_attr "type" "fccmp<s>")]
 311 )
 312
 313 (define_insn "fccmpe<mode>"
 314   [(set (match_operand:CCFPE 1 "cc_register" "")
 315          (if_then_else:CCFPE
 316           (match_operator 4 "aarch64_comparison_operator"
 317            [(match_operand 0 "cc_register" "")
 318           (const_int 0)])
 319            (compare:CCFPE
 320             (match_operand:GPF 2 "register_operand" "w")
 321             (match_operand:GPF 3 "register_operand" "w"))
 322           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 323   "TARGET_FLOAT"
 324   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 325   [(set_attr "type" "fccmp<s>")]
 326 )
 327
 328 ;; Expansion of signed mod by a power of 2 using CSNEG.
 329 ;; For x0 % n where n is a power of 2 produce:
 330 ;; negs   x1, x0
 331 ;; and    x0, x0, #(n - 1)
 332 ;; and    x1, x1, #(n - 1)
 333 ;; csneg  x0, x0, x1, mi
 334
 335 (define_expand "mod<mode>3"
 336   [(match_operand:GPI 0 "register_operand" "")
 337    (match_operand:GPI 1 "register_operand" "")
 338    (match_operand:GPI 2 "const_int_operand" "")]
 339   ""
 340   {
 341     HOST_WIDE_INT val = INTVAL (operands[2]);
 342
 343     if (val <= 0
 344        || exact_log2 (val) <= 0
 345        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 346       FAIL;
 347
 348     rtx mask = GEN_INT (val - 1);
 349
 350     /* In the special case of x0 % 2 we can do the even shorter:
 351         cmp    x0, xzr
 352         and    x0, x0, 1
 353         cneg   x0, x0, lt.  */
 354     if (val == 2)
 355       {
 356         rtx masked = gen_reg_rtx (<MODE>mode);
 357         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 358         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 359         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 360         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 361         DONE;
 362       }
 363
 364     rtx neg_op = gen_reg_rtx (<MODE>mode);
 365     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 366
 367     /* Extract the condition register and mode.  */
 368     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 369     rtx cc_reg = SET_DEST (cmp);
 370     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 371
 372     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 373     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 374
 375     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 376     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 377
 378     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 379                                        masked_neg, masked_pos));
 380     DONE;
 381   }
 382 )
 383
 384 (define_insn "condjump"
 385   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 386                             [(match_operand 1 "cc_register" "") (const_int 0)])
 387                            (label_ref (match_operand 2 "" ""))
 388                            (pc)))]
 389   ""
 390   {
 391     if (get_attr_length (insn) == 8)
 392       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 393     else
 394       return  "b%m0\\t%l2";
 395   }
 396   [(set_attr "type" "branch")
 397    (set (attr "length")
 398         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 399                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 400                       (const_int 4)
 401                       (const_int 8)))
 402    (set (attr "far_branch")
 403         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 404                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 405                       (const_int 0)
 406                       (const_int 1)))]
 407 )
 408
 409 ;; For a 24-bit immediate CST we can optimize the compare for equality
 410 ;; and branch sequence from:
 411 ;;      mov     x0, #imm1
 412 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 413 ;;      cmp     x1, x0
 414 ;;      b<ne,eq> .Label
 415 ;; into the shorter:
 416 ;;      sub     x0, x1, #(CST & 0xfff000)
 417 ;;      subs    x0, x0, #(CST & 0x000fff)
 418 ;;      b<ne,eq> .Label
 419 (define_insn_and_split "*compare_condjump<mode>"
 420   [(set (pc) (if_then_else (EQL
 421                               (match_operand:GPI 0 "register_operand" "r")
 422                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 423                            (label_ref:P (match_operand 2 "" ""))
 424                            (pc)))]
 425   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 426    && !aarch64_plus_operand (operands[1], <MODE>mode)
 427    && !reload_completed"
 428   "#"
 429   "&& true"
 430   [(const_int 0)]
 431   {
 432     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 433     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 434     rtx tmp = gen_reg_rtx (<MODE>mode);
 435     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 436     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 437     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 438     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 439     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 440     DONE;
 441   }
 442 )
 443
 444 (define_expand "casesi"
 445   [(match_operand:SI 0 "register_operand" "")   ; Index
 446    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 447    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 448    (match_operand:DI 3 "" "")                   ; Table label
 449    (match_operand:DI 4 "" "")]                  ; Out of range label
 450   ""
 451   {
 452     if (operands[1] != const0_rtx)
 453       {
 454         rtx reg = gen_reg_rtx (SImode);
 455
 456         /* Canonical RTL says that if you have:
 457
 458            (minus (X) (CONST))
 459
 460            then this should be emitted as:
 461
 462            (plus (X) (-CONST))
 463
 464            The use of trunc_int_for_mode ensures that the resulting
 465            constant can be represented in SImode, this is important
 466            for the corner case where operand[1] is INT_MIN.  */
 467
 468         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 469
 470         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 471               (operands[1], SImode))
 472           operands[1] = force_reg (SImode, operands[1]);
 473         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 474         operands[0] = reg;
 475       }
 476
 477     if (!aarch64_plus_operand (operands[2], SImode))
 478       operands[2] = force_reg (SImode, operands[2]);
 479     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 480                                                  const0_rtx),
 481                                     operands[0], operands[2], operands[4]));
 482
 483     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (VOIDmode, operands[3]));
 484     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 485                                          operands[3]));
 486     DONE;
 487   }
 488 )
 489
 490 (define_insn "casesi_dispatch"
 491   [(parallel
 492     [(set (pc)
 493           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 494                            (match_operand:SI 1 "register_operand" "r")]
 495                         UNSPEC_CASESI)))
 496      (clobber (reg:CC CC_REGNUM))
 497      (clobber (match_scratch:DI 3 "=r"))
 498      (clobber (match_scratch:DI 4 "=r"))
 499      (use (label_ref (match_operand 2 "" "")))])]
 500   ""
 501   "*
 502   return aarch64_output_casesi (operands);
 503   "
 504   [(set_attr "length" "16")
 505    (set_attr "type" "branch")]
 506 )
 507
 508 (define_insn "nop"
 509   [(unspec[(const_int 0)] UNSPEC_NOP)]
 510   ""
 511   "nop"
 512   [(set_attr "type" "no_insn")]
 513 )
 514
 515 (define_insn "prefetch"
 516   [(prefetch (match_operand:DI 0 "register_operand" "r")
 517             (match_operand:QI 1 "const_int_operand" "")
 518             (match_operand:QI 2 "const_int_operand" ""))]
 519   ""
 520   {
 521     const char * pftype[2][4] =
 522     {
 523       {"prfm\\tPLDL1STRM, %a0",
 524        "prfm\\tPLDL3KEEP, %a0",
 525        "prfm\\tPLDL2KEEP, %a0",
 526        "prfm\\tPLDL1KEEP, %a0"},
 527       {"prfm\\tPSTL1STRM, %a0",
 528        "prfm\\tPSTL3KEEP, %a0",
 529        "prfm\\tPSTL2KEEP, %a0",
 530        "prfm\\tPSTL1KEEP, %a0"},
 531     };
 532
 533     int locality = INTVAL (operands[2]);
 534
 535     gcc_assert (IN_RANGE (locality, 0, 3));
 536
 537     return pftype[INTVAL(operands[1])][locality];
 538   }
 539   [(set_attr "type" "load1")]
 540 )
 541
 542 (define_insn "trap"
 543   [(trap_if (const_int 1) (const_int 8))]
 544   ""
 545   "brk #1000"
 546   [(set_attr "type" "trap")])
 547
 548 (define_expand "prologue"
 549   [(clobber (const_int 0))]
 550   ""
 551   "
 552   aarch64_expand_prologue ();
 553   DONE;
 554   "
 555 )
 556
 557 (define_expand "epilogue"
 558   [(clobber (const_int 0))]
 559   ""
 560   "
 561   aarch64_expand_epilogue (false);
 562   DONE;
 563   "
 564 )
 565
 566 (define_expand "sibcall_epilogue"
 567   [(clobber (const_int 0))]
 568   ""
 569   "
 570   aarch64_expand_epilogue (true);
 571   DONE;
 572   "
 573 )
 574
 575 (define_insn "*do_return"
 576   [(return)]
 577   ""
 578   "ret"
 579   [(set_attr "type" "branch")]
 580 )
 581
 582 (define_expand "return"
 583   [(simple_return)]
 584   "aarch64_use_return_insn_p ()"
 585   ""
 586 )
 587
 588 (define_insn "simple_return"
 589   [(simple_return)]
 590   ""
 591   "ret"
 592   [(set_attr "type" "branch")]
 593 )
 594
 595 (define_insn "eh_return"
 596   [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")]
 597     UNSPECV_EH_RETURN)]
 598   ""
 599   "#"
 600   [(set_attr "type" "branch")]
 601
 602 )
 603
 604 (define_split
 605   [(unspec_volatile [(match_operand:DI 0 "register_operand" "")]
 606     UNSPECV_EH_RETURN)]
 607   "reload_completed"
 608   [(set (match_dup 1) (match_dup 0))]
 609   {
 610     operands[1] = aarch64_final_eh_return_addr ();
 611   }
 612 )
 613
 614 (define_insn "*cb<optab><mode>1"
 615   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 616                                 (const_int 0))
 617                            (label_ref (match_operand 1 "" ""))
 618                            (pc)))]
 619   ""
 620   {
 621     if (get_attr_length (insn) == 8)
 622       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 623     else
 624       return "<cbz>\\t%<w>0, %l1";
 625   }
 626   [(set_attr "type" "branch")
 627    (set (attr "length")
 628         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 629                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 630                       (const_int 4)
 631                       (const_int 8)))
 632    (set (attr "far_branch")
 633         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 634                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 635                       (const_int 0)
 636                       (const_int 1)))]
 637 )
 638
 639 (define_insn "*tb<optab><mode>1"
 640   [(set (pc) (if_then_else
 641               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 642                                     (const_int 1)
 643                                     (match_operand 1
 644                                       "aarch64_simd_shift_imm_<mode>" "n"))
 645                    (const_int 0))
 646              (label_ref (match_operand 2 "" ""))
 647              (pc)))
 648    (clobber (reg:CC CC_REGNUM))]
 649   ""
 650   {
 651     if (get_attr_length (insn) == 8)
 652       {
 653         if (get_attr_far_branch (insn) == 1)
 654           return aarch64_gen_far_branch (operands, 2, "Ltb",
 655                                          "<inv_tb>\\t%<w>0, %1, ");
 656         else
 657           {
 658             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 659             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 660           }
 661       }
 662     else
 663       return "<tbz>\t%<w>0, %1, %l2";
 664   }
 665   [(set_attr "type" "branch")
 666    (set (attr "length")
 667         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 668                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 669                       (const_int 4)
 670                       (const_int 8)))
 671    (set (attr "far_branch")
 672         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 673                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 674                       (const_int 0)
 675                       (const_int 1)))]
 676
 677 )
 678
 679 (define_insn "*cb<optab><mode>1"
 680   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 681                                  (const_int 0))
 682                            (label_ref (match_operand 1 "" ""))
 683                            (pc)))
 684    (clobber (reg:CC CC_REGNUM))]
 685   ""
 686   {
 687     if (get_attr_length (insn) == 8)
 688       {
 689         if (get_attr_far_branch (insn) == 1)
 690           return aarch64_gen_far_branch (operands, 1, "Ltb",
 691                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 692         else
 693           {
 694             char buf[64];
 695             uint64_t val = ((uint64_t) 1)
 696                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 697             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 698             output_asm_insn (buf, operands);
 699             return "<bcond>\t%l1";
 700           }
 701       }
 702     else
 703       return "<tbz>\t%<w>0, <sizem1>, %l1";
 704   }
 705   [(set_attr "type" "branch")
 706    (set (attr "length")
 707         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 708                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 709                       (const_int 4)
 710                       (const_int 8)))
 711    (set (attr "far_branch")
 712         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 713                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 714                       (const_int 0)
 715                       (const_int 1)))]
 716 )
 717
 718 ;; -------------------------------------------------------------------
 719 ;; Subroutine calls and sibcalls
 720 ;; -------------------------------------------------------------------
 721
 722 (define_expand "call_internal"
 723   [(parallel [(call (match_operand 0 "memory_operand" "")
 724                     (match_operand 1 "general_operand" ""))
 725               (use (match_operand 2 "" ""))
 726               (clobber (reg:DI LR_REGNUM))])])
 727
 728 (define_expand "call"
 729   [(parallel [(call (match_operand 0 "memory_operand" "")
 730                     (match_operand 1 "general_operand" ""))
 731               (use (match_operand 2 "" ""))
 732               (clobber (reg:DI LR_REGNUM))])]
 733   ""
 734   "
 735   {
 736     rtx callee, pat;
 737
 738     /* In an untyped call, we can get NULL for operand 2.  */
 739     if (operands[2] == NULL)
 740       operands[2] = const0_rtx;
 741
 742     /* Decide if we should generate indirect calls by loading the
 743        64-bit address of the callee into a register before performing
 744        the branch-and-link.  */
 745     callee = XEXP (operands[0], 0);
 746     if (GET_CODE (callee) == SYMBOL_REF
 747         ? (aarch64_is_long_call_p (callee)
 748            || aarch64_is_noplt_call_p (callee))
 749         : !REG_P (callee))
 750       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 751
 752     pat = gen_call_internal (operands[0], operands[1], operands[2]);
 753     aarch64_emit_call_insn (pat);
 754     DONE;
 755   }"
 756 )
 757
 758 (define_insn "*call_reg"
 759   [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
 760          (match_operand 1 "" ""))
 761    (use (match_operand 2 "" ""))
 762    (clobber (reg:DI LR_REGNUM))]
 763   ""
 764   "blr\\t%0"
 765   [(set_attr "type" "call")]
 766 )
 767
 768 (define_insn "*call_symbol"
 769   [(call (mem:DI (match_operand:DI 0 "" ""))
 770          (match_operand 1 "" ""))
 771    (use (match_operand 2 "" ""))
 772    (clobber (reg:DI LR_REGNUM))]
 773   "GET_CODE (operands[0]) == SYMBOL_REF
 774    && !aarch64_is_long_call_p (operands[0])
 775    && !aarch64_is_noplt_call_p (operands[0])"
 776   "bl\\t%a0"
 777   [(set_attr "type" "call")]
 778 )
 779
 780 (define_expand "call_value_internal"
 781   [(parallel [(set (match_operand 0 "" "")
 782                    (call (match_operand 1 "memory_operand" "")
 783                          (match_operand 2 "general_operand" "")))
 784               (use (match_operand 3 "" ""))
 785               (clobber (reg:DI LR_REGNUM))])])
 786
 787 (define_expand "call_value"
 788   [(parallel [(set (match_operand 0 "" "")
 789                    (call (match_operand 1 "memory_operand" "")
 790                          (match_operand 2 "general_operand" "")))
 791               (use (match_operand 3 "" ""))
 792               (clobber (reg:DI LR_REGNUM))])]
 793   ""
 794   "
 795   {
 796     rtx callee, pat;
 797
 798     /* In an untyped call, we can get NULL for operand 3.  */
 799     if (operands[3] == NULL)
 800       operands[3] = const0_rtx;
 801
 802     /* Decide if we should generate indirect calls by loading the
 803        64-bit address of the callee into a register before performing
 804        the branch-and-link.  */
 805     callee = XEXP (operands[1], 0);
 806     if (GET_CODE (callee) == SYMBOL_REF
 807         ? (aarch64_is_long_call_p (callee)
 808            || aarch64_is_noplt_call_p (callee))
 809         : !REG_P (callee))
 810       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 811
 812     pat = gen_call_value_internal (operands[0], operands[1], operands[2],
 813                                    operands[3]);
 814     aarch64_emit_call_insn (pat);
 815     DONE;
 816   }"
 817 )
 818
 819 (define_insn "*call_value_reg"
 820   [(set (match_operand 0 "" "")
 821         (call (mem:DI (match_operand:DI 1 "register_operand" "r"))
 822                       (match_operand 2 "" "")))
 823    (use (match_operand 3 "" ""))
 824    (clobber (reg:DI LR_REGNUM))]
 825   ""
 826   "blr\\t%1"
 827   [(set_attr "type" "call")]
 828
 829 )
 830
 831 (define_insn "*call_value_symbol"
 832   [(set (match_operand 0 "" "")
 833         (call (mem:DI (match_operand:DI 1 "" ""))
 834               (match_operand 2 "" "")))
 835    (use (match_operand 3 "" ""))
 836    (clobber (reg:DI LR_REGNUM))]
 837   "GET_CODE (operands[1]) == SYMBOL_REF
 838    && !aarch64_is_long_call_p (operands[1])
 839    && !aarch64_is_noplt_call_p (operands[1])"
 840   "bl\\t%a1"
 841   [(set_attr "type" "call")]
 842 )
 843
 844 (define_expand "sibcall_internal"
 845   [(parallel [(call (match_operand 0 "memory_operand" "")
 846                     (match_operand 1 "general_operand" ""))
 847               (return)
 848               (use (match_operand 2 "" ""))])])
 849
 850 (define_expand "sibcall"
 851   [(parallel [(call (match_operand 0 "memory_operand" "")
 852                     (match_operand 1 "general_operand" ""))
 853               (return)
 854               (use (match_operand 2 "" ""))])]
 855   ""
 856   {
 857     rtx pat;
 858     rtx callee = XEXP (operands[0], 0);
 859     if (!REG_P (callee)
 860        && ((GET_CODE (callee) != SYMBOL_REF)
 861            || aarch64_is_noplt_call_p (callee)))
 862       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 863
 864     if (operands[2] == NULL_RTX)
 865       operands[2] = const0_rtx;
 866
 867     pat = gen_sibcall_internal (operands[0], operands[1], operands[2]);
 868     aarch64_emit_call_insn (pat);
 869     DONE;
 870   }
 871 )
 872
 873 (define_expand "sibcall_value_internal"
 874   [(parallel [(set (match_operand 0 "" "")
 875                    (call (match_operand 1 "memory_operand" "")
 876                          (match_operand 2 "general_operand" "")))
 877               (return)
 878               (use (match_operand 3 "" ""))])])
 879
 880 (define_expand "sibcall_value"
 881   [(parallel [(set (match_operand 0 "" "")
 882                    (call (match_operand 1 "memory_operand" "")
 883                          (match_operand 2 "general_operand" "")))
 884               (return)
 885               (use (match_operand 3 "" ""))])]
 886   ""
 887   {
 888     rtx pat;
 889     rtx callee = XEXP (operands[1], 0);
 890     if (!REG_P (callee)
 891        && ((GET_CODE (callee) != SYMBOL_REF)
 892            || aarch64_is_noplt_call_p (callee)))
 893       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 894
 895     if (operands[3] == NULL_RTX)
 896       operands[3] = const0_rtx;
 897
 898     pat = gen_sibcall_value_internal (operands[0], operands[1], operands[2],
 899                                       operands[3]);
 900     aarch64_emit_call_insn (pat);
 901     DONE;
 902   }
 903 )
 904
 905 (define_insn "*sibcall_insn"
 906   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 907          (match_operand 1 "" ""))
 908    (return)
 909    (use (match_operand 2 "" ""))]
 910   "SIBLING_CALL_P (insn)"
 911   "@
 912    br\\t%0
 913    b\\t%a0"
 914   [(set_attr "type" "branch, branch")]
 915 )
 916
 917 (define_insn "*sibcall_value_insn"
 918   [(set (match_operand 0 "" "")
 919         (call (mem:DI
 920                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 921               (match_operand 2 "" "")))
 922    (return)
 923    (use (match_operand 3 "" ""))]
 924   "SIBLING_CALL_P (insn)"
 925   "@
 926    br\\t%1
 927    b\\t%a1"
 928   [(set_attr "type" "branch, branch")]
 929 )
 930
 931 ;; Call subroutine returning any type.
 932
 933 (define_expand "untyped_call"
 934   [(parallel [(call (match_operand 0 "")
 935                     (const_int 0))
 936               (match_operand 1 "")
 937               (match_operand 2 "")])]
 938   ""
 939 {
 940   int i;
 941
 942   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 943
 944   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 945     {
 946       rtx set = XVECEXP (operands[2], 0, i);
 947       emit_move_insn (SET_DEST (set), SET_SRC (set));
 948     }
 949
 950   /* The optimizer does not know that the call sets the function value
 951      registers we stored in the result block.  We avoid problems by
 952      claiming that all hard registers are used and clobbered at this
 953      point.  */
 954   emit_insn (gen_blockage ());
 955   DONE;
 956 })
 957
 958 ;; -------------------------------------------------------------------
 959 ;; Moves
 960 ;; -------------------------------------------------------------------
 961
 962 (define_expand "mov<mode>"
 963   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 964         (match_operand:SHORT 1 "general_operand" ""))]
 965   ""
 966   "
 967     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 968       operands[1] = force_reg (<MODE>mode, operands[1]);
 969   "
 970 )
 971
 972 (define_insn "*mov<mode>_aarch64"
 973   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,   *w,r,*w, m, m, r,*w,*w")
 974         (match_operand:SHORT 1 "general_operand"      " r,M,D<hq>,m, m,rZ,*w,*w, r,*w"))]
 975   "(register_operand (operands[0], <MODE>mode)
 976     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 977 {
 978    switch (which_alternative)
 979      {
 980      case 0:
 981        return "mov\t%w0, %w1";
 982      case 1:
 983        return "mov\t%w0, %1";
 984      case 2:
 985        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 986                                                         <MODE>mode);
 987      case 3:
 988        return "ldr<size>\t%w0, %1";
 989      case 4:
 990        return "ldr\t%<size>0, %1";
 991      case 5:
 992        return "str<size>\t%w1, %0";
 993      case 6:
 994        return "str\t%<size>1, %0";
 995      case 7:
 996        return "umov\t%w0, %1.<v>[0]";
 997      case 8:
 998        return "dup\t%0.<Vallxd>, %w1";
 999      case 9:
1000        return "dup\t%<Vetype>0, %1.<v>[0]";
1001      default:
1002        gcc_unreachable ();
1003      }
1004 }
1005   [(set_attr "type" "mov_reg,mov_imm,neon_move,load1,load1,store1,store1,\
1006                      neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1007    (set_attr "simd" "*,*,yes,*,*,*,*,yes,yes,yes")]
1008 )
1009
1010 (define_expand "mov<mode>"
1011   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1012         (match_operand:GPI 1 "general_operand" ""))]
1013   ""
1014   "
1015     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1016         && <MODE>mode == DImode
1017         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1018       DONE;
1019
1020     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1021       operands[1] = force_reg (<MODE>mode, operands[1]);
1022
1023     /* FIXME: RR we still need to fix up what we are doing with
1024        symbol_refs and other types of constants.  */
1025     if (CONSTANT_P (operands[1])
1026         && !CONST_INT_P (operands[1]))
1027      {
1028        aarch64_expand_mov_immediate (operands[0], operands[1]);
1029        DONE;
1030      }
1031   "
1032 )
1033
1034 (define_insn_and_split "*movsi_aarch64"
1035   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r  ,*w, r,*w")
1036         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,m, m,rZ,*w,S,Ush,rZ,*w,*w"))]
1037   "(register_operand (operands[0], SImode)
1038     || aarch64_reg_or_zero (operands[1], SImode))"
1039   "@
1040    mov\\t%w0, %w1
1041    mov\\t%w0, %w1
1042    mov\\t%w0, %w1
1043    mov\\t%w0, %1
1044    #
1045    ldr\\t%w0, %1
1046    ldr\\t%s0, %1
1047    str\\t%w1, %0
1048    str\\t%s1, %0
1049    adr\\t%x0, %a1
1050    adrp\\t%x0, %A1
1051    fmov\\t%s0, %w1
1052    fmov\\t%w0, %s1
1053    fmov\\t%s0, %s1"
1054    "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1055     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1056    [(const_int 0)]
1057    "{
1058        aarch64_expand_mov_immediate (operands[0], operands[1]);
1059        DONE;
1060     }"
1061   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1062                      adr,adr,f_mcr,f_mrc,fmov")
1063    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes")]
1064 )
1065
1066 (define_insn_and_split "*movdi_aarch64"
1067   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r,  *w, r,*w,w")
1068         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,n,m, m,rZ,*w,S,Ush,rZ,*w,*w,Dd"))]
1069   "(register_operand (operands[0], DImode)
1070     || aarch64_reg_or_zero (operands[1], DImode))"
1071   "@
1072    mov\\t%x0, %x1
1073    mov\\t%0, %x1
1074    mov\\t%x0, %1
1075    mov\\t%x0, %1
1076    #
1077    ldr\\t%x0, %1
1078    ldr\\t%d0, %1
1079    str\\t%x1, %0
1080    str\\t%d1, %0
1081    adr\\t%x0, %a1
1082    adrp\\t%x0, %A1
1083    fmov\\t%d0, %x1
1084    fmov\\t%x0, %d1
1085    fmov\\t%d0, %d1
1086    movi\\t%d0, %1"
1087    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1088     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1089    [(const_int 0)]
1090    "{
1091        aarch64_expand_mov_immediate (operands[0], operands[1]);
1092        DONE;
1093     }"
1094   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1095                      adr,adr,f_mcr,f_mrc,fmov,neon_move")
1096    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1097    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")]
1098 )
1099
1100 (define_insn "insv_imm<mode>"
1101   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1102                           (const_int 16)
1103                           (match_operand:GPI 1 "const_int_operand" "n"))
1104         (match_operand:GPI 2 "const_int_operand" "n"))]
1105   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1106    && UINTVAL (operands[1]) % 16 == 0"
1107   "movk\\t%<w>0, %X2, lsl %1"
1108   [(set_attr "type" "mov_imm")]
1109 )
1110
1111 (define_expand "movti"
1112   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1113         (match_operand:TI 1 "general_operand" ""))]
1114   ""
1115   "
1116     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1117       operands[1] = force_reg (TImode, operands[1]);
1118   "
1119 )
1120
1121 (define_insn "*movti_aarch64"
1122   [(set (match_operand:TI 0
1123          "nonimmediate_operand"  "=r, *w,r ,*w,r  ,Ump,Ump,*w,m")
1124         (match_operand:TI 1
1125          "aarch64_movti_operand" " rn,r ,*w,*w,Ump,r  ,Z  , m,*w"))]
1126   "(register_operand (operands[0], TImode)
1127     || aarch64_reg_or_zero (operands[1], TImode))"
1128   "@
1129    #
1130    #
1131    #
1132    orr\\t%0.16b, %1.16b, %1.16b
1133    ldp\\t%0, %H0, %1
1134    stp\\t%1, %H1, %0
1135    stp\\txzr, xzr, %0
1136    ldr\\t%q0, %1
1137    str\\t%q1, %0"
1138   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1139                              load2,store2,store2,f_loadd,f_stored")
1140    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1141    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1142    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1143 )
1144
1145 ;; Split a TImode register-register or register-immediate move into
1146 ;; its component DImode pieces, taking care to handle overlapping
1147 ;; source and dest registers.
1148 (define_split
1149    [(set (match_operand:TI 0 "register_operand" "")
1150          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1151   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1152   [(const_int 0)]
1153 {
1154   aarch64_split_128bit_move (operands[0], operands[1]);
1155   DONE;
1156 })
1157
1158 (define_expand "mov<mode>"
1159   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1160         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1161   ""
1162   {
1163     if (!TARGET_FLOAT)
1164       {
1165         aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1166         FAIL;
1167       }
1168
1169     if (GET_CODE (operands[0]) == MEM
1170         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1171               && aarch64_float_const_zero_rtx_p (operands[1])))
1172       operands[1] = force_reg (<MODE>mode, operands[1]);
1173   }
1174 )
1175
1176 (define_insn "*movhf_aarch64"
1177   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  ,?r,w,w,m,r,m ,r")
1178         (match_operand:HF 1 "general_operand"      "Y ,?rY, w,w,m,w,m,rY,r"))]
1179   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1180     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1181   "@
1182    movi\\t%0.4h, #0
1183    mov\\t%0.h[0], %w1
1184    umov\\t%w0, %1.h[0]
1185    mov\\t%0.h[0], %1.h[0]
1186    ldr\\t%h0, %1
1187    str\\t%h1, %0
1188    ldrh\\t%w0, %1
1189    strh\\t%w1, %0
1190    mov\\t%w0, %w1"
1191   [(set_attr "type" "neon_move,neon_from_gp,neon_to_gp,neon_move,\
1192                      f_loads,f_stores,load1,store1,mov_reg")
1193    (set_attr "simd" "yes,yes,yes,yes,*,*,*,*,*")]
1194 )
1195
1196 (define_insn "*movsf_aarch64"
1197   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w,m,r,m ,r")
1198         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,m,w,m,rY,r"))]
1199   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1200     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1201   "@
1202    movi\\t%0.2s, #0
1203    fmov\\t%s0, %w1
1204    fmov\\t%w0, %s1
1205    fmov\\t%s0, %s1
1206    fmov\\t%s0, %1
1207    ldr\\t%s0, %1
1208    str\\t%s1, %0
1209    ldr\\t%w0, %1
1210    str\\t%w1, %0
1211    mov\\t%w0, %w1"
1212   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,\
1213                      f_loads,f_stores,load1,store1,mov_reg")
1214    (set_attr "simd" "yes,*,*,*,*,*,*,*,*,*")]
1215 )
1216
1217 (define_insn "*movdf_aarch64"
1218   [(set (match_operand:DF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w,m,r,m ,r")
1219         (match_operand:DF 1 "general_operand"      "Y ,?rY, w,w,Ufc,m,w,m,rY,r"))]
1220   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1221     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1222   "@
1223    movi\\t%d0, #0
1224    fmov\\t%d0, %x1
1225    fmov\\t%x0, %d1
1226    fmov\\t%d0, %d1
1227    fmov\\t%d0, %1
1228    ldr\\t%d0, %1
1229    str\\t%d1, %0
1230    ldr\\t%x0, %1
1231    str\\t%x1, %0
1232    mov\\t%x0, %x1"
1233   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,\
1234                      f_loadd,f_stored,load1,store1,mov_reg")
1235    (set_attr "simd" "yes,*,*,*,*,*,*,*,*,*")]
1236 )
1237
1238 (define_insn "*movtf_aarch64"
1239   [(set (match_operand:TF 0
1240          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r ,Ump,Ump")
1241         (match_operand:TF 1
1242          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,Ump,?r ,Y"))]
1243   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1244     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1245   "@
1246    orr\\t%0.16b, %1.16b, %1.16b
1247    #
1248    #
1249    #
1250    movi\\t%0.2d, #0
1251    fmov\\t%s0, wzr
1252    ldr\\t%q0, %1
1253    str\\t%q1, %0
1254    ldp\\t%0, %H0, %1
1255    stp\\t%1, %H1, %0
1256    stp\\txzr, xzr, %0"
1257   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1258                      f_loadd,f_stored,load2,store2,store2")
1259    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1260    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1261 )
1262
1263 (define_split
1264    [(set (match_operand:TF 0 "register_operand" "")
1265          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1266   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1267   [(const_int 0)]
1268   {
1269     aarch64_split_128bit_move (operands[0], operands[1]);
1270     DONE;
1271   }
1272 )
1273
1274 ;; 0 is dst
1275 ;; 1 is src
1276 ;; 2 is size of move in bytes
1277 ;; 3 is alignment
1278
1279 (define_expand "movmemdi"
1280   [(match_operand:BLK 0 "memory_operand")
1281    (match_operand:BLK 1 "memory_operand")
1282    (match_operand:DI 2 "immediate_operand")
1283    (match_operand:DI 3 "immediate_operand")]
1284    "!STRICT_ALIGNMENT"
1285 {
1286   if (aarch64_expand_movmem (operands))
1287     DONE;
1288   FAIL;
1289 }
1290 )
1291
1292 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1293 ;; fairly lax checking on the second memory operation.
1294 (define_insn "load_pairsi"
1295   [(set (match_operand:SI 0 "register_operand" "=r,*w")
1296         (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1297    (set (match_operand:SI 2 "register_operand" "=r,*w")
1298         (match_operand:SI 3 "memory_operand" "m,m"))]
1299   "rtx_equal_p (XEXP (operands[3], 0),
1300                 plus_constant (Pmode,
1301                                XEXP (operands[1], 0),
1302                                GET_MODE_SIZE (SImode)))"
1303   "@
1304    ldp\\t%w0, %w2, %1
1305    ldp\\t%s0, %s2, %1"
1306   [(set_attr "type" "load2,neon_load1_2reg")
1307    (set_attr "fp" "*,yes")]
1308 )
1309
1310 (define_insn "load_pairdi"
1311   [(set (match_operand:DI 0 "register_operand" "=r,*w")
1312         (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1313    (set (match_operand:DI 2 "register_operand" "=r,*w")
1314         (match_operand:DI 3 "memory_operand" "m,m"))]
1315   "rtx_equal_p (XEXP (operands[3], 0),
1316                 plus_constant (Pmode,
1317                                XEXP (operands[1], 0),
1318                                GET_MODE_SIZE (DImode)))"
1319   "@
1320    ldp\\t%x0, %x2, %1
1321    ldp\\t%d0, %d2, %1"
1322   [(set_attr "type" "load2,neon_load1_2reg")
1323    (set_attr "fp" "*,yes")]
1324 )
1325
1326
1327 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1328 ;; fairly lax checking on the second memory operation.
1329 (define_insn "store_pairsi"
1330   [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1331         (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1332    (set (match_operand:SI 2 "memory_operand" "=m,m")
1333         (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1334   "rtx_equal_p (XEXP (operands[2], 0),
1335                 plus_constant (Pmode,
1336                                XEXP (operands[0], 0),
1337                                GET_MODE_SIZE (SImode)))"
1338   "@
1339    stp\\t%w1, %w3, %0
1340    stp\\t%s1, %s3, %0"
1341   [(set_attr "type" "store2,neon_store1_2reg")
1342    (set_attr "fp" "*,yes")]
1343 )
1344
1345 (define_insn "store_pairdi"
1346   [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1347         (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1348    (set (match_operand:DI 2 "memory_operand" "=m,m")
1349         (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1350   "rtx_equal_p (XEXP (operands[2], 0),
1351                 plus_constant (Pmode,
1352                                XEXP (operands[0], 0),
1353                                GET_MODE_SIZE (DImode)))"
1354   "@
1355    stp\\t%x1, %x3, %0
1356    stp\\t%d1, %d3, %0"
1357   [(set_attr "type" "store2,neon_store1_2reg")
1358    (set_attr "fp" "*,yes")]
1359 )
1360
1361 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1362 ;; fairly lax checking on the second memory operation.
1363 (define_insn "load_pairsf"
1364   [(set (match_operand:SF 0 "register_operand" "=w,*r")
1365         (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1366    (set (match_operand:SF 2 "register_operand" "=w,*r")
1367         (match_operand:SF 3 "memory_operand" "m,m"))]
1368   "rtx_equal_p (XEXP (operands[3], 0),
1369                 plus_constant (Pmode,
1370                                XEXP (operands[1], 0),
1371                                GET_MODE_SIZE (SFmode)))"
1372   "@
1373    ldp\\t%s0, %s2, %1
1374    ldp\\t%w0, %w2, %1"
1375   [(set_attr "type" "neon_load1_2reg,load2")
1376    (set_attr "fp" "yes,*")]
1377 )
1378
1379 (define_insn "load_pairdf"
1380   [(set (match_operand:DF 0 "register_operand" "=w,*r")
1381         (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1382    (set (match_operand:DF 2 "register_operand" "=w,*r")
1383         (match_operand:DF 3 "memory_operand" "m,m"))]
1384   "rtx_equal_p (XEXP (operands[3], 0),
1385                 plus_constant (Pmode,
1386                                XEXP (operands[1], 0),
1387                                GET_MODE_SIZE (DFmode)))"
1388   "@
1389    ldp\\t%d0, %d2, %1
1390    ldp\\t%x0, %x2, %1"
1391   [(set_attr "type" "neon_load1_2reg,load2")
1392    (set_attr "fp" "yes,*")]
1393 )
1394
1395 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1396 ;; fairly lax checking on the second memory operation.
1397 (define_insn "store_pairsf"
1398   [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1399         (match_operand:SF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1400    (set (match_operand:SF 2 "memory_operand" "=m,m")
1401         (match_operand:SF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1402   "rtx_equal_p (XEXP (operands[2], 0),
1403                 plus_constant (Pmode,
1404                                XEXP (operands[0], 0),
1405                                GET_MODE_SIZE (SFmode)))"
1406   "@
1407    stp\\t%s1, %s3, %0
1408    stp\\t%w1, %w3, %0"
1409   [(set_attr "type" "neon_store1_2reg,store2")
1410    (set_attr "fp" "yes,*")]
1411 )
1412
1413 (define_insn "store_pairdf"
1414   [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1415         (match_operand:DF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1416    (set (match_operand:DF 2 "memory_operand" "=m,m")
1417         (match_operand:DF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1418   "rtx_equal_p (XEXP (operands[2], 0),
1419                 plus_constant (Pmode,
1420                                XEXP (operands[0], 0),
1421                                GET_MODE_SIZE (DFmode)))"
1422   "@
1423    stp\\t%d1, %d3, %0
1424    stp\\t%x1, %x3, %0"
1425   [(set_attr "type" "neon_store1_2reg,store2")
1426    (set_attr "fp" "yes,*")]
1427 )
1428
1429 ;; Load pair with post-index writeback.  This is primarily used in function
1430 ;; epilogues.
1431 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1432   [(parallel
1433     [(set (match_operand:P 0 "register_operand" "=k")
1434           (plus:P (match_operand:P 1 "register_operand" "0")
1435                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1436      (set (match_operand:GPI 2 "register_operand" "=r")
1437           (mem:GPI (match_dup 1)))
1438      (set (match_operand:GPI 3 "register_operand" "=r")
1439           (mem:GPI (plus:P (match_dup 1)
1440                    (match_operand:P 5 "const_int_operand" "n"))))])]
1441   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1442   "ldp\\t%<w>2, %<w>3, [%1], %4"
1443   [(set_attr "type" "load2")]
1444 )
1445
1446 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1447   [(parallel
1448     [(set (match_operand:P 0 "register_operand" "=k")
1449           (plus:P (match_operand:P 1 "register_operand" "0")
1450                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1451      (set (match_operand:GPF 2 "register_operand" "=w")
1452           (mem:GPF (match_dup 1)))
1453      (set (match_operand:GPF 3 "register_operand" "=w")
1454           (mem:GPF (plus:P (match_dup 1)
1455                    (match_operand:P 5 "const_int_operand" "n"))))])]
1456   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1457   "ldp\\t%<w>2, %<w>3, [%1], %4"
1458   [(set_attr "type" "neon_load1_2reg")]
1459 )
1460
1461 ;; Store pair with pre-index writeback.  This is primarily used in function
1462 ;; prologues.
1463 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1464   [(parallel
1465     [(set (match_operand:P 0 "register_operand" "=&k")
1466           (plus:P (match_operand:P 1 "register_operand" "0")
1467                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1468      (set (mem:GPI (plus:P (match_dup 0)
1469                    (match_dup 4)))
1470           (match_operand:GPI 2 "register_operand" "r"))
1471      (set (mem:GPI (plus:P (match_dup 0)
1472                    (match_operand:P 5 "const_int_operand" "n")))
1473           (match_operand:GPI 3 "register_operand" "r"))])]
1474   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1475   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1476   [(set_attr "type" "store2")]
1477 )
1478
1479 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1480   [(parallel
1481     [(set (match_operand:P 0 "register_operand" "=&k")
1482           (plus:P (match_operand:P 1 "register_operand" "0")
1483                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1484      (set (mem:GPF (plus:P (match_dup 0)
1485                    (match_dup 4)))
1486           (match_operand:GPF 2 "register_operand" "w"))
1487      (set (mem:GPF (plus:P (match_dup 0)
1488                    (match_operand:P 5 "const_int_operand" "n")))
1489           (match_operand:GPF 3 "register_operand" "w"))])]
1490   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1491   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1492   [(set_attr "type" "neon_store1_2reg<q>")]
1493 )
1494
1495 ;; -------------------------------------------------------------------
1496 ;; Sign/Zero extension
1497 ;; -------------------------------------------------------------------
1498
1499 (define_expand "<optab>sidi2"
1500   [(set (match_operand:DI 0 "register_operand")
1501         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1502   ""
1503 )
1504
1505 (define_insn "*extendsidi2_aarch64"
1506   [(set (match_operand:DI 0 "register_operand" "=r,r")
1507         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1508   ""
1509   "@
1510    sxtw\t%0, %w1
1511    ldrsw\t%0, %1"
1512   [(set_attr "type" "extend,load1")]
1513 )
1514
1515 (define_insn "*load_pair_extendsidi2_aarch64"
1516   [(set (match_operand:DI 0 "register_operand" "=r")
1517         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1518    (set (match_operand:DI 2 "register_operand" "=r")
1519         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1520   "rtx_equal_p (XEXP (operands[3], 0),
1521                 plus_constant (Pmode,
1522                                XEXP (operands[1], 0),
1523                                GET_MODE_SIZE (SImode)))"
1524   "ldpsw\\t%0, %2, %1"
1525   [(set_attr "type" "load2")]
1526 )
1527
1528 (define_insn "*zero_extendsidi2_aarch64"
1529   [(set (match_operand:DI 0 "register_operand" "=r,r")
1530         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1531   ""
1532   "@
1533    uxtw\t%0, %w1
1534    ldr\t%w0, %1"
1535   [(set_attr "type" "extend,load1")]
1536 )
1537
1538 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1539   [(set (match_operand:DI 0 "register_operand" "=r")
1540         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1541    (set (match_operand:DI 2 "register_operand" "=r")
1542         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1543   "rtx_equal_p (XEXP (operands[3], 0),
1544                 plus_constant (Pmode,
1545                                XEXP (operands[1], 0),
1546                                GET_MODE_SIZE (SImode)))"
1547   "ldp\\t%w0, %w2, %1"
1548   [(set_attr "type" "load2")]
1549 )
1550
1551 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1552   [(set (match_operand:GPI 0 "register_operand")
1553         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1554   ""
1555 )
1556
1557 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1558   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1559         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1560   ""
1561   "@
1562    sxt<SHORT:size>\t%<GPI:w>0, %w1
1563    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1564   [(set_attr "type" "extend,load1")]
1565 )
1566
1567 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1568   [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1569         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1570   ""
1571   "@
1572    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1573    ldr<SHORT:size>\t%w0, %1
1574    ldr\t%<SHORT:size>0, %1"
1575   [(set_attr "type" "logic_imm,load1,load1")]
1576 )
1577
1578 (define_expand "<optab>qihi2"
1579   [(set (match_operand:HI 0 "register_operand")
1580         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1581   ""
1582 )
1583
1584 (define_insn "*extendqihi2_aarch64"
1585   [(set (match_operand:HI 0 "register_operand" "=r,r")
1586         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1587   ""
1588   "@
1589    sxtb\t%w0, %w1
1590    ldrsb\t%w0, %1"
1591   [(set_attr "type" "extend,load1")]
1592 )
1593
1594 (define_insn "*zero_extendqihi2_aarch64"
1595   [(set (match_operand:HI 0 "register_operand" "=r,r")
1596         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1597   ""
1598   "@
1599    and\t%w0, %w1, 255
1600    ldrb\t%w0, %1"
1601   [(set_attr "type" "logic_imm,load1")]
1602 )
1603
1604 ;; -------------------------------------------------------------------
1605 ;; Simple arithmetic
1606 ;; -------------------------------------------------------------------
1607
1608 (define_expand "add<mode>3"
1609   [(set
1610     (match_operand:GPI 0 "register_operand" "")
1611     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1612               (match_operand:GPI 2 "aarch64_pluslong_operand" "")))]
1613   ""
1614 {
1615   /* If operands[1] is a subreg extract the inner RTX.  */
1616   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1617
1618   /* If the constant is too large for a single instruction and isn't frame
1619      based, split off the immediate so it is available for CSE.  */
1620   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1621       && can_create_pseudo_p ()
1622       && (!REG_P (op1)
1623          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1624     operands[2] = force_reg (<MODE>mode, operands[2]);
1625 })
1626
1627 (define_insn "*add<mode>3_aarch64"
1628   [(set
1629     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r")
1630     (plus:GPI
1631      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk")
1632      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Upl")))]
1633   ""
1634   "@
1635   add\\t%<w>0, %<w>1, %2
1636   add\\t%<w>0, %<w>1, %<w>2
1637   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1638   sub\\t%<w>0, %<w>1, #%n2
1639   #"
1640   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple")
1641    (set_attr "simd" "*,*,yes,*,*")]
1642 )
1643
1644 ;; zero_extend version of above
1645 (define_insn "*addsi3_aarch64_uxtw"
1646   [(set
1647     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1648     (zero_extend:DI
1649      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1650               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Upl"))))]
1651   ""
1652   "@
1653   add\\t%w0, %w1, %2
1654   add\\t%w0, %w1, %w2
1655   sub\\t%w0, %w1, #%n2
1656   #"
1657   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1658 )
1659
1660 ;; If there's a free register, and we can load the constant with a
1661 ;; single instruction, do so.  This has a chance to improve scheduling.
1662 (define_peephole2
1663   [(match_scratch:GPI 3 "r")
1664    (set (match_operand:GPI 0 "register_operand")
1665         (plus:GPI
1666           (match_operand:GPI 1 "register_operand")
1667           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1668   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1669   [(set (match_dup 3) (match_dup 2))
1670    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1671 )
1672
1673 (define_peephole2
1674   [(match_scratch:SI 3 "r")
1675    (set (match_operand:DI 0 "register_operand")
1676         (zero_extend:DI
1677           (plus:SI
1678             (match_operand:SI 1 "register_operand")
1679             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1680   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1681   [(set (match_dup 3) (match_dup 2))
1682    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1683 )
1684
1685 ;; After peephole2 has had a chance to run, split any remaining long
1686 ;; additions into two add immediates.
1687 (define_split
1688   [(set (match_operand:GPI 0 "register_operand")
1689         (plus:GPI
1690           (match_operand:GPI 1 "register_operand")
1691           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1692   "epilogue_completed"
1693   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1694    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1695   {
1696     HOST_WIDE_INT i = INTVAL (operands[2]);
1697     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1698     operands[3] = GEN_INT (i - s);
1699     operands[4] = GEN_INT (s);
1700   }
1701 )
1702
1703 (define_split
1704   [(set (match_operand:DI 0 "register_operand")
1705         (zero_extend:DI
1706           (plus:SI
1707             (match_operand:SI 1 "register_operand")
1708             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1709   "epilogue_completed"
1710   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1711    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1712   {
1713     HOST_WIDE_INT i = INTVAL (operands[2]);
1714     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1715     operands[3] = GEN_INT (i - s);
1716     operands[4] = GEN_INT (s);
1717     operands[5] = gen_lowpart (SImode, operands[0]);
1718   }
1719 )
1720
1721 (define_expand "addti3"
1722   [(set (match_operand:TI 0 "register_operand" "")
1723         (plus:TI (match_operand:TI 1 "register_operand" "")
1724                  (match_operand:TI 2 "register_operand" "")))]
1725   ""
1726 {
1727   rtx low = gen_reg_rtx (DImode);
1728   emit_insn (gen_adddi3_compareC (low, gen_lowpart (DImode, operands[1]),
1729                                   gen_lowpart (DImode, operands[2])));
1730
1731   rtx high = gen_reg_rtx (DImode);
1732   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1733                                  gen_highpart (DImode, operands[2])));
1734
1735   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1736   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1737   DONE;
1738 })
1739
1740 (define_insn "add<mode>3_compare0"
1741   [(set (reg:CC_NZ CC_REGNUM)
1742         (compare:CC_NZ
1743          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1744                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1745          (const_int 0)))
1746    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1747         (plus:GPI (match_dup 1) (match_dup 2)))]
1748   ""
1749   "@
1750   adds\\t%<w>0, %<w>1, %<w>2
1751   adds\\t%<w>0, %<w>1, %2
1752   subs\\t%<w>0, %<w>1, #%n2"
1753   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1754 )
1755
1756 ;; zero_extend version of above
1757 (define_insn "*addsi3_compare0_uxtw"
1758   [(set (reg:CC_NZ CC_REGNUM)
1759         (compare:CC_NZ
1760          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1761                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1762          (const_int 0)))
1763    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1764         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1765   ""
1766   "@
1767   adds\\t%w0, %w1, %w2
1768   adds\\t%w0, %w1, %2
1769   subs\\t%w0, %w1, #%n2"
1770   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1771 )
1772
1773 (define_insn "*add<mode>3_compareC_cconly_imm"
1774   [(set (reg:CC_C CC_REGNUM)
1775         (ne:CC_C
1776           (plus:<DWI>
1777             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1778             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1779           (zero_extend:<DWI>
1780             (plus:GPI
1781               (match_dup 0)
1782               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1783   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1784                                  <MODE>mode, operands[1])"
1785   "@
1786   cmn\\t%<w>0, %1
1787   cmp\\t%<w>0, #%n1"
1788   [(set_attr "type" "alus_imm")]
1789 )
1790
1791 (define_insn "*add<mode>3_compareC_cconly"
1792   [(set (reg:CC_C CC_REGNUM)
1793         (ne:CC_C
1794           (plus:<DWI>
1795             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1796             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1797           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1798   ""
1799   "cmn\\t%<w>0, %<w>1"
1800   [(set_attr "type" "alus_sreg")]
1801 )
1802
1803 (define_insn "*add<mode>3_compareC_imm"
1804   [(set (reg:CC_C CC_REGNUM)
1805         (ne:CC_C
1806           (plus:<DWI>
1807             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1808             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1809           (zero_extend:<DWI>
1810             (plus:GPI
1811               (match_dup 1)
1812               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1813    (set (match_operand:GPI 0 "register_operand" "=r,r")
1814         (plus:GPI (match_dup 1) (match_dup 2)))]
1815   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1816                                  <MODE>mode, operands[2])"
1817   "@
1818   adds\\t%<w>0, %<w>1, %2
1819   subs\\t%<w>0, %<w>1, #%n2"
1820   [(set_attr "type" "alus_imm")]
1821 )
1822
1823 (define_insn "add<mode>3_compareC"
1824   [(set (reg:CC_C CC_REGNUM)
1825         (ne:CC_C
1826           (plus:<DWI>
1827             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1828             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1829           (zero_extend:<DWI>
1830             (plus:GPI (match_dup 1) (match_dup 2)))))
1831    (set (match_operand:GPI 0 "register_operand" "=r")
1832         (plus:GPI (match_dup 1) (match_dup 2)))]
1833   ""
1834   "adds\\t%<w>0, %<w>1, %<w>2"
1835   [(set_attr "type" "alus_sreg")]
1836 )
1837
1838 (define_insn "*adds_shift_imm_<mode>"
1839   [(set (reg:CC_NZ CC_REGNUM)
1840         (compare:CC_NZ
1841          (plus:GPI (ASHIFT:GPI
1842                     (match_operand:GPI 1 "register_operand" "r")
1843                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1844                    (match_operand:GPI 3 "register_operand" "r"))
1845          (const_int 0)))
1846    (set (match_operand:GPI 0 "register_operand" "=r")
1847         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1848                   (match_dup 3)))]
1849   ""
1850   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1851   [(set_attr "type" "alus_shift_imm")]
1852 )
1853
1854 (define_insn "*subs_shift_imm_<mode>"
1855   [(set (reg:CC_NZ CC_REGNUM)
1856         (compare:CC_NZ
1857          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1858                     (ASHIFT:GPI
1859                      (match_operand:GPI 2 "register_operand" "r")
1860                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1861          (const_int 0)))
1862    (set (match_operand:GPI 0 "register_operand" "=r")
1863         (minus:GPI (match_dup 1)
1864                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1865   ""
1866   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1867   [(set_attr "type" "alus_shift_imm")]
1868 )
1869
1870 (define_insn "*adds_mul_imm_<mode>"
1871   [(set (reg:CC_NZ CC_REGNUM)
1872         (compare:CC_NZ
1873          (plus:GPI (mult:GPI
1874                     (match_operand:GPI 1 "register_operand" "r")
1875                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1876                    (match_operand:GPI 3 "register_operand" "r"))
1877          (const_int 0)))
1878    (set (match_operand:GPI 0 "register_operand" "=r")
1879         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1880                   (match_dup 3)))]
1881   ""
1882   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1883   [(set_attr "type" "alus_shift_imm")]
1884 )
1885
1886 (define_insn "*subs_mul_imm_<mode>"
1887   [(set (reg:CC_NZ CC_REGNUM)
1888         (compare:CC_NZ
1889          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1890                     (mult:GPI
1891                      (match_operand:GPI 2 "register_operand" "r")
1892                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1893          (const_int 0)))
1894    (set (match_operand:GPI 0 "register_operand" "=r")
1895         (minus:GPI (match_dup 1)
1896                    (mult:GPI (match_dup 2) (match_dup 3))))]
1897   ""
1898   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1899   [(set_attr "type" "alus_shift_imm")]
1900 )
1901
1902 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1903   [(set (reg:CC_NZ CC_REGNUM)
1904         (compare:CC_NZ
1905          (plus:GPI
1906           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1907           (match_operand:GPI 2 "register_operand" "r"))
1908         (const_int 0)))
1909    (set (match_operand:GPI 0 "register_operand" "=r")
1910         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1911   ""
1912   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1913   [(set_attr "type" "alus_ext")]
1914 )
1915
1916 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1917   [(set (reg:CC_NZ CC_REGNUM)
1918         (compare:CC_NZ
1919          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1920                     (ANY_EXTEND:GPI
1921                      (match_operand:ALLX 2 "register_operand" "r")))
1922         (const_int 0)))
1923    (set (match_operand:GPI 0 "register_operand" "=r")
1924         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1925   ""
1926   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1927   [(set_attr "type" "alus_ext")]
1928 )
1929
1930 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1931   [(set (reg:CC_NZ CC_REGNUM)
1932         (compare:CC_NZ
1933          (plus:GPI (ashift:GPI
1934                     (ANY_EXTEND:GPI
1935                      (match_operand:ALLX 1 "register_operand" "r"))
1936                     (match_operand 2 "aarch64_imm3" "Ui3"))
1937                    (match_operand:GPI 3 "register_operand" "r"))
1938          (const_int 0)))
1939    (set (match_operand:GPI 0 "register_operand" "=rk")
1940         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1941                               (match_dup 2))
1942                   (match_dup 3)))]
1943   ""
1944   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1945   [(set_attr "type" "alus_ext")]
1946 )
1947
1948 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1949   [(set (reg:CC_NZ CC_REGNUM)
1950         (compare:CC_NZ
1951          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1952                     (ashift:GPI
1953                      (ANY_EXTEND:GPI
1954                       (match_operand:ALLX 2 "register_operand" "r"))
1955                      (match_operand 3 "aarch64_imm3" "Ui3")))
1956          (const_int 0)))
1957    (set (match_operand:GPI 0 "register_operand" "=rk")
1958         (minus:GPI (match_dup 1)
1959                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
1960                                (match_dup 3))))]
1961   ""
1962   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
1963   [(set_attr "type" "alus_ext")]
1964 )
1965
1966 (define_insn "*adds_<optab><mode>_multp2"
1967   [(set (reg:CC_NZ CC_REGNUM)
1968         (compare:CC_NZ
1969          (plus:GPI (ANY_EXTRACT:GPI
1970                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1971                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1972                     (match_operand 3 "const_int_operand" "n")
1973                     (const_int 0))
1974                    (match_operand:GPI 4 "register_operand" "r"))
1975         (const_int 0)))
1976    (set (match_operand:GPI 0 "register_operand" "=r")
1977         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
1978                                    (match_dup 3)
1979                                    (const_int 0))
1980                   (match_dup 4)))]
1981   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1982   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1983   [(set_attr "type" "alus_ext")]
1984 )
1985
1986 (define_insn "*subs_<optab><mode>_multp2"
1987   [(set (reg:CC_NZ CC_REGNUM)
1988         (compare:CC_NZ
1989          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
1990                     (ANY_EXTRACT:GPI
1991                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1992                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1993                      (match_operand 3 "const_int_operand" "n")
1994                      (const_int 0)))
1995         (const_int 0)))
1996    (set (match_operand:GPI 0 "register_operand" "=r")
1997         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
1998                                   (mult:GPI (match_dup 1) (match_dup 2))
1999                                   (match_dup 3)
2000                                   (const_int 0))))]
2001   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2002   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2003   [(set_attr "type" "alus_ext")]
2004 )
2005
2006 (define_insn "*add<mode>3nr_compare0"
2007   [(set (reg:CC_NZ CC_REGNUM)
2008         (compare:CC_NZ
2009          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2010                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2011          (const_int 0)))]
2012   ""
2013   "@
2014   cmn\\t%<w>0, %<w>1
2015   cmn\\t%<w>0, %1
2016   cmp\\t%<w>0, #%n1"
2017   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2018 )
2019
2020 (define_insn "*compare_neg<mode>"
2021   [(set (reg:CC_Z CC_REGNUM)
2022         (compare:CC_Z
2023          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2024          (match_operand:GPI 1 "register_operand" "r")))]
2025   ""
2026   "cmn\\t%<w>1, %<w>0"
2027   [(set_attr "type" "alus_sreg")]
2028 )
2029
2030 (define_insn "*add_<shift>_<mode>"
2031   [(set (match_operand:GPI 0 "register_operand" "=r")
2032         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2033                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2034                   (match_operand:GPI 3 "register_operand" "r")))]
2035   ""
2036   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2037   [(set_attr "type" "alu_shift_imm")]
2038 )
2039
2040 ;; zero_extend version of above
2041 (define_insn "*add_<shift>_si_uxtw"
2042   [(set (match_operand:DI 0 "register_operand" "=r")
2043         (zero_extend:DI
2044          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2045                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2046                   (match_operand:SI 3 "register_operand" "r"))))]
2047   ""
2048   "add\\t%w0, %w3, %w1, <shift> %2"
2049   [(set_attr "type" "alu_shift_imm")]
2050 )
2051
2052 (define_insn "*add_mul_imm_<mode>"
2053   [(set (match_operand:GPI 0 "register_operand" "=r")
2054         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2055                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2056                   (match_operand:GPI 3 "register_operand" "r")))]
2057   ""
2058   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2059   [(set_attr "type" "alu_shift_imm")]
2060 )
2061
2062 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2063   [(set (match_operand:GPI 0 "register_operand" "=rk")
2064         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2065                   (match_operand:GPI 2 "register_operand" "r")))]
2066   ""
2067   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2068   [(set_attr "type" "alu_ext")]
2069 )
2070
2071 ;; zero_extend version of above
2072 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2073   [(set (match_operand:DI 0 "register_operand" "=rk")
2074         (zero_extend:DI
2075          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2076                   (match_operand:GPI 2 "register_operand" "r"))))]
2077   ""
2078   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2079   [(set_attr "type" "alu_ext")]
2080 )
2081
2082 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2083   [(set (match_operand:GPI 0 "register_operand" "=rk")
2084         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2085                                (match_operand:ALLX 1 "register_operand" "r"))
2086                               (match_operand 2 "aarch64_imm3" "Ui3"))
2087                   (match_operand:GPI 3 "register_operand" "r")))]
2088   ""
2089   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2090   [(set_attr "type" "alu_ext")]
2091 )
2092
2093 ;; zero_extend version of above
2094 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2095   [(set (match_operand:DI 0 "register_operand" "=rk")
2096         (zero_extend:DI
2097          (plus:SI (ashift:SI (ANY_EXTEND:SI
2098                               (match_operand:SHORT 1 "register_operand" "r"))
2099                              (match_operand 2 "aarch64_imm3" "Ui3"))
2100                   (match_operand:SI 3 "register_operand" "r"))))]
2101   ""
2102   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2103   [(set_attr "type" "alu_ext")]
2104 )
2105
2106 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2107   [(set (match_operand:GPI 0 "register_operand" "=rk")
2108         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2109                              (match_operand:ALLX 1 "register_operand" "r"))
2110                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2111                   (match_operand:GPI 3 "register_operand" "r")))]
2112   ""
2113   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2114   [(set_attr "type" "alu_ext")]
2115 )
2116
2117 ;; zero_extend version of above
2118 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2119   [(set (match_operand:DI 0 "register_operand" "=rk")
2120         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2121                              (match_operand:SHORT 1 "register_operand" "r"))
2122                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2123                   (match_operand:SI 3 "register_operand" "r"))))]
2124   ""
2125   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2126   [(set_attr "type" "alu_ext")]
2127 )
2128
2129 (define_insn "*add_<optab><mode>_multp2"
2130   [(set (match_operand:GPI 0 "register_operand" "=rk")
2131         (plus:GPI (ANY_EXTRACT:GPI
2132                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2133                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2134                    (match_operand 3 "const_int_operand" "n")
2135                    (const_int 0))
2136                   (match_operand:GPI 4 "register_operand" "r")))]
2137   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2138   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2139   [(set_attr "type" "alu_ext")]
2140 )
2141
2142 ;; zero_extend version of above
2143 (define_insn "*add_<optab>si_multp2_uxtw"
2144   [(set (match_operand:DI 0 "register_operand" "=rk")
2145         (zero_extend:DI
2146          (plus:SI (ANY_EXTRACT:SI
2147                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2148                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2149                    (match_operand 3 "const_int_operand" "n")
2150                    (const_int 0))
2151                   (match_operand:SI 4 "register_operand" "r"))))]
2152   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2153   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2154   [(set_attr "type" "alu_ext")]
2155 )
2156
2157 (define_expand "add<mode>3_carryin"
2158   [(set (match_operand:GPI 0 "register_operand")
2159         (plus:GPI
2160           (plus:GPI
2161             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2162             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2163           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2164    ""
2165    ""
2166 )
2167
2168 ;; Note that add with carry with two zero inputs is matched by cset,
2169 ;; and that add with carry with one zero input is matched by cinc.
2170
2171 (define_insn "*add<mode>3_carryin"
2172   [(set (match_operand:GPI 0 "register_operand" "=r")
2173         (plus:GPI
2174           (plus:GPI
2175             (match_operand:GPI 3 "aarch64_carry_operation" "")
2176             (match_operand:GPI 1 "register_operand" "r"))
2177           (match_operand:GPI 2 "register_operand" "r")))]
2178    ""
2179    "adc\\t%<w>0, %<w>1, %<w>2"
2180   [(set_attr "type" "adc_reg")]
2181 )
2182
2183 ;; zero_extend version of above
2184 (define_insn "*addsi3_carryin_uxtw"
2185   [(set (match_operand:DI 0 "register_operand" "=r")
2186         (zero_extend:DI
2187           (plus:SI
2188             (plus:SI
2189               (match_operand:SI 3 "aarch64_carry_operation" "")
2190               (match_operand:SI 1 "register_operand" "r"))
2191             (match_operand:SI 2 "register_operand" "r"))))]
2192    ""
2193    "adc\\t%w0, %w1, %w2"
2194   [(set_attr "type" "adc_reg")]
2195 )
2196
2197 (define_insn "*add_uxt<mode>_shift2"
2198   [(set (match_operand:GPI 0 "register_operand" "=rk")
2199         (plus:GPI (and:GPI
2200                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2201                                (match_operand 2 "aarch64_imm3" "Ui3"))
2202                    (match_operand 3 "const_int_operand" "n"))
2203                   (match_operand:GPI 4 "register_operand" "r")))]
2204   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2205   "*
2206   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2207                                            INTVAL (operands[3])));
2208   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2209   [(set_attr "type" "alu_ext")]
2210 )
2211
2212 ;; zero_extend version of above
2213 (define_insn "*add_uxtsi_shift2_uxtw"
2214   [(set (match_operand:DI 0 "register_operand" "=rk")
2215         (zero_extend:DI
2216          (plus:SI (and:SI
2217                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2218                               (match_operand 2 "aarch64_imm3" "Ui3"))
2219                    (match_operand 3 "const_int_operand" "n"))
2220                   (match_operand:SI 4 "register_operand" "r"))))]
2221   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2222   "*
2223   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2224                                            INTVAL (operands[3])));
2225   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2226   [(set_attr "type" "alu_ext")]
2227 )
2228
2229 (define_insn "*add_uxt<mode>_multp2"
2230   [(set (match_operand:GPI 0 "register_operand" "=rk")
2231         (plus:GPI (and:GPI
2232                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2233                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2234                    (match_operand 3 "const_int_operand" "n"))
2235                   (match_operand:GPI 4 "register_operand" "r")))]
2236   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2237   "*
2238   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2239                                            INTVAL (operands[3])));
2240   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2241   [(set_attr "type" "alu_ext")]
2242 )
2243
2244 ;; zero_extend version of above
2245 (define_insn "*add_uxtsi_multp2_uxtw"
2246   [(set (match_operand:DI 0 "register_operand" "=rk")
2247         (zero_extend:DI
2248          (plus:SI (and:SI
2249                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2250                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2251                    (match_operand 3 "const_int_operand" "n"))
2252                   (match_operand:SI 4 "register_operand" "r"))))]
2253   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2254   "*
2255   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2256                                            INTVAL (operands[3])));
2257   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2258   [(set_attr "type" "alu_ext")]
2259 )
2260
2261 (define_insn "subsi3"
2262   [(set (match_operand:SI 0 "register_operand" "=rk")
2263         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2264                   (match_operand:SI 2 "register_operand" "r")))]
2265   ""
2266   "sub\\t%w0, %w1, %w2"
2267   [(set_attr "type" "alu_sreg")]
2268 )
2269
2270 ;; zero_extend version of above
2271 (define_insn "*subsi3_uxtw"
2272   [(set (match_operand:DI 0 "register_operand" "=rk")
2273         (zero_extend:DI
2274          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2275                    (match_operand:SI 2 "register_operand" "r"))))]
2276   ""
2277   "sub\\t%w0, %w1, %w2"
2278   [(set_attr "type" "alu_sreg")]
2279 )
2280
2281 (define_insn "subdi3"
2282   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2283         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2284                   (match_operand:DI 2 "register_operand" "r,w")))]
2285   ""
2286   "@
2287    sub\\t%x0, %x1, %x2
2288    sub\\t%d0, %d1, %d2"
2289   [(set_attr "type" "alu_sreg, neon_sub")
2290    (set_attr "simd" "*,yes")]
2291 )
2292
2293 (define_expand "subti3"
2294   [(set (match_operand:TI 0 "register_operand" "")
2295         (minus:TI (match_operand:TI 1 "register_operand" "")
2296                   (match_operand:TI 2 "register_operand" "")))]
2297   ""
2298 {
2299   rtx low = gen_reg_rtx (DImode);
2300   emit_insn (gen_subdi3_compare1 (low, gen_lowpart (DImode, operands[1]),
2301                                   gen_lowpart (DImode, operands[2])));
2302
2303   rtx high = gen_reg_rtx (DImode);
2304   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2305                                  gen_highpart (DImode, operands[2])));
2306
2307   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2308   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2309   DONE;
2310 })
2311
2312 (define_insn "*sub<mode>3_compare0"
2313   [(set (reg:CC_NZ CC_REGNUM)
2314         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2315                                   (match_operand:GPI 2 "register_operand" "r"))
2316                        (const_int 0)))
2317    (set (match_operand:GPI 0 "register_operand" "=r")
2318         (minus:GPI (match_dup 1) (match_dup 2)))]
2319   ""
2320   "subs\\t%<w>0, %<w>1, %<w>2"
2321   [(set_attr "type" "alus_sreg")]
2322 )
2323
2324 ;; zero_extend version of above
2325 (define_insn "*subsi3_compare0_uxtw"
2326   [(set (reg:CC_NZ CC_REGNUM)
2327         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2328                                  (match_operand:SI 2 "register_operand" "r"))
2329                        (const_int 0)))
2330    (set (match_operand:DI 0 "register_operand" "=r")
2331         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2332   ""
2333   "subs\\t%w0, %w1, %w2"
2334   [(set_attr "type" "alus_sreg")]
2335 )
2336
2337 (define_insn "sub<mode>3_compare1"
2338   [(set (reg:CC CC_REGNUM)
2339         (compare:CC
2340           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2341           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2342    (set (match_operand:GPI 0 "register_operand" "=r")
2343         (minus:GPI (match_dup 1) (match_dup 2)))]
2344   ""
2345   "subs\\t%<w>0, %<w>1, %<w>2"
2346   [(set_attr "type" "alus_sreg")]
2347 )
2348
2349 (define_insn "*sub_<shift>_<mode>"
2350   [(set (match_operand:GPI 0 "register_operand" "=r")
2351         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2352                    (ASHIFT:GPI
2353                     (match_operand:GPI 1 "register_operand" "r")
2354                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2355   ""
2356   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2357   [(set_attr "type" "alu_shift_imm")]
2358 )
2359
2360 ;; zero_extend version of above
2361 (define_insn "*sub_<shift>_si_uxtw"
2362   [(set (match_operand:DI 0 "register_operand" "=r")
2363         (zero_extend:DI
2364          (minus:SI (match_operand:SI 3 "register_operand" "r")
2365                    (ASHIFT:SI
2366                     (match_operand:SI 1 "register_operand" "r")
2367                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2368   ""
2369   "sub\\t%w0, %w3, %w1, <shift> %2"
2370   [(set_attr "type" "alu_shift_imm")]
2371 )
2372
2373 (define_insn "*sub_mul_imm_<mode>"
2374   [(set (match_operand:GPI 0 "register_operand" "=r")
2375         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2376                    (mult:GPI
2377                     (match_operand:GPI 1 "register_operand" "r")
2378                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2379   ""
2380   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2381   [(set_attr "type" "alu_shift_imm")]
2382 )
2383
2384 ;; zero_extend version of above
2385 (define_insn "*sub_mul_imm_si_uxtw"
2386   [(set (match_operand:DI 0 "register_operand" "=r")
2387         (zero_extend:DI
2388          (minus:SI (match_operand:SI 3 "register_operand" "r")
2389                    (mult:SI
2390                     (match_operand:SI 1 "register_operand" "r")
2391                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2392   ""
2393   "sub\\t%w0, %w3, %w1, lsl %p2"
2394   [(set_attr "type" "alu_shift_imm")]
2395 )
2396
2397 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2398   [(set (match_operand:GPI 0 "register_operand" "=rk")
2399         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2400                    (ANY_EXTEND:GPI
2401                     (match_operand:ALLX 2 "register_operand" "r"))))]
2402   ""
2403   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2404   [(set_attr "type" "alu_ext")]
2405 )
2406
2407 ;; zero_extend version of above
2408 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2409   [(set (match_operand:DI 0 "register_operand" "=rk")
2410         (zero_extend:DI
2411          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2412                    (ANY_EXTEND:SI
2413                     (match_operand:SHORT 2 "register_operand" "r")))))]
2414   ""
2415   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2416   [(set_attr "type" "alu_ext")]
2417 )
2418
2419 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2420   [(set (match_operand:GPI 0 "register_operand" "=rk")
2421         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2422                    (ashift:GPI (ANY_EXTEND:GPI
2423                                 (match_operand:ALLX 2 "register_operand" "r"))
2424                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2425   ""
2426   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2427   [(set_attr "type" "alu_ext")]
2428 )
2429
2430 ;; zero_extend version of above
2431 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2432   [(set (match_operand:DI 0 "register_operand" "=rk")
2433         (zero_extend:DI
2434          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2435                    (ashift:SI (ANY_EXTEND:SI
2436                                (match_operand:SHORT 2 "register_operand" "r"))
2437                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2438   ""
2439   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2440   [(set_attr "type" "alu_ext")]
2441 )
2442
2443 (define_insn "*sub_<optab><mode>_multp2"
2444   [(set (match_operand:GPI 0 "register_operand" "=rk")
2445         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2446                    (ANY_EXTRACT:GPI
2447                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2448                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2449                     (match_operand 3 "const_int_operand" "n")
2450                     (const_int 0))))]
2451   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2452   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2453   [(set_attr "type" "alu_ext")]
2454 )
2455
2456 ;; zero_extend version of above
2457 (define_insn "*sub_<optab>si_multp2_uxtw"
2458   [(set (match_operand:DI 0 "register_operand" "=rk")
2459         (zero_extend:DI
2460          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2461                    (ANY_EXTRACT:SI
2462                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2463                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2464                     (match_operand 3 "const_int_operand" "n")
2465                     (const_int 0)))))]
2466   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2467   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2468   [(set_attr "type" "alu_ext")]
2469 )
2470
2471 ;; The hardware description is op1 + ~op2 + C.
2472 ;;                           = op1 + (-op2 + 1) + (1 - !C)
2473 ;;                           = op1 - op2 - 1 + 1 - !C
2474 ;;                           = op1 - op2 - !C.
2475 ;; We describe the latter.
2476
2477 (define_insn "*sub<mode>3_carryin0"
2478   [(set (match_operand:GPI 0 "register_operand" "=r")
2479         (minus:GPI
2480           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2481           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2482    ""
2483    "sbc\\t%<w>0, %<w>1, <w>zr"
2484   [(set_attr "type" "adc_reg")]
2485 )
2486
2487 ;; zero_extend version of the above
2488 (define_insn "*subsi3_carryin_uxtw"
2489   [(set (match_operand:DI 0 "register_operand" "=r")
2490         (zero_extend:DI
2491           (minus:SI
2492             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2493             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2494    ""
2495    "sbc\\t%w0, %w1, wzr"
2496   [(set_attr "type" "adc_reg")]
2497 )
2498
2499 (define_expand "sub<mode>3_carryin"
2500   [(set (match_operand:GPI 0 "register_operand")
2501         (minus:GPI
2502           (minus:GPI
2503             (match_operand:GPI 1 "aarch64_reg_or_zero")
2504             (match_operand:GPI 2 "register_operand"))
2505           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2506    ""
2507    ""
2508 )
2509
2510 (define_insn "*sub<mode>3_carryin"
2511   [(set (match_operand:GPI 0 "register_operand" "=r")
2512         (minus:GPI
2513           (minus:GPI
2514             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2515             (match_operand:GPI 2 "register_operand" "r"))
2516           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
2517
2518    ""
2519    "sbc\\t%<w>0, %<w>1, %<w>2"
2520   [(set_attr "type" "adc_reg")]
2521 )
2522
2523 ;; zero_extend version of the above
2524 (define_insn "*subsi3_carryin_uxtw"
2525   [(set (match_operand:DI 0 "register_operand" "=r")
2526         (zero_extend:DI
2527           (minus:SI
2528             (minus:SI
2529               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2530               (match_operand:SI 2 "register_operand" "r"))
2531             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
2532
2533    ""
2534    "sbc\\t%w0, %w1, %w2"
2535   [(set_attr "type" "adc_reg")]
2536 )
2537
2538 (define_insn "*sub<mode>3_carryin_alt"
2539   [(set (match_operand:GPI 0 "register_operand" "=r")
2540         (minus:GPI
2541           (minus:GPI
2542             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2543             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
2544           (match_operand:GPI 2 "register_operand" "r")))]
2545    ""
2546    "sbc\\t%<w>0, %<w>1, %<w>2"
2547   [(set_attr "type" "adc_reg")]
2548 )
2549
2550 ;; zero_extend version of the above
2551 (define_insn "*subsi3_carryin_alt_uxtw"
2552   [(set (match_operand:DI 0 "register_operand" "=r")
2553         (zero_extend:DI
2554           (minus:SI
2555             (minus:SI
2556               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2557               (match_operand:SI 3 "aarch64_borrow_operation" ""))
2558             (match_operand:SI 2 "register_operand" "r"))))]
2559    ""
2560    "sbc\\t%w0, %w1, %w2"
2561   [(set_attr "type" "adc_reg")]
2562 )
2563
2564 (define_insn "*sub_uxt<mode>_shift2"
2565   [(set (match_operand:GPI 0 "register_operand" "=rk")
2566         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2567                    (and:GPI
2568                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2569                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2570                     (match_operand 3 "const_int_operand" "n"))))]
2571   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2572   "*
2573   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2574                                            INTVAL (operands[3])));
2575   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2576   [(set_attr "type" "alu_ext")]
2577 )
2578
2579 ;; zero_extend version of above
2580 (define_insn "*sub_uxtsi_shift2_uxtw"
2581   [(set (match_operand:DI 0 "register_operand" "=rk")
2582         (zero_extend:DI
2583          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2584                    (and:SI
2585                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2586                                (match_operand 2 "aarch64_imm3" "Ui3"))
2587                     (match_operand 3 "const_int_operand" "n")))))]
2588   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2589   "*
2590   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2591                                            INTVAL (operands[3])));
2592   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2593   [(set_attr "type" "alu_ext")]
2594 )
2595
2596 (define_insn "*sub_uxt<mode>_multp2"
2597   [(set (match_operand:GPI 0 "register_operand" "=rk")
2598         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2599                    (and:GPI
2600                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2601                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2602                     (match_operand 3 "const_int_operand" "n"))))]
2603   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2604   "*
2605   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2606                                            INTVAL (operands[3])));
2607   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2608   [(set_attr "type" "alu_ext")]
2609 )
2610
2611 ;; zero_extend version of above
2612 (define_insn "*sub_uxtsi_multp2_uxtw"
2613   [(set (match_operand:DI 0 "register_operand" "=rk")
2614         (zero_extend:DI
2615          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2616                    (and:SI
2617                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2618                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2619                     (match_operand 3 "const_int_operand" "n")))))]
2620   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2621   "*
2622   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2623                                            INTVAL (operands[3])));
2624   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2625   [(set_attr "type" "alu_ext")]
2626 )
2627
2628 (define_expand "abs<mode>2"
2629   [(match_operand:GPI 0 "register_operand" "")
2630    (match_operand:GPI 1 "register_operand" "")]
2631   ""
2632   {
2633     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2634     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2635     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2636     DONE;
2637   }
2638 )
2639
2640 (define_insn "neg<mode>2"
2641   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2642         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2643   ""
2644   "@
2645    neg\\t%<w>0, %<w>1
2646    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2647   [(set_attr "type" "alu_sreg, neon_neg<q>")
2648    (set_attr "simd" "*,yes")]
2649 )
2650
2651 ;; zero_extend version of above
2652 (define_insn "*negsi2_uxtw"
2653   [(set (match_operand:DI 0 "register_operand" "=r")
2654         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2655   ""
2656   "neg\\t%w0, %w1"
2657   [(set_attr "type" "alu_sreg")]
2658 )
2659
2660 (define_insn "*ngc<mode>"
2661   [(set (match_operand:GPI 0 "register_operand" "=r")
2662         (minus:GPI
2663           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
2664           (match_operand:GPI 1 "register_operand" "r")))]
2665   ""
2666   "ngc\\t%<w>0, %<w>1"
2667   [(set_attr "type" "adc_reg")]
2668 )
2669
2670 (define_insn "*ngcsi_uxtw"
2671   [(set (match_operand:DI 0 "register_operand" "=r")
2672         (zero_extend:DI
2673           (minus:SI
2674             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
2675             (match_operand:SI 1 "register_operand" "r"))))]
2676   ""
2677   "ngc\\t%w0, %w1"
2678   [(set_attr "type" "adc_reg")]
2679 )
2680
2681 (define_insn "neg<mode>2_compare0"
2682   [(set (reg:CC_NZ CC_REGNUM)
2683         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2684                        (const_int 0)))
2685    (set (match_operand:GPI 0 "register_operand" "=r")
2686         (neg:GPI (match_dup 1)))]
2687   ""
2688   "negs\\t%<w>0, %<w>1"
2689   [(set_attr "type" "alus_sreg")]
2690 )
2691
2692 ;; zero_extend version of above
2693 (define_insn "*negsi2_compare0_uxtw"
2694   [(set (reg:CC_NZ CC_REGNUM)
2695         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2696                        (const_int 0)))
2697    (set (match_operand:DI 0 "register_operand" "=r")
2698         (zero_extend:DI (neg:SI (match_dup 1))))]
2699   ""
2700   "negs\\t%w0, %w1"
2701   [(set_attr "type" "alus_sreg")]
2702 )
2703
2704 (define_insn "*neg_<shift><mode>3_compare0"
2705   [(set (reg:CC_NZ CC_REGNUM)
2706         (compare:CC_NZ
2707          (neg:GPI (ASHIFT:GPI
2708                    (match_operand:GPI 1 "register_operand" "r")
2709                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2710          (const_int 0)))
2711    (set (match_operand:GPI 0 "register_operand" "=r")
2712         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2713   ""
2714   "negs\\t%<w>0, %<w>1, <shift> %2"
2715   [(set_attr "type" "alus_shift_imm")]
2716 )
2717
2718 (define_insn "*neg_<shift>_<mode>2"
2719   [(set (match_operand:GPI 0 "register_operand" "=r")
2720         (neg:GPI (ASHIFT:GPI
2721                   (match_operand:GPI 1 "register_operand" "r")
2722                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2723   ""
2724   "neg\\t%<w>0, %<w>1, <shift> %2"
2725   [(set_attr "type" "alu_shift_imm")]
2726 )
2727
2728 ;; zero_extend version of above
2729 (define_insn "*neg_<shift>_si2_uxtw"
2730   [(set (match_operand:DI 0 "register_operand" "=r")
2731         (zero_extend:DI
2732          (neg:SI (ASHIFT:SI
2733                   (match_operand:SI 1 "register_operand" "r")
2734                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2735   ""
2736   "neg\\t%w0, %w1, <shift> %2"
2737   [(set_attr "type" "alu_shift_imm")]
2738 )
2739
2740 (define_insn "*neg_mul_imm_<mode>2"
2741   [(set (match_operand:GPI 0 "register_operand" "=r")
2742         (neg:GPI (mult:GPI
2743                   (match_operand:GPI 1 "register_operand" "r")
2744                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2745   ""
2746   "neg\\t%<w>0, %<w>1, lsl %p2"
2747   [(set_attr "type" "alu_shift_imm")]
2748 )
2749
2750 ;; zero_extend version of above
2751 (define_insn "*neg_mul_imm_si2_uxtw"
2752   [(set (match_operand:DI 0 "register_operand" "=r")
2753         (zero_extend:DI
2754          (neg:SI (mult:SI
2755                   (match_operand:SI 1 "register_operand" "r")
2756                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2757   ""
2758   "neg\\t%w0, %w1, lsl %p2"
2759   [(set_attr "type" "alu_shift_imm")]
2760 )
2761
2762 (define_insn "mul<mode>3"
2763   [(set (match_operand:GPI 0 "register_operand" "=r")
2764         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2765                   (match_operand:GPI 2 "register_operand" "r")))]
2766   ""
2767   "mul\\t%<w>0, %<w>1, %<w>2"
2768   [(set_attr "type" "mul")]
2769 )
2770
2771 ;; zero_extend version of above
2772 (define_insn "*mulsi3_uxtw"
2773   [(set (match_operand:DI 0 "register_operand" "=r")
2774         (zero_extend:DI
2775          (mult:SI (match_operand:SI 1 "register_operand" "r")
2776                   (match_operand:SI 2 "register_operand" "r"))))]
2777   ""
2778   "mul\\t%w0, %w1, %w2"
2779   [(set_attr "type" "mul")]
2780 )
2781
2782 (define_insn "madd<mode>"
2783   [(set (match_operand:GPI 0 "register_operand" "=r")
2784         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2785                             (match_operand:GPI 2 "register_operand" "r"))
2786                   (match_operand:GPI 3 "register_operand" "r")))]
2787   ""
2788   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2789   [(set_attr "type" "mla")]
2790 )
2791
2792 ;; zero_extend version of above
2793 (define_insn "*maddsi_uxtw"
2794   [(set (match_operand:DI 0 "register_operand" "=r")
2795         (zero_extend:DI
2796          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2797                            (match_operand:SI 2 "register_operand" "r"))
2798                   (match_operand:SI 3 "register_operand" "r"))))]
2799   ""
2800   "madd\\t%w0, %w1, %w2, %w3"
2801   [(set_attr "type" "mla")]
2802 )
2803
2804 (define_insn "*msub<mode>"
2805   [(set (match_operand:GPI 0 "register_operand" "=r")
2806         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2807                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2808                              (match_operand:GPI 2 "register_operand" "r"))))]
2809
2810   ""
2811   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2812   [(set_attr "type" "mla")]
2813 )
2814
2815 ;; zero_extend version of above
2816 (define_insn "*msubsi_uxtw"
2817   [(set (match_operand:DI 0 "register_operand" "=r")
2818         (zero_extend:DI
2819          (minus:SI (match_operand:SI 3 "register_operand" "r")
2820                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2821                             (match_operand:SI 2 "register_operand" "r")))))]
2822
2823   ""
2824   "msub\\t%w0, %w1, %w2, %w3"
2825   [(set_attr "type" "mla")]
2826 )
2827
2828 (define_insn "*mul<mode>_neg"
2829   [(set (match_operand:GPI 0 "register_operand" "=r")
2830         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2831                   (match_operand:GPI 2 "register_operand" "r")))]
2832
2833   ""
2834   "mneg\\t%<w>0, %<w>1, %<w>2"
2835   [(set_attr "type" "mul")]
2836 )
2837
2838 ;; zero_extend version of above
2839 (define_insn "*mulsi_neg_uxtw"
2840   [(set (match_operand:DI 0 "register_operand" "=r")
2841         (zero_extend:DI
2842          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2843                   (match_operand:SI 2 "register_operand" "r"))))]
2844
2845   ""
2846   "mneg\\t%w0, %w1, %w2"
2847   [(set_attr "type" "mul")]
2848 )
2849
2850 (define_insn "<su_optab>mulsidi3"
2851   [(set (match_operand:DI 0 "register_operand" "=r")
2852         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2853                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2854   ""
2855   "<su>mull\\t%0, %w1, %w2"
2856   [(set_attr "type" "<su>mull")]
2857 )
2858
2859 (define_insn "<su_optab>maddsidi4"
2860   [(set (match_operand:DI 0 "register_operand" "=r")
2861         (plus:DI (mult:DI
2862                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2863                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
2864                  (match_operand:DI 3 "register_operand" "r")))]
2865   ""
2866   "<su>maddl\\t%0, %w1, %w2, %3"
2867   [(set_attr "type" "<su>mlal")]
2868 )
2869
2870 (define_insn "<su_optab>msubsidi4"
2871   [(set (match_operand:DI 0 "register_operand" "=r")
2872         (minus:DI
2873          (match_operand:DI 3 "register_operand" "r")
2874          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2875                   (ANY_EXTEND:DI
2876                    (match_operand:SI 2 "register_operand" "r")))))]
2877   ""
2878   "<su>msubl\\t%0, %w1, %w2, %3"
2879   [(set_attr "type" "<su>mlal")]
2880 )
2881
2882 (define_insn "*<su_optab>mulsidi_neg"
2883   [(set (match_operand:DI 0 "register_operand" "=r")
2884         (mult:DI (neg:DI
2885                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
2886                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2887   ""
2888   "<su>mnegl\\t%0, %w1, %w2"
2889   [(set_attr "type" "<su>mull")]
2890 )
2891
2892 (define_expand "<su_optab>mulditi3"
2893   [(set (match_operand:TI 0 "register_operand")
2894         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
2895                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
2896   ""
2897 {
2898   rtx low = gen_reg_rtx (DImode);
2899   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
2900
2901   rtx high = gen_reg_rtx (DImode);
2902   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
2903
2904   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2905   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2906   DONE;
2907 })
2908
2909 ;; The default expansion of multi3 using umuldi3_highpart will perform
2910 ;; the additions in an order that fails to combine into two madd insns.
2911 (define_expand "multi3"
2912   [(set (match_operand:TI 0 "register_operand")
2913         (mult:TI (match_operand:TI 1 "register_operand")
2914                  (match_operand:TI 2 "register_operand")))]
2915   ""
2916 {
2917   rtx l0 = gen_reg_rtx (DImode);
2918   rtx l1 = gen_lowpart (DImode, operands[1]);
2919   rtx l2 = gen_lowpart (DImode, operands[2]);
2920   rtx h0 = gen_reg_rtx (DImode);
2921   rtx h1 = gen_highpart (DImode, operands[1]);
2922   rtx h2 = gen_highpart (DImode, operands[2]);
2923
2924   emit_insn (gen_muldi3 (l0, l1, l2));
2925   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
2926   emit_insn (gen_madddi (h0, h1, l2, h0));
2927   emit_insn (gen_madddi (h0, l1, h2, h0));
2928
2929   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
2930   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
2931   DONE;
2932 })
2933
2934 (define_insn "<su>muldi3_highpart"
2935   [(set (match_operand:DI 0 "register_operand" "=r")
2936         (truncate:DI
2937          (lshiftrt:TI
2938           (mult:TI
2939            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
2940            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
2941           (const_int 64))))]
2942   ""
2943   "<su>mulh\\t%0, %1, %2"
2944   [(set_attr "type" "<su>mull")]
2945 )
2946
2947 (define_insn "<su_optab>div<mode>3"
2948   [(set (match_operand:GPI 0 "register_operand" "=r")
2949         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
2950                      (match_operand:GPI 2 "register_operand" "r")))]
2951   ""
2952   "<su>div\\t%<w>0, %<w>1, %<w>2"
2953   [(set_attr "type" "<su>div")]
2954 )
2955
2956 ;; zero_extend version of above
2957 (define_insn "*<su_optab>divsi3_uxtw"
2958   [(set (match_operand:DI 0 "register_operand" "=r")
2959         (zero_extend:DI
2960          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
2961                      (match_operand:SI 2 "register_operand" "r"))))]
2962   ""
2963   "<su>div\\t%w0, %w1, %w2"
2964   [(set_attr "type" "<su>div")]
2965 )
2966
2967 ;; -------------------------------------------------------------------
2968 ;; Comparison insns
2969 ;; -------------------------------------------------------------------
2970
2971 (define_insn "cmp<mode>"
2972   [(set (reg:CC CC_REGNUM)
2973         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
2974                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
2975   ""
2976   "@
2977    cmp\\t%<w>0, %<w>1
2978    cmp\\t%<w>0, %1
2979    cmn\\t%<w>0, #%n1"
2980   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2981 )
2982
2983 (define_insn "fcmp<mode>"
2984   [(set (reg:CCFP CC_REGNUM)
2985         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
2986                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2987    "TARGET_FLOAT"
2988    "@
2989     fcmp\\t%<s>0, #0.0
2990     fcmp\\t%<s>0, %<s>1"
2991   [(set_attr "type" "fcmp<s>")]
2992 )
2993
2994 (define_insn "fcmpe<mode>"
2995   [(set (reg:CCFPE CC_REGNUM)
2996         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
2997                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2998    "TARGET_FLOAT"
2999    "@
3000     fcmpe\\t%<s>0, #0.0
3001     fcmpe\\t%<s>0, %<s>1"
3002   [(set_attr "type" "fcmp<s>")]
3003 )
3004
3005 (define_insn "*cmp_swp_<shift>_reg<mode>"
3006   [(set (reg:CC_SWP CC_REGNUM)
3007         (compare:CC_SWP (ASHIFT:GPI
3008                          (match_operand:GPI 0 "register_operand" "r")
3009                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3010                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3011   ""
3012   "cmp\\t%<w>2, %<w>0, <shift> %1"
3013   [(set_attr "type" "alus_shift_imm")]
3014 )
3015
3016 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3017   [(set (reg:CC_SWP CC_REGNUM)
3018         (compare:CC_SWP (ANY_EXTEND:GPI
3019                          (match_operand:ALLX 0 "register_operand" "r"))
3020                         (match_operand:GPI 1 "register_operand" "r")))]
3021   ""
3022   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3023   [(set_attr "type" "alus_ext")]
3024 )
3025
3026 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3027   [(set (reg:CC_SWP CC_REGNUM)
3028         (compare:CC_SWP (ashift:GPI
3029                          (ANY_EXTEND:GPI
3030                           (match_operand:ALLX 0 "register_operand" "r"))
3031                          (match_operand 1 "aarch64_imm3" "Ui3"))
3032         (match_operand:GPI 2 "register_operand" "r")))]
3033   ""
3034   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3035   [(set_attr "type" "alus_ext")]
3036 )
3037
3038 ;; -------------------------------------------------------------------
3039 ;; Store-flag and conditional select insns
3040 ;; -------------------------------------------------------------------
3041
3042 (define_expand "cstore<mode>4"
3043   [(set (match_operand:SI 0 "register_operand" "")
3044         (match_operator:SI 1 "aarch64_comparison_operator"
3045          [(match_operand:GPI 2 "register_operand" "")
3046           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3047   ""
3048   "
3049   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3050                                       operands[3]);
3051   operands[3] = const0_rtx;
3052   "
3053 )
3054
3055 (define_expand "cstorecc4"
3056   [(set (match_operand:SI 0 "register_operand")
3057        (match_operator 1 "aarch64_comparison_operator_mode"
3058         [(match_operand 2 "cc_register")
3059          (match_operand 3 "const0_operand")]))]
3060   ""
3061 "{
3062   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3063   DONE;
3064 }")
3065
3066
3067 (define_expand "cstore<mode>4"
3068   [(set (match_operand:SI 0 "register_operand" "")
3069         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3070          [(match_operand:GPF 2 "register_operand" "")
3071           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3072   ""
3073   "
3074   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3075                                       operands[3]);
3076   operands[3] = const0_rtx;
3077   "
3078 )
3079
3080 (define_insn "aarch64_cstore<mode>"
3081   [(set (match_operand:ALLI 0 "register_operand" "=r")
3082         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3083          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3084   ""
3085   "cset\\t%<w>0, %m1"
3086   [(set_attr "type" "csel")]
3087 )
3088
3089 ;; For a 24-bit immediate CST we can optimize the compare for equality
3090 ;; and branch sequence from:
3091 ;;      mov     x0, #imm1
3092 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3093 ;;      cmp     x1, x0
3094 ;;      cset    x2, <ne,eq>
3095 ;; into the shorter:
3096 ;;      sub     x0, x1, #(CST & 0xfff000)
3097 ;;      subs    x0, x0, #(CST & 0x000fff)
3098 ;;      cset x2, <ne, eq>.
3099 (define_insn_and_split "*compare_cstore<mode>_insn"
3100   [(set (match_operand:GPI 0 "register_operand" "=r")
3101          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3102                   (match_operand:GPI 2 "aarch64_imm24" "n")))]
3103   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3104    && !aarch64_plus_operand (operands[2], <MODE>mode)
3105    && !reload_completed"
3106   "#"
3107   "&& true"
3108   [(const_int 0)]
3109   {
3110     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3111     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3112     rtx tmp = gen_reg_rtx (<MODE>mode);
3113     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3114     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3115     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3116     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3117     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3118     DONE;
3119   }
3120   [(set_attr "type" "csel")]
3121 )
3122
3123 ;; zero_extend version of the above
3124 (define_insn "*cstoresi_insn_uxtw"
3125   [(set (match_operand:DI 0 "register_operand" "=r")
3126         (zero_extend:DI
3127          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3128           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3129   ""
3130   "cset\\t%w0, %m1"
3131   [(set_attr "type" "csel")]
3132 )
3133
3134 (define_insn "cstore<mode>_neg"
3135   [(set (match_operand:ALLI 0 "register_operand" "=r")
3136         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3137                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3138   ""
3139   "csetm\\t%<w>0, %m1"
3140   [(set_attr "type" "csel")]
3141 )
3142
3143 ;; zero_extend version of the above
3144 (define_insn "*cstoresi_neg_uxtw"
3145   [(set (match_operand:DI 0 "register_operand" "=r")
3146         (zero_extend:DI
3147          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3148                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3149   ""
3150   "csetm\\t%w0, %m1"
3151   [(set_attr "type" "csel")]
3152 )
3153
3154 (define_expand "cmov<mode>6"
3155   [(set (match_operand:GPI 0 "register_operand" "")
3156         (if_then_else:GPI
3157          (match_operator 1 "aarch64_comparison_operator"
3158           [(match_operand:GPI 2 "register_operand" "")
3159            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3160          (match_operand:GPI 4 "register_operand" "")
3161          (match_operand:GPI 5 "register_operand" "")))]
3162   ""
3163   "
3164   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3165                                       operands[3]);
3166   operands[3] = const0_rtx;
3167   "
3168 )
3169
3170 (define_expand "cmov<mode>6"
3171   [(set (match_operand:GPF 0 "register_operand" "")
3172         (if_then_else:GPF
3173          (match_operator 1 "aarch64_comparison_operator"
3174           [(match_operand:GPF 2 "register_operand" "")
3175            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3176          (match_operand:GPF 4 "register_operand" "")
3177          (match_operand:GPF 5 "register_operand" "")))]
3178   ""
3179   "
3180   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3181                                       operands[3]);
3182   operands[3] = const0_rtx;
3183   "
3184 )
3185
3186 (define_insn "*cmov<mode>_insn"
3187   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3188         (if_then_else:ALLI
3189          (match_operator 1 "aarch64_comparison_operator"
3190           [(match_operand 2 "cc_register" "") (const_int 0)])
3191          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3192          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3193   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3194      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3195   ;; Final two alternatives should be unreachable, but included for completeness
3196   "@
3197    csel\\t%<w>0, %<w>3, %<w>4, %m1
3198    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3199    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3200    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3201    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3202    mov\\t%<w>0, -1
3203    mov\\t%<w>0, 1"
3204   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3205 )
3206
3207 ;; zero_extend version of above
3208 (define_insn "*cmovsi_insn_uxtw"
3209   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3210         (zero_extend:DI
3211          (if_then_else:SI
3212           (match_operator 1 "aarch64_comparison_operator"
3213            [(match_operand 2 "cc_register" "") (const_int 0)])
3214           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3215           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3216   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3217      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3218   ;; Final two alternatives should be unreachable, but included for completeness
3219   "@
3220    csel\\t%w0, %w3, %w4, %m1
3221    csinv\\t%w0, %w3, wzr, %m1
3222    csinv\\t%w0, %w4, wzr, %M1
3223    csinc\\t%w0, %w3, wzr, %m1
3224    csinc\\t%w0, %w4, wzr, %M1
3225    mov\\t%w0, -1
3226    mov\\t%w0, 1"
3227   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3228 )
3229
3230 (define_insn "*cmovdi_insn_uxtw"
3231   [(set (match_operand:DI 0 "register_operand" "=r")
3232         (if_then_else:DI
3233          (match_operator 1 "aarch64_comparison_operator"
3234           [(match_operand 2 "cc_register" "") (const_int 0)])
3235          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3236          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3237   ""
3238   "csel\\t%w0, %w3, %w4, %m1"
3239   [(set_attr "type" "csel")]
3240 )
3241
3242 (define_insn "*cmov<mode>_insn"
3243   [(set (match_operand:GPF 0 "register_operand" "=w")
3244         (if_then_else:GPF
3245          (match_operator 1 "aarch64_comparison_operator"
3246           [(match_operand 2 "cc_register" "") (const_int 0)])
3247          (match_operand:GPF 3 "register_operand" "w")
3248          (match_operand:GPF 4 "register_operand" "w")))]
3249   "TARGET_FLOAT"
3250   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3251   [(set_attr "type" "fcsel")]
3252 )
3253
3254 (define_expand "mov<mode>cc"
3255   [(set (match_operand:ALLI 0 "register_operand" "")
3256         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3257                            (match_operand:ALLI 2 "register_operand" "")
3258                            (match_operand:ALLI 3 "register_operand" "")))]
3259   ""
3260   {
3261     rtx ccreg;
3262     enum rtx_code code = GET_CODE (operands[1]);
3263
3264     if (code == UNEQ || code == LTGT)
3265       FAIL;
3266
3267     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3268                                      XEXP (operands[1], 1));
3269     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3270   }
3271 )
3272
3273 (define_expand "mov<GPF:mode><GPI:mode>cc"
3274   [(set (match_operand:GPI 0 "register_operand" "")
3275         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3276                           (match_operand:GPF 2 "register_operand" "")
3277                           (match_operand:GPF 3 "register_operand" "")))]
3278   ""
3279   {
3280     rtx ccreg;
3281     enum rtx_code code = GET_CODE (operands[1]);
3282
3283     if (code == UNEQ || code == LTGT)
3284       FAIL;
3285
3286     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3287                                   XEXP (operands[1], 1));
3288     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3289   }
3290 )
3291
3292 (define_expand "mov<mode>cc"
3293   [(set (match_operand:GPF 0 "register_operand" "")
3294         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3295                           (match_operand:GPF 2 "register_operand" "")
3296                           (match_operand:GPF 3 "register_operand" "")))]
3297   ""
3298   {
3299     rtx ccreg;
3300     enum rtx_code code = GET_CODE (operands[1]);
3301
3302     if (code == UNEQ || code == LTGT)
3303       FAIL;
3304
3305     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3306                                   XEXP (operands[1], 1));
3307     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3308   }
3309 )
3310
3311 (define_expand "<neg_not_op><mode>cc"
3312   [(set (match_operand:GPI 0 "register_operand" "")
3313         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3314                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3315                           (match_operand:GPI 3 "register_operand" "")))]
3316   ""
3317   {
3318     rtx ccreg;
3319     enum rtx_code code = GET_CODE (operands[1]);
3320
3321     if (code == UNEQ || code == LTGT)
3322       FAIL;
3323
3324     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3325                                       XEXP (operands[1], 1));
3326     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3327   }
3328 )
3329
3330 ;; CRC32 instructions.
3331 (define_insn "aarch64_<crc_variant>"
3332   [(set (match_operand:SI 0 "register_operand" "=r")
3333         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3334                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3335          CRC))]
3336   "TARGET_CRC32"
3337   {
3338     if (GET_MODE_BITSIZE (GET_MODE (operands[2])) >= 64)
3339       return "<crc_variant>\\t%w0, %w1, %x2";
3340     else
3341       return "<crc_variant>\\t%w0, %w1, %w2";
3342   }
3343   [(set_attr "type" "crc")]
3344 )
3345
3346 (define_insn "*csinc2<mode>_insn"
3347   [(set (match_operand:GPI 0 "register_operand" "=r")
3348         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3349                   (match_operand:GPI 1 "register_operand" "r")))]
3350   ""
3351   "cinc\\t%<w>0, %<w>1, %m2"
3352   [(set_attr "type" "csel")]
3353 )
3354
3355 (define_insn "csinc3<mode>_insn"
3356   [(set (match_operand:GPI 0 "register_operand" "=r")
3357         (if_then_else:GPI
3358           (match_operand 1 "aarch64_comparison_operation" "")
3359           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3360                     (const_int 1))
3361           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3362   ""
3363   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3364   [(set_attr "type" "csel")]
3365 )
3366
3367 (define_insn "*csinv3<mode>_insn"
3368   [(set (match_operand:GPI 0 "register_operand" "=r")
3369         (if_then_else:GPI
3370           (match_operand 1 "aarch64_comparison_operation" "")
3371           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3372           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3373   ""
3374   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3375   [(set_attr "type" "csel")]
3376 )
3377
3378 (define_insn "csneg3_uxtw_insn"
3379   [(set (match_operand:DI 0 "register_operand" "=r")
3380         (zero_extend:DI
3381           (if_then_else:SI
3382             (match_operand 1 "aarch64_comparison_operation" "")
3383             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3384             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3385   ""
3386   "csneg\\t%w0, %w3, %w2, %M1"
3387   [(set_attr "type" "csel")]
3388 )
3389
3390 (define_insn "csneg3<mode>_insn"
3391   [(set (match_operand:GPI 0 "register_operand" "=r")
3392         (if_then_else:GPI
3393           (match_operand 1 "aarch64_comparison_operation" "")
3394           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3395           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3396   ""
3397   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3398   [(set_attr "type" "csel")]
3399 )
3400
3401 ;; -------------------------------------------------------------------
3402 ;; Logical operations
3403 ;; -------------------------------------------------------------------
3404
3405
3406 (define_insn_and_split "*aarch64_and<mode>_imm2"
3407   [(set (match_operand:GPI 0 "register_operand" "=rk")
3408         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
3409                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
3410   ""
3411   "#"
3412   "true"
3413   [(const_int 0)]
3414   {
3415      HOST_WIDE_INT val = INTVAL (operands[2]);
3416      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
3417      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
3418
3419      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
3420      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
3421      DONE;
3422   }
3423 )
3424
3425 (define_insn "<optab><mode>3"
3426   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3427         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3428                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3429   ""
3430   "@
3431   <logical>\\t%<w>0, %<w>1, %<w>2
3432   <logical>\\t%<w>0, %<w>1, %2
3433   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3434   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3435    (set_attr "simd" "*,*,yes")]
3436 )
3437
3438 ;; zero_extend version of above
3439 (define_insn "*<optab>si3_uxtw"
3440   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3441         (zero_extend:DI
3442          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3443                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3444   ""
3445   "@
3446    <logical>\\t%w0, %w1, %w2
3447    <logical>\\t%w0, %w1, %2"
3448   [(set_attr "type" "logic_reg,logic_imm")]
3449 )
3450
3451 (define_insn "*and<mode>3_compare0"
3452   [(set (reg:CC_NZ CC_REGNUM)
3453         (compare:CC_NZ
3454          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3455                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3456          (const_int 0)))
3457    (set (match_operand:GPI 0 "register_operand" "=r,r")
3458         (and:GPI (match_dup 1) (match_dup 2)))]
3459   ""
3460   "@
3461    ands\\t%<w>0, %<w>1, %<w>2
3462    ands\\t%<w>0, %<w>1, %2"
3463   [(set_attr "type" "logics_reg,logics_imm")]
3464 )
3465
3466 ;; zero_extend version of above
3467 (define_insn "*andsi3_compare0_uxtw"
3468   [(set (reg:CC_NZ CC_REGNUM)
3469         (compare:CC_NZ
3470          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3471                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3472          (const_int 0)))
3473    (set (match_operand:DI 0 "register_operand" "=r,r")
3474         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3475   ""
3476   "@
3477    ands\\t%w0, %w1, %w2
3478    ands\\t%w0, %w1, %2"
3479   [(set_attr "type" "logics_reg,logics_imm")]
3480 )
3481
3482 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3483   [(set (reg:CC_NZ CC_REGNUM)
3484         (compare:CC_NZ
3485          (and:GPI (SHIFT:GPI
3486                    (match_operand:GPI 1 "register_operand" "r")
3487                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3488                   (match_operand:GPI 3 "register_operand" "r"))
3489          (const_int 0)))
3490    (set (match_operand:GPI 0 "register_operand" "=r")
3491         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3492   ""
3493   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3494   [(set_attr "type" "logics_shift_imm")]
3495 )
3496
3497 ;; zero_extend version of above
3498 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3499   [(set (reg:CC_NZ CC_REGNUM)
3500         (compare:CC_NZ
3501          (and:SI (SHIFT:SI
3502                   (match_operand:SI 1 "register_operand" "r")
3503                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3504                  (match_operand:SI 3 "register_operand" "r"))
3505          (const_int 0)))
3506    (set (match_operand:DI 0 "register_operand" "=r")
3507         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3508                                 (match_dup 3))))]
3509   ""
3510   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3511   [(set_attr "type" "logics_shift_imm")]
3512 )
3513
3514 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3515   [(set (match_operand:GPI 0 "register_operand" "=r")
3516         (LOGICAL:GPI (SHIFT:GPI
3517                       (match_operand:GPI 1 "register_operand" "r")
3518                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3519                      (match_operand:GPI 3 "register_operand" "r")))]
3520   ""
3521   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3522   [(set_attr "type" "logic_shift_imm")]
3523 )
3524
3525 (define_insn "*<optab>_rol<mode>3"
3526   [(set (match_operand:GPI 0 "register_operand" "=r")
3527         (LOGICAL:GPI (rotate:GPI
3528                       (match_operand:GPI 1 "register_operand" "r")
3529                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3530                      (match_operand:GPI 3 "register_operand" "r")))]
3531   ""
3532   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3533   [(set_attr "type" "logic_shift_imm")]
3534 )
3535
3536 ;; zero_extend versions of above
3537 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3538   [(set (match_operand:DI 0 "register_operand" "=r")
3539         (zero_extend:DI
3540          (LOGICAL:SI (SHIFT:SI
3541                       (match_operand:SI 1 "register_operand" "r")
3542                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3543                      (match_operand:SI 3 "register_operand" "r"))))]
3544   ""
3545   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3546   [(set_attr "type" "logic_shift_imm")]
3547 )
3548
3549 (define_insn "*<optab>_rolsi3_uxtw"
3550   [(set (match_operand:DI 0 "register_operand" "=r")
3551         (zero_extend:DI
3552          (LOGICAL:SI (rotate:SI
3553                       (match_operand:SI 1 "register_operand" "r")
3554                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3555                      (match_operand:SI 3 "register_operand" "r"))))]
3556   ""
3557   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3558   [(set_attr "type" "logic_shift_imm")]
3559 )
3560
3561 (define_insn "one_cmpl<mode>2"
3562   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3563         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3564   ""
3565   "@
3566   mvn\\t%<w>0, %<w>1
3567   mvn\\t%0.8b, %1.8b"
3568   [(set_attr "type" "logic_reg,neon_logic")
3569    (set_attr "simd" "*,yes")]
3570 )
3571
3572 (define_insn "*one_cmpl_<optab><mode>2"
3573   [(set (match_operand:GPI 0 "register_operand" "=r")
3574         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3575                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3576   ""
3577   "mvn\\t%<w>0, %<w>1, <shift> %2"
3578   [(set_attr "type" "logic_shift_imm")]
3579 )
3580
3581 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3582
3583 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3584   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3585         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3586                      (match_operand:GPI 2 "register_operand" "r,w")))]
3587   ""
3588   "@
3589   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3590   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3591   [(set_attr "type" "logic_reg,neon_logic")
3592    (set_attr "simd" "*,yes")]
3593 )
3594
3595 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3596   [(set (match_operand:DI 0 "register_operand" "=r")
3597         (zero_extend:DI
3598           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3599                        (match_operand:SI 2 "register_operand" "r"))))]
3600   ""
3601   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3602   [(set_attr "type" "logic_reg")]
3603 )
3604
3605 (define_insn "*xor_one_cmplsidi3_ze"
3606   [(set (match_operand:DI 0 "register_operand" "=r")
3607         (zero_extend:DI
3608           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3609                           (match_operand:SI 2 "register_operand" "r")))))]
3610   ""
3611   "eon\\t%w0, %w1, %w2"
3612   [(set_attr "type" "logic_reg")]
3613 )
3614
3615 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3616 ;; eon does not operate on SIMD registers so the vector variant must be split.
3617 (define_insn_and_split "*xor_one_cmpl<mode>3"
3618   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3619         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3620                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3621   ""
3622   "@
3623   eon\\t%<w>0, %<w>1, %<w>2
3624   #"
3625   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3626   [(set (match_operand:GPI 0 "register_operand" "=w")
3627         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3628                  (match_operand:GPI 2 "register_operand" "w")))
3629    (set (match_dup 0) (not:GPI (match_dup 0)))]
3630   ""
3631   [(set_attr "type" "logic_reg,multiple")
3632    (set_attr "simd" "*,yes")]
3633 )
3634
3635 (define_insn "*and_one_cmpl<mode>3_compare0"
3636   [(set (reg:CC_NZ CC_REGNUM)
3637         (compare:CC_NZ
3638          (and:GPI (not:GPI
3639                    (match_operand:GPI 1 "register_operand" "r"))
3640                   (match_operand:GPI 2 "register_operand" "r"))
3641          (const_int 0)))
3642    (set (match_operand:GPI 0 "register_operand" "=r")
3643         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3644   ""
3645   "bics\\t%<w>0, %<w>2, %<w>1"
3646   [(set_attr "type" "logics_reg")]
3647 )
3648
3649 ;; zero_extend version of above
3650 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3651   [(set (reg:CC_NZ CC_REGNUM)
3652         (compare:CC_NZ
3653          (and:SI (not:SI
3654                   (match_operand:SI 1 "register_operand" "r"))
3655                  (match_operand:SI 2 "register_operand" "r"))
3656          (const_int 0)))
3657    (set (match_operand:DI 0 "register_operand" "=r")
3658         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3659   ""
3660   "bics\\t%w0, %w2, %w1"
3661   [(set_attr "type" "logics_reg")]
3662 )
3663
3664 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3665   [(set (reg:CC_NZ CC_REGNUM)
3666     (compare:CC_NZ
3667      (and:GPI (not:GPI
3668            (match_operand:GPI 0 "register_operand" "r"))
3669           (match_operand:GPI 1 "register_operand" "r"))
3670      (const_int 0)))]
3671   ""
3672   "bics\\t<w>zr, %<w>1, %<w>0"
3673   [(set_attr "type" "logics_reg")]
3674 )
3675
3676 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3677   [(set (match_operand:GPI 0 "register_operand" "=r")
3678         (LOGICAL:GPI (not:GPI
3679                       (SHIFT:GPI
3680                        (match_operand:GPI 1 "register_operand" "r")
3681                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3682                      (match_operand:GPI 3 "register_operand" "r")))]
3683   ""
3684   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3685   [(set_attr "type" "logic_shift_imm")]
3686 )
3687
3688 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3689   [(set (match_operand:GPI 0 "register_operand" "=r")
3690         (not:GPI (xor:GPI
3691                       (SHIFT:GPI
3692                        (match_operand:GPI 1 "register_operand" "r")
3693                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3694                      (match_operand:GPI 3 "register_operand" "r"))))]
3695   ""
3696   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3697   [(set_attr "type" "logic_shift_imm")]
3698 )
3699
3700 ;; Zero-extend version of the above.
3701 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3702   [(set (match_operand:DI 0 "register_operand" "=r")
3703         (zero_extend:DI
3704           (not:SI (xor:SI
3705                     (SHIFT:SI
3706                       (match_operand:SI 1 "register_operand" "r")
3707                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3708                     (match_operand:SI 3 "register_operand" "r")))))]
3709   ""
3710   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3711   [(set_attr "type" "logic_shift_imm")]
3712 )
3713
3714 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3715   [(set (reg:CC_NZ CC_REGNUM)
3716         (compare:CC_NZ
3717          (and:GPI (not:GPI
3718                    (SHIFT:GPI
3719                     (match_operand:GPI 1 "register_operand" "r")
3720                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3721                   (match_operand:GPI 3 "register_operand" "r"))
3722          (const_int 0)))
3723    (set (match_operand:GPI 0 "register_operand" "=r")
3724         (and:GPI (not:GPI
3725                   (SHIFT:GPI
3726                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3727   ""
3728   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3729   [(set_attr "type" "logics_shift_imm")]
3730 )
3731
3732 ;; zero_extend version of above
3733 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3734   [(set (reg:CC_NZ CC_REGNUM)
3735         (compare:CC_NZ
3736          (and:SI (not:SI
3737                   (SHIFT:SI
3738                    (match_operand:SI 1 "register_operand" "r")
3739                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3740                  (match_operand:SI 3 "register_operand" "r"))
3741          (const_int 0)))
3742    (set (match_operand:DI 0 "register_operand" "=r")
3743         (zero_extend:DI (and:SI
3744                          (not:SI
3745                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3746   ""
3747   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3748   [(set_attr "type" "logics_shift_imm")]
3749 )
3750
3751 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3752   [(set (reg:CC_NZ CC_REGNUM)
3753     (compare:CC_NZ
3754      (and:GPI (not:GPI
3755            (SHIFT:GPI
3756             (match_operand:GPI 0 "register_operand" "r")
3757             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3758           (match_operand:GPI 2 "register_operand" "r"))
3759      (const_int 0)))]
3760   ""
3761   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3762   [(set_attr "type" "logics_shift_imm")]
3763 )
3764
3765 (define_insn "clz<mode>2"
3766   [(set (match_operand:GPI 0 "register_operand" "=r")
3767         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3768   ""
3769   "clz\\t%<w>0, %<w>1"
3770   [(set_attr "type" "clz")]
3771 )
3772
3773 (define_expand "ffs<mode>2"
3774   [(match_operand:GPI 0 "register_operand")
3775    (match_operand:GPI 1 "register_operand")]
3776   ""
3777   {
3778     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3779     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3780
3781     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3782     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3783     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3784     DONE;
3785   }
3786 )
3787
3788 (define_insn "clrsb<mode>2"
3789   [(set (match_operand:GPI 0 "register_operand" "=r")
3790         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
3791   ""
3792   "cls\\t%<w>0, %<w>1"
3793   [(set_attr "type" "clz")]
3794 )
3795
3796 (define_insn "rbit<mode>2"
3797   [(set (match_operand:GPI 0 "register_operand" "=r")
3798         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
3799   ""
3800   "rbit\\t%<w>0, %<w>1"
3801   [(set_attr "type" "rbit")]
3802 )
3803
3804 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
3805 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
3806 ;; expression and split after reload to enable scheduling them apart if
3807 ;; needed.
3808
3809 (define_insn_and_split "ctz<mode>2"
3810  [(set (match_operand:GPI           0 "register_operand" "=r")
3811        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
3812   ""
3813   "#"
3814   "reload_completed"
3815   [(const_int 0)]
3816   "
3817   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3818   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3819   DONE;
3820 ")
3821
3822 (define_insn "*and<mode>_compare0"
3823   [(set (reg:CC_NZ CC_REGNUM)
3824         (compare:CC_NZ
3825          (match_operand:SHORT 0 "register_operand" "r")
3826          (const_int 0)))]
3827   ""
3828   "tst\\t%<w>0, <short_mask>"
3829   [(set_attr "type" "alus_imm")]
3830 )
3831
3832 (define_insn "*ands<mode>_compare0"
3833   [(set (reg:CC_NZ CC_REGNUM)
3834         (compare:CC_NZ
3835          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
3836          (const_int 0)))
3837    (set (match_operand:GPI 0 "register_operand" "=r")
3838         (zero_extend:GPI (match_dup 1)))]
3839   ""
3840   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
3841   [(set_attr "type" "alus_imm")]
3842 )
3843
3844 (define_insn "*and<mode>3nr_compare0"
3845   [(set (reg:CC_NZ CC_REGNUM)
3846         (compare:CC_NZ
3847          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
3848                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
3849          (const_int 0)))]
3850   ""
3851   "@
3852    tst\\t%<w>0, %<w>1
3853    tst\\t%<w>0, %1"
3854   [(set_attr "type" "logics_reg,logics_imm")]
3855 )
3856
3857 (define_insn "*and<mode>3nr_compare0_zextract"
3858   [(set (reg:CC_NZ CC_REGNUM)
3859         (compare:CC_NZ
3860          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
3861                   (match_operand:GPI 1 "const_int_operand" "n")
3862                   (match_operand:GPI 2 "const_int_operand" "n"))
3863          (const_int 0)))]
3864   "INTVAL (operands[1]) > 0
3865    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
3866         <= GET_MODE_BITSIZE (<MODE>mode))
3867    && aarch64_bitmask_imm (
3868         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
3869                                                  operands[2])),
3870         <MODE>mode)"
3871   {
3872     operands[1]
3873       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
3874     return "tst\\t%<w>0, %1";
3875   }
3876   [(set_attr "type" "logics_shift_imm")]
3877 )
3878
3879 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
3880   [(set (reg:CC_NZ CC_REGNUM)
3881         (compare:CC_NZ
3882          (and:GPI (SHIFT:GPI
3883                    (match_operand:GPI 0 "register_operand" "r")
3884                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3885                   (match_operand:GPI 2 "register_operand" "r"))
3886         (const_int 0)))]
3887   ""
3888   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
3889   [(set_attr "type" "logics_shift_imm")]
3890 )
3891
3892 ;; -------------------------------------------------------------------
3893 ;; Shifts
3894 ;; -------------------------------------------------------------------
3895
3896 (define_expand "<optab><mode>3"
3897   [(set (match_operand:GPI 0 "register_operand")
3898         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
3899                     (match_operand:QI 2 "nonmemory_operand")))]
3900   ""
3901   {
3902     if (CONST_INT_P (operands[2]))
3903       {
3904         operands[2] = GEN_INT (INTVAL (operands[2])
3905                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3906
3907         if (operands[2] == const0_rtx)
3908           {
3909             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3910             DONE;
3911           }
3912       }
3913   }
3914 )
3915
3916 (define_expand "ashl<mode>3"
3917   [(set (match_operand:SHORT 0 "register_operand")
3918         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
3919                       (match_operand:QI 2 "const_int_operand")))]
3920   ""
3921   {
3922     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
3923
3924     if (operands[2] == const0_rtx)
3925       {
3926         emit_insn (gen_mov<mode> (operands[0], operands[1]));
3927         DONE;
3928       }
3929   }
3930 )
3931
3932 (define_expand "rotr<mode>3"
3933   [(set (match_operand:GPI 0 "register_operand")
3934         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3935                       (match_operand:QI 2 "nonmemory_operand")))]
3936   ""
3937   {
3938     if (CONST_INT_P (operands[2]))
3939       {
3940         operands[2] = GEN_INT (INTVAL (operands[2])
3941                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3942
3943         if (operands[2] == const0_rtx)
3944           {
3945             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3946             DONE;
3947           }
3948       }
3949   }
3950 )
3951
3952 (define_expand "rotl<mode>3"
3953   [(set (match_operand:GPI 0 "register_operand")
3954         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3955                       (match_operand:QI 2 "nonmemory_operand")))]
3956   ""
3957   {
3958     /* (SZ - cnt) % SZ == -cnt % SZ */
3959     if (CONST_INT_P (operands[2]))
3960       {
3961         operands[2] = GEN_INT ((-INTVAL (operands[2]))
3962                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3963         if (operands[2] == const0_rtx)
3964           {
3965             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3966             DONE;
3967           }
3968       }
3969     else
3970       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
3971                                         NULL_RTX, 1);
3972   }
3973 )
3974
3975 ;; Logical left shift using SISD or Integer instruction
3976 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3977   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
3978         (ashift:GPI
3979           (match_operand:GPI 1 "register_operand" "r,r,w,w")
3980           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
3981   ""
3982   "@
3983    lsl\t%<w>0, %<w>1, %2
3984    lsl\t%<w>0, %<w>1, %<w>2
3985    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3986    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
3987   [(set_attr "simd" "no,no,yes,yes")
3988    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
3989 )
3990
3991 ;; Logical right shift using SISD or Integer instruction
3992 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3993   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
3994         (lshiftrt:GPI
3995          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
3996          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w,0")))]
3997   ""
3998   "@
3999    lsr\t%<w>0, %<w>1, %2
4000    lsr\t%<w>0, %<w>1, %<w>2
4001    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4002    #
4003    #"
4004   [(set_attr "simd" "no,no,yes,yes,yes")
4005    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4006 )
4007
4008 (define_split
4009   [(set (match_operand:DI 0 "aarch64_simd_register")
4010         (lshiftrt:DI
4011            (match_operand:DI 1 "aarch64_simd_register")
4012            (match_operand:QI 2 "aarch64_simd_register")))]
4013   "TARGET_SIMD && reload_completed"
4014   [(set (match_dup 3)
4015         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4016    (set (match_dup 0)
4017         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4018   {
4019     operands[3] = gen_lowpart (QImode, operands[0]);
4020   }
4021 )
4022
4023 (define_split
4024   [(set (match_operand:SI 0 "aarch64_simd_register")
4025         (lshiftrt:SI
4026            (match_operand:SI 1 "aarch64_simd_register")
4027            (match_operand:QI 2 "aarch64_simd_register")))]
4028   "TARGET_SIMD && reload_completed"
4029   [(set (match_dup 3)
4030         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4031    (set (match_dup 0)
4032         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4033   {
4034     operands[3] = gen_lowpart (QImode, operands[0]);
4035   }
4036 )
4037
4038 ;; Arithmetic right shift using SISD or Integer instruction
4039 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4040   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4041         (ashiftrt:GPI
4042           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4043           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "Us<cmode>,r,Us<cmode>,w,0")))]
4044   ""
4045   "@
4046    asr\t%<w>0, %<w>1, %2
4047    asr\t%<w>0, %<w>1, %<w>2
4048    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4049    #
4050    #"
4051   [(set_attr "simd" "no,no,yes,yes,yes")
4052    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4053 )
4054
4055 (define_split
4056   [(set (match_operand:DI 0 "aarch64_simd_register")
4057         (ashiftrt:DI
4058            (match_operand:DI 1 "aarch64_simd_register")
4059            (match_operand:QI 2 "aarch64_simd_register")))]
4060   "TARGET_SIMD && reload_completed"
4061   [(set (match_dup 3)
4062         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4063    (set (match_dup 0)
4064         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4065 {
4066   operands[3] = gen_lowpart (QImode, operands[0]);
4067 }
4068 )
4069
4070 (define_split
4071   [(set (match_operand:SI 0 "aarch64_simd_register")
4072         (ashiftrt:SI
4073            (match_operand:SI 1 "aarch64_simd_register")
4074            (match_operand:QI 2 "aarch64_simd_register")))]
4075   "TARGET_SIMD && reload_completed"
4076   [(set (match_dup 3)
4077         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4078    (set (match_dup 0)
4079         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4080 {
4081   operands[3] = gen_lowpart (QImode, operands[0]);
4082 }
4083 )
4084
4085 (define_insn "*aarch64_sisd_ushl"
4086   [(set (match_operand:DI 0 "register_operand" "=w")
4087         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4088                     (match_operand:QI 2 "register_operand" "w")]
4089                    UNSPEC_SISD_USHL))]
4090   "TARGET_SIMD"
4091   "ushl\t%d0, %d1, %d2"
4092   [(set_attr "simd" "yes")
4093    (set_attr "type" "neon_shift_reg")]
4094 )
4095
4096 (define_insn "*aarch64_ushl_2s"
4097   [(set (match_operand:SI 0 "register_operand" "=w")
4098         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4099                     (match_operand:QI 2 "register_operand" "w")]
4100                    UNSPEC_USHL_2S))]
4101   "TARGET_SIMD"
4102   "ushl\t%0.2s, %1.2s, %2.2s"
4103   [(set_attr "simd" "yes")
4104    (set_attr "type" "neon_shift_reg")]
4105 )
4106
4107 (define_insn "*aarch64_sisd_sshl"
4108   [(set (match_operand:DI 0 "register_operand" "=w")
4109         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4110                     (match_operand:QI 2 "register_operand" "w")]
4111                    UNSPEC_SISD_SSHL))]
4112   "TARGET_SIMD"
4113   "sshl\t%d0, %d1, %d2"
4114   [(set_attr "simd" "yes")
4115    (set_attr "type" "neon_shift_reg")]
4116 )
4117
4118 (define_insn "*aarch64_sshl_2s"
4119   [(set (match_operand:SI 0 "register_operand" "=w")
4120         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4121                     (match_operand:QI 2 "register_operand" "w")]
4122                    UNSPEC_SSHL_2S))]
4123   "TARGET_SIMD"
4124   "sshl\t%0.2s, %1.2s, %2.2s"
4125   [(set_attr "simd" "yes")
4126    (set_attr "type" "neon_shift_reg")]
4127 )
4128
4129 (define_insn "*aarch64_sisd_neg_qi"
4130   [(set (match_operand:QI 0 "register_operand" "=w")
4131         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4132                    UNSPEC_SISD_NEG))]
4133   "TARGET_SIMD"
4134   "neg\t%d0, %d1"
4135   [(set_attr "simd" "yes")
4136    (set_attr "type" "neon_neg")]
4137 )
4138
4139 ;; Rotate right
4140 (define_insn "*ror<mode>3_insn"
4141   [(set (match_operand:GPI 0 "register_operand" "=r,r")
4142      (rotatert:GPI
4143        (match_operand:GPI 1 "register_operand" "r,r")
4144        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4145   ""
4146   "@
4147    ror\\t%<w>0, %<w>1, %2
4148    ror\\t%<w>0, %<w>1, %<w>2"
4149   [(set_attr "type" "rotate_imm,shift_reg")]
4150 )
4151
4152 ;; zero_extend version of above
4153 (define_insn "*<optab>si3_insn_uxtw"
4154   [(set (match_operand:DI 0 "register_operand" "=r,r")
4155         (zero_extend:DI (SHIFT:SI
4156          (match_operand:SI 1 "register_operand" "r,r")
4157          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4158   ""
4159   "@
4160    <shift>\\t%w0, %w1, %2
4161    <shift>\\t%w0, %w1, %w2"
4162   [(set_attr "type" "bfx,shift_reg")]
4163 )
4164
4165 (define_insn "*<optab><mode>3_insn"
4166   [(set (match_operand:SHORT 0 "register_operand" "=r")
4167         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4168                       (match_operand 2 "const_int_operand" "n")))]
4169   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4170 {
4171   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4172   return "<bfshift>\t%w0, %w1, %2, %3";
4173 }
4174   [(set_attr "type" "bfx")]
4175 )
4176
4177 (define_insn "*extr<mode>5_insn"
4178   [(set (match_operand:GPI 0 "register_operand" "=r")
4179         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4180                              (match_operand 3 "const_int_operand" "n"))
4181                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4182                                (match_operand 4 "const_int_operand" "n"))))]
4183   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4184    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4185   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4186   [(set_attr "type" "rotate_imm")]
4187 )
4188
4189 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4190 ;; so we have to match both orderings.
4191 (define_insn "*extr<mode>5_insn_alt"
4192   [(set (match_operand:GPI 0 "register_operand" "=r")
4193         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4194                                 (match_operand 4 "const_int_operand" "n"))
4195                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4196                               (match_operand 3 "const_int_operand" "n"))))]
4197   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4198    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4199        == GET_MODE_BITSIZE (<MODE>mode))"
4200   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4201   [(set_attr "type" "rotate_imm")]
4202 )
4203
4204 ;; zero_extend version of the above
4205 (define_insn "*extrsi5_insn_uxtw"
4206   [(set (match_operand:DI 0 "register_operand" "=r")
4207         (zero_extend:DI
4208          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4209                             (match_operand 3 "const_int_operand" "n"))
4210                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4211                               (match_operand 4 "const_int_operand" "n")))))]
4212   "UINTVAL (operands[3]) < 32 &&
4213    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4214   "extr\\t%w0, %w1, %w2, %4"
4215   [(set_attr "type" "rotate_imm")]
4216 )
4217
4218 (define_insn "*extrsi5_insn_uxtw_alt"
4219   [(set (match_operand:DI 0 "register_operand" "=r")
4220         (zero_extend:DI
4221          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4222                                (match_operand 4 "const_int_operand" "n"))
4223                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
4224                             (match_operand 3 "const_int_operand" "n")))))]
4225   "UINTVAL (operands[3]) < 32 &&
4226    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4227   "extr\\t%w0, %w1, %w2, %4"
4228   [(set_attr "type" "rotate_imm")]
4229 )
4230
4231 (define_insn "*ror<mode>3_insn"
4232   [(set (match_operand:GPI 0 "register_operand" "=r")
4233         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4234                     (match_operand 2 "const_int_operand" "n")))]
4235   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4236 {
4237   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4238   return "ror\\t%<w>0, %<w>1, %3";
4239 }
4240   [(set_attr "type" "rotate_imm")]
4241 )
4242
4243 ;; zero_extend version of the above
4244 (define_insn "*rorsi3_insn_uxtw"
4245   [(set (match_operand:DI 0 "register_operand" "=r")
4246         (zero_extend:DI
4247          (rotate:SI (match_operand:SI 1 "register_operand" "r")
4248                     (match_operand 2 "const_int_operand" "n"))))]
4249   "UINTVAL (operands[2]) < 32"
4250 {
4251   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4252   return "ror\\t%w0, %w1, %3";
4253 }
4254   [(set_attr "type" "rotate_imm")]
4255 )
4256
4257 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4258   [(set (match_operand:GPI 0 "register_operand" "=r")
4259         (ANY_EXTEND:GPI
4260          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4261                        (match_operand 2 "const_int_operand" "n"))))]
4262   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4263 {
4264   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4265   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4266 }
4267   [(set_attr "type" "bfx")]
4268 )
4269
4270 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4271   [(set (match_operand:GPI 0 "register_operand" "=r")
4272         (zero_extend:GPI
4273          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4274                          (match_operand 2 "const_int_operand" "n"))))]
4275   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4276 {
4277   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4278   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4279 }
4280   [(set_attr "type" "bfx")]
4281 )
4282
4283 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4284   [(set (match_operand:GPI 0 "register_operand" "=r")
4285         (sign_extend:GPI
4286          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4287                          (match_operand 2 "const_int_operand" "n"))))]
4288   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4289 {
4290   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4291   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4292 }
4293   [(set_attr "type" "bfx")]
4294 )
4295
4296 ;; -------------------------------------------------------------------
4297 ;; Bitfields
4298 ;; -------------------------------------------------------------------
4299
4300 (define_expand "<optab>"
4301   [(set (match_operand:DI 0 "register_operand" "=r")
4302         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
4303                         (match_operand 2
4304                           "aarch64_simd_shift_imm_offset_di")
4305                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
4306   ""
4307   {
4308     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4309                    1, GET_MODE_BITSIZE (DImode) - 1))
4310      FAIL;
4311   }
4312 )
4313
4314
4315 (define_insn "*<optab><mode>"
4316   [(set (match_operand:GPI 0 "register_operand" "=r")
4317         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4318                          (match_operand 2
4319                            "aarch64_simd_shift_imm_offset_<mode>" "n")
4320                          (match_operand 3
4321                            "aarch64_simd_shift_imm_<mode>" "n")))]
4322   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4323              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4324   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4325   [(set_attr "type" "bfx")]
4326 )
4327
4328 ;; Bitfield Insert (insv)
4329 (define_expand "insv<mode>"
4330   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4331                           (match_operand 1 "const_int_operand")
4332                           (match_operand 2 "const_int_operand"))
4333         (match_operand:GPI 3 "general_operand"))]
4334   ""
4335 {
4336   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4337   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4338   rtx value = operands[3];
4339
4340   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4341     FAIL;
4342
4343   if (CONST_INT_P (value))
4344     {
4345       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4346
4347       /* Prefer AND/OR for inserting all zeros or all ones.  */
4348       if ((UINTVAL (value) & mask) == 0
4349            || (UINTVAL (value) & mask) == mask)
4350         FAIL;
4351
4352       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
4353       if (width == 16 && (pos % 16) == 0)
4354         DONE;
4355     }
4356   operands[3] = force_reg (<MODE>mode, value);
4357 })
4358
4359 (define_insn "*insv_reg<mode>"
4360   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4361                           (match_operand 1 "const_int_operand" "n")
4362                           (match_operand 2 "const_int_operand" "n"))
4363         (match_operand:GPI 3 "register_operand" "r"))]
4364   "!(UINTVAL (operands[1]) == 0
4365      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4366          > GET_MODE_BITSIZE (<MODE>mode)))"
4367   "bfi\\t%<w>0, %<w>3, %2, %1"
4368   [(set_attr "type" "bfm")]
4369 )
4370
4371 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4372   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4373                           (match_operand 1 "const_int_operand" "n")
4374                           (match_operand 2 "const_int_operand" "n"))
4375         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
4376   "UINTVAL (operands[1]) <= <ALLX:sizen>"
4377   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4378   [(set_attr "type" "bfm")]
4379 )
4380
4381 (define_insn "*extr_insv_lower_reg<mode>"
4382   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4383                           (match_operand 1 "const_int_operand" "n")
4384                           (const_int 0))
4385         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4386                           (match_dup 1)
4387                           (match_operand 3 "const_int_operand" "n")))]
4388   "!(UINTVAL (operands[1]) == 0
4389      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4390          > GET_MODE_BITSIZE (<MODE>mode)))"
4391   "bfxil\\t%<w>0, %<w>2, %3, %1"
4392   [(set_attr "type" "bfm")]
4393 )
4394
4395 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4396   [(set (match_operand:GPI 0 "register_operand" "=r")
4397         (ashift:GPI (ANY_EXTEND:GPI
4398                      (match_operand:ALLX 1 "register_operand" "r"))
4399                     (match_operand 2 "const_int_operand" "n")))]
4400   "UINTVAL (operands[2]) < <GPI:sizen>"
4401 {
4402   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4403               ? GEN_INT (<ALLX:sizen>)
4404               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4405   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4406 }
4407   [(set_attr "type" "bfx")]
4408 )
4409
4410 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4411
4412 (define_insn "*andim_ashift<mode>_bfiz"
4413   [(set (match_operand:GPI 0 "register_operand" "=r")
4414         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4415                              (match_operand 2 "const_int_operand" "n"))
4416                  (match_operand 3 "const_int_operand" "n")))]
4417   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
4418   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4419   [(set_attr "type" "bfx")]
4420 )
4421
4422 (define_insn "bswap<mode>2"
4423   [(set (match_operand:GPI 0 "register_operand" "=r")
4424         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4425   ""
4426   "rev\\t%<w>0, %<w>1"
4427   [(set_attr "type" "rev")]
4428 )
4429
4430 (define_insn "bswaphi2"
4431   [(set (match_operand:HI 0 "register_operand" "=r")
4432         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4433   ""
4434   "rev16\\t%w0, %w1"
4435   [(set_attr "type" "rev")]
4436 )
4437
4438 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4439 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4440 ;; each valid permutation.
4441
4442 (define_insn "rev16<mode>2"
4443   [(set (match_operand:GPI 0 "register_operand" "=r")
4444         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4445                                       (const_int 8))
4446                           (match_operand:GPI 3 "const_int_operand" "n"))
4447                  (and:GPI (lshiftrt:GPI (match_dup 1)
4448                                         (const_int 8))
4449                           (match_operand:GPI 2 "const_int_operand" "n"))))]
4450   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4451    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4452   "rev16\\t%<w>0, %<w>1"
4453   [(set_attr "type" "rev")]
4454 )
4455
4456 (define_insn "rev16<mode>2_alt"
4457   [(set (match_operand:GPI 0 "register_operand" "=r")
4458         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4459                                         (const_int 8))
4460                           (match_operand:GPI 2 "const_int_operand" "n"))
4461                  (and:GPI (ashift:GPI (match_dup 1)
4462                                       (const_int 8))
4463                           (match_operand:GPI 3 "const_int_operand" "n"))))]
4464   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4465    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4466   "rev16\\t%<w>0, %<w>1"
4467   [(set_attr "type" "rev")]
4468 )
4469
4470 ;; zero_extend version of above
4471 (define_insn "*bswapsi2_uxtw"
4472   [(set (match_operand:DI 0 "register_operand" "=r")
4473         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4474   ""
4475   "rev\\t%w0, %w1"
4476   [(set_attr "type" "rev")]
4477 )
4478
4479 ;; -------------------------------------------------------------------
4480 ;; Floating-point intrinsics
4481 ;; -------------------------------------------------------------------
4482
4483 ;; frint floating-point round to integral standard patterns.
4484 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4485
4486 (define_insn "<frint_pattern><mode>2"
4487   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4488         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4489          FRINT))]
4490   "TARGET_FLOAT"
4491   "frint<frint_suffix>\\t%<s>0, %<s>1"
4492   [(set_attr "type" "f_rint<stype>")]
4493 )
4494
4495 ;; frcvt floating-point round to integer and convert standard patterns.
4496 ;; Expands to lbtrunc, lceil, lfloor, lround.
4497 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
4498   [(set (match_operand:GPI 0 "register_operand" "=r")
4499         (FIXUORS:GPI
4500           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4501            FCVT)))]
4502   "TARGET_FLOAT"
4503   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
4504   [(set_attr "type" "f_cvtf2i")]
4505 )
4506
4507 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4508   [(set (match_operand:GPI 0 "register_operand" "=r")
4509         (FIXUORS:GPI
4510           (mult:GPF
4511             (match_operand:GPF 1 "register_operand" "w")
4512             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4513   "TARGET_FLOAT
4514    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4515                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4516   {
4517     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4518     char buf[64];
4519     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4520     output_asm_insn (buf, operands);
4521     return "";
4522   }
4523   [(set_attr "type" "f_cvtf2i")]
4524 )
4525
4526 ;; fma - no throw
4527
4528 (define_insn "fma<mode>4"
4529   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4530         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
4531                      (match_operand:GPF_F16 2 "register_operand" "w")
4532                      (match_operand:GPF_F16 3 "register_operand" "w")))]
4533   "TARGET_FLOAT"
4534   "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4535   [(set_attr "type" "fmac<stype>")]
4536 )
4537
4538 (define_insn "fnma<mode>4"
4539   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4540         (fma:GPF_F16
4541           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w"))
4542           (match_operand:GPF_F16 2 "register_operand" "w")
4543           (match_operand:GPF_F16 3 "register_operand" "w")))]
4544   "TARGET_FLOAT"
4545   "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4546   [(set_attr "type" "fmac<stype>")]
4547 )
4548
4549 (define_insn "fms<mode>4"
4550   [(set (match_operand:GPF 0 "register_operand" "=w")
4551         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4552                  (match_operand:GPF 2 "register_operand" "w")
4553                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4554   "TARGET_FLOAT"
4555   "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4556   [(set_attr "type" "fmac<s>")]
4557 )
4558
4559 (define_insn "fnms<mode>4"
4560   [(set (match_operand:GPF 0 "register_operand" "=w")
4561         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4562                  (match_operand:GPF 2 "register_operand" "w")
4563                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4564   "TARGET_FLOAT"
4565   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4566   [(set_attr "type" "fmac<s>")]
4567 )
4568
4569 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4570 (define_insn "*fnmadd<mode>4"
4571   [(set (match_operand:GPF 0 "register_operand" "=w")
4572         (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4573                           (match_operand:GPF 2 "register_operand" "w")
4574                           (match_operand:GPF 3 "register_operand" "w"))))]
4575   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
4576   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4577   [(set_attr "type" "fmac<s>")]
4578 )
4579
4580 ;; -------------------------------------------------------------------
4581 ;; Floating-point conversions
4582 ;; -------------------------------------------------------------------
4583
4584 (define_insn "extendsfdf2"
4585   [(set (match_operand:DF 0 "register_operand" "=w")
4586         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
4587   "TARGET_FLOAT"
4588   "fcvt\\t%d0, %s1"
4589   [(set_attr "type" "f_cvt")]
4590 )
4591
4592 (define_insn "extendhfsf2"
4593   [(set (match_operand:SF 0 "register_operand" "=w")
4594         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
4595   "TARGET_FLOAT"
4596   "fcvt\\t%s0, %h1"
4597   [(set_attr "type" "f_cvt")]
4598 )
4599
4600 (define_insn "extendhfdf2"
4601   [(set (match_operand:DF 0 "register_operand" "=w")
4602         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
4603   "TARGET_FLOAT"
4604   "fcvt\\t%d0, %h1"
4605   [(set_attr "type" "f_cvt")]
4606 )
4607
4608 (define_insn "truncdfsf2"
4609   [(set (match_operand:SF 0 "register_operand" "=w")
4610         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
4611   "TARGET_FLOAT"
4612   "fcvt\\t%s0, %d1"
4613   [(set_attr "type" "f_cvt")]
4614 )
4615
4616 (define_insn "truncsfhf2"
4617   [(set (match_operand:HF 0 "register_operand" "=w")
4618         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
4619   "TARGET_FLOAT"
4620   "fcvt\\t%h0, %s1"
4621   [(set_attr "type" "f_cvt")]
4622 )
4623
4624 (define_insn "truncdfhf2"
4625   [(set (match_operand:HF 0 "register_operand" "=w")
4626         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
4627   "TARGET_FLOAT"
4628   "fcvt\\t%h0, %d1"
4629   [(set_attr "type" "f_cvt")]
4630 )
4631
4632 (define_insn "<optab>_trunc<GPF_F16:mode><GPI:mode>2"
4633   [(set (match_operand:GPI 0 "register_operand" "=r")
4634         (FIXUORS:GPI (match_operand:GPF_F16 1 "register_operand" "w")))]
4635   "TARGET_FLOAT"
4636   "fcvtz<su>\t%<GPI:w>0, %<GPF_F16:s>1"
4637   [(set_attr "type" "f_cvtf2i")]
4638 )
4639
4640 (define_insn "<optab><fcvt_target><GPF:mode>2"
4641   [(set (match_operand:GPF 0 "register_operand" "=w,w")
4642         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
4643   "TARGET_FLOAT"
4644   "@
4645    <su_optab>cvtf\t%<GPF:s>0, %<s>1
4646    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
4647   [(set_attr "simd" "yes,no")
4648    (set_attr "fp" "no,yes")
4649    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
4650 )
4651
4652 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
4653   [(set (match_operand:GPF 0 "register_operand" "=w")
4654         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
4655   "TARGET_FLOAT"
4656   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
4657   [(set_attr "type" "f_cvti2f")]
4658 )
4659
4660 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
4661 ;; midend will arrange for an SImode conversion to HFmode to first go
4662 ;; through DFmode, then to HFmode.  But first it will try converting
4663 ;; to DImode then down, which would match our DImode pattern below and
4664 ;; give very poor code-generation.  So, we must provide our own emulation
4665 ;; of the mid-end logic.
4666
4667 (define_insn "aarch64_fp16_<optab><mode>hf2"
4668   [(set (match_operand:HF 0 "register_operand" "=w")
4669         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
4670   "TARGET_FP_F16INST"
4671   "<su_optab>cvtf\t%h0, %<w>1"
4672   [(set_attr "type" "f_cvti2f")]
4673 )
4674
4675 (define_expand "<optab>sihf2"
4676   [(set (match_operand:HF 0 "register_operand")
4677         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
4678   "TARGET_FLOAT"
4679 {
4680   if (TARGET_FP_F16INST)
4681     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
4682   else
4683     {
4684       rtx convert_target = gen_reg_rtx (DFmode);
4685       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
4686       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
4687     }
4688   DONE;
4689 }
4690 )
4691
4692 ;; For DImode there is no wide enough floating-point mode that we
4693 ;; can convert through natively (TFmode would work, but requires a library
4694 ;; call).  However, we know that any value >= 65504 will be rounded
4695 ;; to infinity on conversion.  This is well within the range of SImode, so
4696 ;; we can:
4697 ;;   Saturate to SImode.
4698 ;;   Convert from that to DFmode
4699 ;;   Convert from that to HFmode (phew!).
4700 ;; Note that the saturation to SImode requires the SIMD extensions.  If
4701 ;; we ever need to provide this pattern where the SIMD extensions are not
4702 ;; available, we would need a different approach.
4703
4704 (define_expand "<optab>dihf2"
4705   [(set (match_operand:HF 0 "register_operand")
4706         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
4707   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
4708 {
4709   if (TARGET_FP_F16INST)
4710     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
4711   else
4712     {
4713       rtx sat_target = gen_reg_rtx (SImode);
4714       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
4715       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
4716     }
4717
4718   DONE;
4719 }
4720 )
4721
4722 ;; Convert between fixed-point and floating-point (scalar modes)
4723
4724 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
4725   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
4726         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
4727                                    (match_operand:SI 2 "immediate_operand" "i, i")]
4728          FCVT_F2FIXED))]
4729   ""
4730   "@
4731    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
4732    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
4733   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
4734    (set_attr "fp" "yes, *")
4735    (set_attr "simd" "*, yes")]
4736 )
4737
4738 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
4739   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
4740         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
4741                                    (match_operand:SI 2 "immediate_operand" "i, i")]
4742          FCVT_FIXED2F))]
4743   ""
4744   "@
4745    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
4746    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
4747   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
4748    (set_attr "fp" "yes, *")
4749    (set_attr "simd" "*, yes")]
4750 )
4751
4752 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
4753   [(set (match_operand:GPI 0 "register_operand" "=r")
4754         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
4755                      (match_operand:SI 2 "immediate_operand" "i")]
4756          FCVT_F2FIXED))]
4757   "TARGET_FP_F16INST"
4758    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
4759   [(set_attr "type" "f_cvtf2i")]
4760 )
4761
4762 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
4763   [(set (match_operand:HF 0 "register_operand" "=w")
4764         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
4765                     (match_operand:SI 2 "immediate_operand" "i")]
4766          FCVT_FIXED2F))]
4767   "TARGET_FP_F16INST"
4768   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
4769   [(set_attr "type" "f_cvti2f")]
4770 )
4771
4772 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
4773   [(set (match_operand:HI 0 "register_operand" "=w")
4774         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
4775                     (match_operand:SI 2 "immediate_operand" "i")]
4776          FCVT_F2FIXED))]
4777   "TARGET_SIMD"
4778   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
4779   [(set_attr "type" "neon_fp_to_int_s")]
4780 )
4781
4782 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
4783   [(set (match_operand:HF 0 "register_operand" "=w")
4784         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
4785                     (match_operand:SI 2 "immediate_operand" "i")]
4786          FCVT_FIXED2F))]
4787   "TARGET_SIMD"
4788   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
4789   [(set_attr "type" "neon_int_to_fp_s")]
4790 )
4791
4792 ;; -------------------------------------------------------------------
4793 ;; Floating-point arithmetic
4794 ;; -------------------------------------------------------------------
4795
4796 (define_insn "add<mode>3"
4797   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4798         (plus:GPF_F16
4799          (match_operand:GPF_F16 1 "register_operand" "w")
4800          (match_operand:GPF_F16 2 "register_operand" "w")))]
4801   "TARGET_FLOAT"
4802   "fadd\\t%<s>0, %<s>1, %<s>2"
4803   [(set_attr "type" "fadd<stype>")]
4804 )
4805
4806 (define_insn "sub<mode>3"
4807   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4808         (minus:GPF_F16
4809          (match_operand:GPF_F16 1 "register_operand" "w")
4810          (match_operand:GPF_F16 2 "register_operand" "w")))]
4811   "TARGET_FLOAT"
4812   "fsub\\t%<s>0, %<s>1, %<s>2"
4813   [(set_attr "type" "fadd<stype>")]
4814 )
4815
4816 (define_insn "mul<mode>3"
4817   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4818         (mult:GPF_F16
4819          (match_operand:GPF_F16 1 "register_operand" "w")
4820          (match_operand:GPF_F16 2 "register_operand" "w")))]
4821   "TARGET_FLOAT"
4822   "fmul\\t%<s>0, %<s>1, %<s>2"
4823   [(set_attr "type" "fmul<stype>")]
4824 )
4825
4826 (define_insn "*fnmul<mode>3"
4827   [(set (match_operand:GPF 0 "register_operand" "=w")
4828         (mult:GPF
4829                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4830                  (match_operand:GPF 2 "register_operand" "w")))]
4831   "TARGET_FLOAT && !flag_rounding_math"
4832   "fnmul\\t%<s>0, %<s>1, %<s>2"
4833   [(set_attr "type" "fmul<s>")]
4834 )
4835
4836 (define_insn "*fnmul<mode>3"
4837   [(set (match_operand:GPF 0 "register_operand" "=w")
4838         (neg:GPF (mult:GPF
4839                  (match_operand:GPF 1 "register_operand" "w")
4840                  (match_operand:GPF 2 "register_operand" "w"))))]
4841   "TARGET_FLOAT"
4842   "fnmul\\t%<s>0, %<s>1, %<s>2"
4843   [(set_attr "type" "fmul<s>")]
4844 )
4845
4846 (define_expand "div<mode>3"
4847  [(set (match_operand:GPF_F16 0 "register_operand")
4848        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
4849                     (match_operand:GPF_F16 2 "register_operand")))]
4850  "TARGET_SIMD"
4851 {
4852   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
4853     DONE;
4854
4855   operands[1] = force_reg (<MODE>mode, operands[1]);
4856 })
4857
4858 (define_insn "*div<mode>3"
4859   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4860         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
4861                      (match_operand:GPF_F16 2 "register_operand" "w")))]
4862   "TARGET_FLOAT"
4863   "fdiv\\t%<s>0, %<s>1, %<s>2"
4864   [(set_attr "type" "fdiv<stype>")]
4865 )
4866
4867 (define_insn "neg<mode>2"
4868   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4869         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
4870   "TARGET_FLOAT"
4871   "fneg\\t%<s>0, %<s>1"
4872   [(set_attr "type" "ffarith<stype>")]
4873 )
4874
4875 (define_expand "sqrt<mode>2"
4876   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4877         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
4878   "TARGET_FLOAT"
4879 {
4880   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
4881     DONE;
4882 })
4883
4884 (define_insn "*sqrt<mode>2"
4885   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4886         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
4887   "TARGET_FLOAT"
4888   "fsqrt\\t%<s>0, %<s>1"
4889   [(set_attr "type" "fsqrt<stype>")]
4890 )
4891
4892 (define_insn "abs<mode>2"
4893   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4894         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
4895   "TARGET_FLOAT"
4896   "fabs\\t%<s>0, %<s>1"
4897   [(set_attr "type" "ffarith<stype>")]
4898 )
4899
4900 ;; Given that smax/smin do not specify the result when either input is NaN,
4901 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
4902 ;; for smin.
4903
4904 (define_insn "smax<mode>3"
4905   [(set (match_operand:GPF 0 "register_operand" "=w")
4906         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
4907                   (match_operand:GPF 2 "register_operand" "w")))]
4908   "TARGET_FLOAT"
4909   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
4910   [(set_attr "type" "f_minmax<s>")]
4911 )
4912
4913 (define_insn "smin<mode>3"
4914   [(set (match_operand:GPF 0 "register_operand" "=w")
4915         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
4916                   (match_operand:GPF 2 "register_operand" "w")))]
4917   "TARGET_FLOAT"
4918   "fminnm\\t%<s>0, %<s>1, %<s>2"
4919   [(set_attr "type" "f_minmax<s>")]
4920 )
4921
4922 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
4923 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
4924 ;; which implement the IEEE fmax ()/fmin () functions.
4925 (define_insn "<maxmin_uns><mode>3"
4926   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4927         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
4928                      (match_operand:GPF_F16 2 "register_operand" "w")]
4929                      FMAXMIN_UNS))]
4930   "TARGET_FLOAT"
4931   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
4932   [(set_attr "type" "f_minmax<stype>")]
4933 )
4934
4935 ;; For copysign (x, y), we want to generate:
4936 ;;
4937 ;;   LDR d2, #(1 << 63)
4938 ;;   BSL v2.8b, [y], [x]
4939 ;;
4940 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
4941 ;; aarch64_simd_bsldf will select the best suited of these instructions
4942 ;; to generate based on register allocation, and knows how to partially
4943 ;; constant fold based on the values of X and Y, so expand through that.
4944
4945 (define_expand "copysigndf3"
4946   [(match_operand:DF 0 "register_operand")
4947    (match_operand:DF 1 "register_operand")
4948    (match_operand:DF 2 "register_operand")]
4949   "TARGET_FLOAT && TARGET_SIMD"
4950 {
4951   rtx mask = gen_reg_rtx (DImode);
4952   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
4953   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
4954                                      operands[2], operands[1]));
4955   DONE;
4956 }
4957 )
4958
4959 ;; As above, but we must first get to a 64-bit value if we wish to use
4960 ;; aarch64_simd_bslv2sf.
4961
4962 (define_expand "copysignsf3"
4963   [(match_operand:SF 0 "register_operand")
4964    (match_operand:SF 1 "register_operand")
4965    (match_operand:SF 2 "register_operand")]
4966   "TARGET_FLOAT && TARGET_SIMD"
4967 {
4968   rtx mask = gen_reg_rtx (DImode);
4969
4970   /* Juggle modes to get us in to a vector mode for BSL.  */
4971   rtx op1 = lowpart_subreg (V2SFmode, operands[1], SFmode);
4972   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
4973   rtx tmp = gen_reg_rtx (V2SFmode);
4974   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 31));
4975   emit_insn (gen_aarch64_simd_bslv2sf (tmp, mask, op2, op1));
4976   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
4977   DONE;
4978 }
4979 )
4980
4981 ;; -------------------------------------------------------------------
4982 ;; Reload support
4983 ;; -------------------------------------------------------------------
4984 ;; Reload Scalar Floating point modes from constant pool.
4985 ;; The AArch64 port doesn't have __int128 constant move support.
4986 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
4987  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
4988        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
4989   (clobber (match_operand:P 2 "register_operand" "=&r"))]
4990  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
4991  {
4992    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
4993    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
4994    DONE;
4995  }
4996 )
4997
4998 ;; Reload Vector modes from constant pool.
4999 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
5000  [(set (match_operand:VALL 0 "register_operand" "=w")
5001        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
5002   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5003  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
5004  {
5005    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5006    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
5007    DONE;
5008  }
5009 )
5010
5011 (define_expand "aarch64_reload_mov<mode>"
5012   [(set (match_operand:TX 0 "register_operand" "=w")
5013         (match_operand:TX 1 "register_operand" "w"))
5014    (clobber (match_operand:DI 2 "register_operand" "=&r"))
5015   ]
5016   "TARGET_FLOAT"
5017   {
5018     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
5019     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
5020     gen_aarch64_movtilow_tilow (op0, op1);
5021     gen_aarch64_movdi_tihigh (operands[2], op1);
5022     gen_aarch64_movtihigh_di (op0, operands[2]);
5023     DONE;
5024   }
5025 )
5026
5027 ;; The following secondary reload helpers patterns are invoked
5028 ;; after or during reload as we don't want these patterns to start
5029 ;; kicking in during the combiner.
5030
5031 (define_insn "aarch64_movdi_<mode>low"
5032   [(set (match_operand:DI 0 "register_operand" "=r")
5033         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5034                          (const_int 64) (const_int 0)))]
5035   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5036   "fmov\\t%x0, %d1"
5037   [(set_attr "type" "f_mrc")
5038    (set_attr "length" "4")
5039   ])
5040
5041 (define_insn "aarch64_movdi_<mode>high"
5042   [(set (match_operand:DI 0 "register_operand" "=r")
5043         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5044                          (const_int 64) (const_int 64)))]
5045   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5046   "fmov\\t%x0, %1.d[1]"
5047   [(set_attr "type" "f_mrc")
5048    (set_attr "length" "4")
5049   ])
5050
5051 (define_insn "aarch64_mov<mode>high_di"
5052   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
5053                          (const_int 64) (const_int 64))
5054         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5055   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5056   "fmov\\t%0.d[1], %x1"
5057   [(set_attr "type" "f_mcr")
5058    (set_attr "length" "4")
5059   ])
5060
5061 (define_insn "aarch64_mov<mode>low_di"
5062   [(set (match_operand:TX 0 "register_operand" "=w")
5063         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5064   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5065   "fmov\\t%d0, %x1"
5066   [(set_attr "type" "f_mcr")
5067    (set_attr "length" "4")
5068   ])
5069
5070 (define_insn "aarch64_movtilow_tilow"
5071   [(set (match_operand:TI 0 "register_operand" "=w")
5072         (zero_extend:TI
5073           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
5074   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5075   "fmov\\t%d0, %d1"
5076   [(set_attr "type" "fmov")
5077    (set_attr "length" "4")
5078   ])
5079
5080 ;; There is a deliberate reason why the parameters of high and lo_sum's
5081 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
5082 ;; and lo_sum's to be used with the labels defining the jump tables in
5083 ;; rodata section.
5084
5085 (define_expand "add_losym"
5086   [(set (match_operand 0 "register_operand" "=r")
5087         (lo_sum (match_operand 1 "register_operand" "r")
5088                 (match_operand 2 "aarch64_valid_symref" "S")))]
5089   ""
5090 {
5091   machine_mode mode = GET_MODE (operands[0]);
5092
5093   emit_insn ((mode == DImode
5094               ? gen_add_losym_di
5095               : gen_add_losym_si) (operands[0],
5096                                    operands[1],
5097                                    operands[2]));
5098   DONE;
5099 })
5100
5101 (define_insn "add_losym_<mode>"
5102   [(set (match_operand:P 0 "register_operand" "=r")
5103         (lo_sum:P (match_operand:P 1 "register_operand" "r")
5104                   (match_operand 2 "aarch64_valid_symref" "S")))]
5105   ""
5106   "add\\t%<w>0, %<w>1, :lo12:%a2"
5107   [(set_attr "type" "alu_imm")]
5108 )
5109
5110 (define_insn "ldr_got_small_<mode>"
5111   [(set (match_operand:PTR 0 "register_operand" "=r")
5112         (unspec:PTR [(mem:PTR (lo_sum:PTR
5113                               (match_operand:PTR 1 "register_operand" "r")
5114                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5115                     UNSPEC_GOTSMALLPIC))]
5116   ""
5117   "ldr\\t%<w>0, [%1, #:got_lo12:%a2]"
5118   [(set_attr "type" "load1")]
5119 )
5120
5121 (define_insn "ldr_got_small_sidi"
5122   [(set (match_operand:DI 0 "register_operand" "=r")
5123         (zero_extend:DI
5124          (unspec:SI [(mem:SI (lo_sum:DI
5125                              (match_operand:DI 1 "register_operand" "r")
5126                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5127                     UNSPEC_GOTSMALLPIC)))]
5128   "TARGET_ILP32"
5129   "ldr\\t%w0, [%1, #:got_lo12:%a2]"
5130   [(set_attr "type" "load1")]
5131 )
5132
5133 (define_insn "ldr_got_small_28k_<mode>"
5134   [(set (match_operand:PTR 0 "register_operand" "=r")
5135         (unspec:PTR [(mem:PTR (lo_sum:PTR
5136                               (match_operand:PTR 1 "register_operand" "r")
5137                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5138                     UNSPEC_GOTSMALLPIC28K))]
5139   ""
5140   "ldr\\t%<w>0, [%1, #:<got_modifier>:%a2]"
5141   [(set_attr "type" "load1")]
5142 )
5143
5144 (define_insn "ldr_got_small_28k_sidi"
5145   [(set (match_operand:DI 0 "register_operand" "=r")
5146         (zero_extend:DI
5147          (unspec:SI [(mem:SI (lo_sum:DI
5148                              (match_operand:DI 1 "register_operand" "r")
5149                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5150                     UNSPEC_GOTSMALLPIC28K)))]
5151   "TARGET_ILP32"
5152   "ldr\\t%w0, [%1, #:gotpage_lo14:%a2]"
5153   [(set_attr "type" "load1")]
5154 )
5155
5156 (define_insn "ldr_got_tiny"
5157   [(set (match_operand:DI 0 "register_operand" "=r")
5158         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
5159                    UNSPEC_GOTTINYPIC))]
5160   ""
5161   "ldr\\t%0, %L1"
5162   [(set_attr "type" "load1")]
5163 )
5164
5165 (define_insn "aarch64_load_tp_hard"
5166   [(set (match_operand:DI 0 "register_operand" "=r")
5167         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
5168   ""
5169   "mrs\\t%0, tpidr_el0"
5170   [(set_attr "type" "mrs")]
5171 )
5172
5173 ;; The TLS ABI specifically requires that the compiler does not schedule
5174 ;; instructions in the TLS stubs, in order to enable linker relaxation.
5175 ;; Therefore we treat the stubs as an atomic sequence.
5176 (define_expand "tlsgd_small"
5177  [(parallel [(set (match_operand 0 "register_operand" "")
5178                   (call (mem:DI (match_dup 2)) (const_int 1)))
5179              (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
5180              (clobber (reg:DI LR_REGNUM))])]
5181  ""
5182 {
5183   operands[2] = aarch64_tls_get_addr ();
5184 })
5185
5186 (define_insn "*tlsgd_small"
5187   [(set (match_operand 0 "register_operand" "")
5188         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
5189    (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
5190    (clobber (reg:DI LR_REGNUM))
5191   ]
5192   ""
5193   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
5194   [(set_attr "type" "call")
5195    (set_attr "length" "16")])
5196
5197 (define_insn "tlsie_small_<mode>"
5198   [(set (match_operand:PTR 0 "register_operand" "=r")
5199         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5200                    UNSPEC_GOTSMALLTLS))]
5201   ""
5202   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
5203   [(set_attr "type" "load1")
5204    (set_attr "length" "8")]
5205 )
5206
5207 (define_insn "tlsie_small_sidi"
5208   [(set (match_operand:DI 0 "register_operand" "=r")
5209         (zero_extend:DI
5210           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5211                       UNSPEC_GOTSMALLTLS)))]
5212   ""
5213   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
5214   [(set_attr "type" "load1")
5215    (set_attr "length" "8")]
5216 )
5217
5218 (define_insn "tlsie_tiny_<mode>"
5219   [(set (match_operand:PTR 0 "register_operand" "=&r")
5220         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
5221                      (match_operand:PTR 2 "register_operand" "r")]
5222                    UNSPEC_GOTTINYTLS))]
5223   ""
5224   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
5225   [(set_attr "type" "multiple")
5226    (set_attr "length" "8")]
5227 )
5228
5229 (define_insn "tlsie_tiny_sidi"
5230   [(set (match_operand:DI 0 "register_operand" "=&r")
5231         (zero_extend:DI
5232           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
5233                       (match_operand:DI 2 "register_operand" "r")
5234                       ]
5235                       UNSPEC_GOTTINYTLS)))]
5236   ""
5237   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5238   [(set_attr "type" "multiple")
5239    (set_attr "length" "8")]
5240 )
5241
5242 (define_insn "tlsle12_<mode>"
5243   [(set (match_operand:P 0 "register_operand" "=r")
5244         (unspec:P [(match_operand:P 1 "register_operand" "r")
5245                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5246                    UNSPEC_TLSLE12))]
5247   ""
5248   "add\\t%<w>0, %<w>1, #%L2";
5249   [(set_attr "type" "alu_sreg")
5250    (set_attr "length" "4")]
5251 )
5252
5253 (define_insn "tlsle24_<mode>"
5254   [(set (match_operand:P 0 "register_operand" "=r")
5255         (unspec:P [(match_operand:P 1 "register_operand" "r")
5256                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5257                    UNSPEC_TLSLE24))]
5258   ""
5259   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
5260   [(set_attr "type" "multiple")
5261    (set_attr "length" "8")]
5262 )
5263
5264 (define_insn "tlsle32_<mode>"
5265   [(set (match_operand:P 0 "register_operand" "=r")
5266         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5267                    UNSPEC_TLSLE32))]
5268   ""
5269   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5270   [(set_attr "type" "multiple")
5271    (set_attr "length" "8")]
5272 )
5273
5274 (define_insn "tlsle48_<mode>"
5275   [(set (match_operand:P 0 "register_operand" "=r")
5276         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5277                    UNSPEC_TLSLE48))]
5278   ""
5279   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5280   [(set_attr "type" "multiple")
5281    (set_attr "length" "12")]
5282 )
5283
5284 (define_insn "tlsdesc_small_<mode>"
5285   [(set (reg:PTR R0_REGNUM)
5286         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5287                    UNSPEC_TLSDESC))
5288    (clobber (reg:DI LR_REGNUM))
5289    (clobber (reg:CC CC_REGNUM))
5290    (clobber (match_scratch:DI 1 "=r"))]
5291   "TARGET_TLS_DESC"
5292   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5293   [(set_attr "type" "call")
5294    (set_attr "length" "16")])
5295
5296 (define_insn "stack_tie"
5297   [(set (mem:BLK (scratch))
5298         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
5299                      (match_operand:DI 1 "register_operand" "rk")]
5300                     UNSPEC_PRLG_STK))]
5301   ""
5302   ""
5303   [(set_attr "length" "0")]
5304 )
5305
5306 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
5307 ;; all of memory.  This blocks insns from being moved across this point.
5308
5309 (define_insn "blockage"
5310   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
5311   ""
5312   ""
5313   [(set_attr "length" "0")
5314    (set_attr "type" "block")]
5315 )
5316
5317 (define_insn "probe_stack_range_<PTR:mode>"
5318   [(set (match_operand:PTR 0 "register_operand" "=r")
5319         (unspec_volatile:PTR [(match_operand:PTR 1 "register_operand" "0")
5320                               (match_operand:PTR 2 "register_operand" "r")]
5321                                UNSPECV_PROBE_STACK_RANGE))]
5322   ""
5323 {
5324   return aarch64_output_probe_stack_range (operands[0], operands[2]);
5325 }
5326   [(set_attr "length" "32")]
5327 )
5328
5329 ;; Named pattern for expanding thread pointer reference.
5330 (define_expand "get_thread_pointerdi"
5331   [(match_operand:DI 0 "register_operand" "=r")]
5332   ""
5333 {
5334   rtx tmp = aarch64_load_tp (operands[0]);
5335   if (tmp != operands[0])
5336     emit_move_insn (operands[0], tmp);
5337   DONE;
5338 })
5339
5340 ;; Named patterns for stack smashing protection.
5341 (define_expand "stack_protect_set"
5342   [(match_operand 0 "memory_operand")
5343    (match_operand 1 "memory_operand")]
5344   ""
5345 {
5346   machine_mode mode = GET_MODE (operands[0]);
5347
5348   emit_insn ((mode == DImode
5349               ? gen_stack_protect_set_di
5350               : gen_stack_protect_set_si) (operands[0], operands[1]));
5351   DONE;
5352 })
5353
5354 (define_insn "stack_protect_set_<mode>"
5355   [(set (match_operand:PTR 0 "memory_operand" "=m")
5356         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
5357          UNSPEC_SP_SET))
5358    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
5359   ""
5360   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
5361   [(set_attr "length" "12")
5362    (set_attr "type" "multiple")])
5363
5364 (define_expand "stack_protect_test"
5365   [(match_operand 0 "memory_operand")
5366    (match_operand 1 "memory_operand")
5367    (match_operand 2)]
5368   ""
5369 {
5370   rtx result;
5371   machine_mode mode = GET_MODE (operands[0]);
5372
5373   result = gen_reg_rtx(mode);
5374
5375   emit_insn ((mode == DImode
5376               ? gen_stack_protect_test_di
5377               : gen_stack_protect_test_si) (result,
5378                                             operands[0],
5379                                             operands[1]));
5380
5381   if (mode == DImode)
5382     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5383                                     result, const0_rtx, operands[2]));
5384   else
5385     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5386                                     result, const0_rtx, operands[2]));
5387   DONE;
5388 })
5389
5390 (define_insn "stack_protect_test_<mode>"
5391   [(set (match_operand:PTR 0 "register_operand" "=r")
5392         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
5393                      (match_operand:PTR 2 "memory_operand" "m")]
5394          UNSPEC_SP_TEST))
5395    (clobber (match_scratch:PTR 3 "=&r"))]
5396   ""
5397   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
5398   [(set_attr "length" "12")
5399    (set_attr "type" "multiple")])
5400
5401 ;; Write Floating-point Control Register.
5402 (define_insn "set_fpcr"
5403   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
5404   ""
5405   "msr\\tfpcr, %0"
5406   [(set_attr "type" "mrs")])
5407
5408 ;; Read Floating-point Control Register.
5409 (define_insn "get_fpcr"
5410   [(set (match_operand:SI 0 "register_operand" "=r")
5411         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
5412   ""
5413   "mrs\\t%0, fpcr"
5414   [(set_attr "type" "mrs")])
5415
5416 ;; Write Floating-point Status Register.
5417 (define_insn "set_fpsr"
5418   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
5419   ""
5420   "msr\\tfpsr, %0"
5421   [(set_attr "type" "mrs")])
5422
5423 ;; Read Floating-point Status Register.
5424 (define_insn "get_fpsr"
5425   [(set (match_operand:SI 0 "register_operand" "=r")
5426         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
5427   ""
5428   "mrs\\t%0, fpsr"
5429   [(set_attr "type" "mrs")])
5430
5431
5432 ;; Define the subtract-one-and-jump insns so loop.c
5433 ;; knows what to generate.
5434 (define_expand "doloop_end"
5435   [(use (match_operand 0 "" ""))      ; loop pseudo
5436    (use (match_operand 1 "" ""))]     ; label
5437   "optimize > 0 && flag_modulo_sched"
5438 {
5439   rtx s0;
5440   rtx bcomp;
5441   rtx loc_ref;
5442   rtx cc_reg;
5443   rtx insn;
5444   rtx cmp;
5445
5446   /* Currently SMS relies on the do-loop pattern to recognize loops
5447      where (1) the control part consists of all insns defining and/or
5448      using a certain 'count' register and (2) the loop count can be
5449      adjusted by modifying this register prior to the loop.
5450      ??? The possible introduction of a new block to initialize the
5451      new IV can potentially affect branch optimizations.  */
5452
5453   if (GET_MODE (operands[0]) != DImode)
5454     FAIL;
5455
5456   s0 = operands [0];
5457   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
5458
5459   cmp = XVECEXP (PATTERN (insn), 0, 0);
5460   cc_reg = SET_DEST (cmp);
5461   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
5462   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
5463   emit_jump_insn (gen_rtx_SET (pc_rtx,
5464                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
5465                                                      loc_ref, pc_rtx)));
5466   DONE;
5467 })
5468
5469 ;; AdvSIMD Stuff
5470 (include "aarch64-simd.md")
5471
5472 ;; Atomic Operations
5473 (include "atomics.md")
5474
5475 ;; ldp/stp peephole patterns
5476 (include "aarch64-ldpstp.md")