gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2015 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V15_REGNUM         47)
  61     (V31_REGNUM         63)
  62     (SFP_REGNUM         64)
  63     (AP_REGNUM          65)
  64     (CC_REGNUM          66)
  65   ]
  66 )
  67
  68 (define_c_enum "unspec" [
  69     UNSPEC_CASESI
  70     UNSPEC_CRC32B
  71     UNSPEC_CRC32CB
  72     UNSPEC_CRC32CH
  73     UNSPEC_CRC32CW
  74     UNSPEC_CRC32CX
  75     UNSPEC_CRC32H
  76     UNSPEC_CRC32W
  77     UNSPEC_CRC32X
  78     UNSPEC_URECPE
  79     UNSPEC_FRECPE
  80     UNSPEC_FRECPS
  81     UNSPEC_FRECPX
  82     UNSPEC_FRINTA
  83     UNSPEC_FRINTI
  84     UNSPEC_FRINTM
  85     UNSPEC_FRINTN
  86     UNSPEC_FRINTP
  87     UNSPEC_FRINTX
  88     UNSPEC_FRINTZ
  89     UNSPEC_GOTSMALLPIC
  90     UNSPEC_GOTSMALLPIC28K
  91     UNSPEC_GOTSMALLTLS
  92     UNSPEC_GOTTINYPIC
  93     UNSPEC_GOTTINYTLS
  94     UNSPEC_LD1
  95     UNSPEC_LD2
  96     UNSPEC_LD2_DUP
  97     UNSPEC_LD3
  98     UNSPEC_LD3_DUP
  99     UNSPEC_LD4
 100     UNSPEC_LD4_DUP
 101     UNSPEC_LD2_LANE
 102     UNSPEC_LD3_LANE
 103     UNSPEC_LD4_LANE
 104     UNSPEC_MB
 105     UNSPEC_NOP
 106     UNSPEC_PRLG_STK
 107     UNSPEC_RBIT
 108     UNSPEC_SISD_NEG
 109     UNSPEC_SISD_SSHL
 110     UNSPEC_SISD_USHL
 111     UNSPEC_SSHL_2S
 112     UNSPEC_ST1
 113     UNSPEC_ST2
 114     UNSPEC_ST3
 115     UNSPEC_ST4
 116     UNSPEC_ST2_LANE
 117     UNSPEC_ST3_LANE
 118     UNSPEC_ST4_LANE
 119     UNSPEC_TLS
 120     UNSPEC_TLSDESC
 121     UNSPEC_TLSLE12
 122     UNSPEC_TLSLE24
 123     UNSPEC_TLSLE32
 124     UNSPEC_TLSLE48
 125     UNSPEC_USHL_2S
 126     UNSPEC_VSTRUCTDUMMY
 127     UNSPEC_SP_SET
 128     UNSPEC_SP_TEST
 129 ])
 130
 131 (define_c_enum "unspecv" [
 132     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 133     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 134     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 135     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 136     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 137   ]
 138 )
 139
 140 ;; If further include files are added the defintion of MD_INCLUDES
 141 ;; must be updated.
 142
 143 (include "constraints.md")
 144 (include "predicates.md")
 145 (include "iterators.md")
 146
 147 ;; -------------------------------------------------------------------
 148 ;; Instruction types and attributes
 149 ;; -------------------------------------------------------------------
 150
 151 ; The "type" attribute is included here from AArch32 backend to be able
 152 ; to share pipeline descriptions.
 153 (include "../arm/types.md")
 154
 155 ;; It is important to set the fp or simd attributes to yes when a pattern
 156 ;; alternative uses the FP or SIMD register files, usually signified by use of
 157 ;; the 'w' constraint.  This will ensure that the alternative will be
 158 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 159 ;; architecture extensions.  If all the alternatives in a pattern use the
 160 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 161 ;; or TARGET_SIMD.
 162
 163 ;; Attribute that specifies whether or not the instruction touches fp
 164 ;; registers.  When this is set to yes for an alternative, that alternative
 165 ;; will be disabled when !TARGET_FLOAT.
 166 (define_attr "fp" "no,yes" (const_string "no"))
 167
 168 ;; Attribute that specifies whether or not the instruction touches simd
 169 ;; registers.  When this is set to yes for an alternative, that alternative
 170 ;; will be disabled when !TARGET_SIMD.
 171 (define_attr "simd" "no,yes" (const_string "no"))
 172
 173 (define_attr "length" ""
 174   (const_int 4))
 175
 176 ;; Attribute that controls whether an alternative is enabled or not.
 177 ;; Currently it is only used to disable alternatives which touch fp or simd
 178 ;; registers when -mgeneral-regs-only is specified.
 179 (define_attr "enabled" "no,yes"
 180   (cond [(ior
 181         (and (eq_attr "fp" "yes")
 182              (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 183         (and (eq_attr "simd" "yes")
 184              (eq (symbol_ref "TARGET_SIMD") (const_int 0))))
 185              (const_string "no")
 186         ] (const_string "yes")))
 187
 188 ;; Attribute that specifies whether we are dealing with a branch to a
 189 ;; label that is far away, i.e. further away than the maximum/minimum
 190 ;; representable in a signed 21-bits number.
 191 ;; 0 :=: no
 192 ;; 1 :=: yes
 193 (define_attr "far_branch" "" (const_int 0))
 194
 195 ;; -------------------------------------------------------------------
 196 ;; Pipeline descriptions and scheduling
 197 ;; -------------------------------------------------------------------
 198
 199 ;; Processor types.
 200 (include "aarch64-tune.md")
 201
 202 ;; Scheduling
 203 (include "../arm/cortex-a53.md")
 204 (include "../arm/cortex-a57.md")
 205 (include "thunderx.md")
 206 (include "../arm/xgene1.md")
 207
 208 ;; -------------------------------------------------------------------
 209 ;; Jumps and other miscellaneous insns
 210 ;; -------------------------------------------------------------------
 211
 212 (define_insn "indirect_jump"
 213   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 214   ""
 215   "br\\t%0"
 216   [(set_attr "type" "branch")]
 217 )
 218
 219 (define_insn "jump"
 220   [(set (pc) (label_ref (match_operand 0 "" "")))]
 221   ""
 222   "b\\t%l0"
 223   [(set_attr "type" "branch")]
 224 )
 225
 226 (define_expand "cbranch<mode>4"
 227   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 228                             [(match_operand:GPI 1 "register_operand" "")
 229                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 230                            (label_ref (match_operand 3 "" ""))
 231                            (pc)))]
 232   ""
 233   "
 234   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 235                                          operands[2]);
 236   operands[2] = const0_rtx;
 237   "
 238 )
 239
 240 (define_expand "cbranch<mode>4"
 241   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 242                             [(match_operand:GPF 1 "register_operand" "")
 243                              (match_operand:GPF 2 "aarch64_reg_or_zero" "")])
 244                            (label_ref (match_operand 3 "" ""))
 245                            (pc)))]
 246   ""
 247   "
 248   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 249                                          operands[2]);
 250   operands[2] = const0_rtx;
 251   "
 252 )
 253
 254 (define_expand "cbranchcc4"
 255   [(set (pc) (if_then_else
 256               (match_operator 0 "aarch64_comparison_operator"
 257                [(match_operand 1 "cc_register" "")
 258                 (match_operand 2 "const0_operand")])
 259               (label_ref (match_operand 3 "" ""))
 260               (pc)))]
 261   ""
 262   "")
 263
 264 (define_insn "ccmp_and<mode>"
 265   [(set (match_operand 1 "ccmp_cc_register" "")
 266         (compare
 267          (and:SI
 268           (match_operator 4 "aarch64_comparison_operator"
 269            [(match_operand 0 "ccmp_cc_register" "")
 270             (const_int 0)])
 271           (match_operator 5 "aarch64_comparison_operator"
 272            [(match_operand:GPI 2 "register_operand" "r,r,r")
 273             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 274          (const_int 0)))]
 275   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 276   "@
 277    ccmp\\t%<w>2, %<w>3, %k5, %m4
 278    ccmp\\t%<w>2, %<w>3, %k5, %m4
 279    ccmn\\t%<w>2, #%n3, %k5, %m4"
 280   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 281 )
 282
 283 (define_insn "ccmp_ior<mode>"
 284   [(set (match_operand 1 "ccmp_cc_register" "")
 285         (compare
 286          (ior:SI
 287           (match_operator 4 "aarch64_comparison_operator"
 288            [(match_operand 0 "ccmp_cc_register" "")
 289             (const_int 0)])
 290           (match_operator 5 "aarch64_comparison_operator"
 291            [(match_operand:GPI 2 "register_operand" "r,r,r")
 292             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 293          (const_int 0)))]
 294   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 295   "@
 296    ccmp\\t%<w>2, %<w>3, %K5, %M4
 297    ccmp\\t%<w>2, %<w>3, %K5, %M4
 298    ccmn\\t%<w>2, #%n3, %K5, %M4"
 299   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 300 )
 301
 302 (define_expand "cmp<mode>"
 303   [(set (match_operand 0 "cc_register" "")
 304         (match_operator:CC 1 "aarch64_comparison_operator"
 305          [(match_operand:GPI 2 "register_operand" "")
 306           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
 307   ""
 308   {
 309     operands[1] = gen_rtx_fmt_ee (COMPARE,
 310                                   SELECT_CC_MODE (GET_CODE (operands[1]),
 311                                                   operands[2], operands[3]),
 312                                   operands[2], operands[3]);
 313   }
 314 )
 315
 316 ;; Expansion of signed mod by a power of 2 using CSNEG.
 317 ;; For x0 % n where n is a power of 2 produce:
 318 ;; negs   x1, x0
 319 ;; and    x0, x0, #(n - 1)
 320 ;; and    x1, x1, #(n - 1)
 321 ;; csneg  x0, x0, x1, mi
 322
 323 (define_expand "mod<mode>3"
 324   [(match_operand:GPI 0 "register_operand" "")
 325    (match_operand:GPI 1 "register_operand" "")
 326    (match_operand:GPI 2 "const_int_operand" "")]
 327   ""
 328   {
 329     HOST_WIDE_INT val = INTVAL (operands[2]);
 330
 331     if (val <= 0
 332        || exact_log2 (val) <= 0
 333        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 334       FAIL;
 335
 336     rtx mask = GEN_INT (val - 1);
 337
 338     /* In the special case of x0 % 2 we can do the even shorter:
 339         cmp    x0, xzr
 340         and    x0, x0, 1
 341         cneg   x0, x0, lt.  */
 342     if (val == 2)
 343       {
 344         rtx masked = gen_reg_rtx (<MODE>mode);
 345         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 346         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 347         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 348         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 349         DONE;
 350       }
 351
 352     rtx neg_op = gen_reg_rtx (<MODE>mode);
 353     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 354
 355     /* Extract the condition register and mode.  */
 356     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 357     rtx cc_reg = SET_DEST (cmp);
 358     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 359
 360     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 361     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 362
 363     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 364     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 365
 366     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 367                                        masked_neg, masked_pos));
 368     DONE;
 369   }
 370 )
 371
 372 (define_insn "*condjump"
 373   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 374                             [(match_operand 1 "cc_register" "") (const_int 0)])
 375                            (label_ref (match_operand 2 "" ""))
 376                            (pc)))]
 377   ""
 378   {
 379     if (get_attr_length (insn) == 8)
 380       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 381     else
 382       return  "b%m0\\t%l2";
 383   }
 384   [(set_attr "type" "branch")
 385    (set (attr "length")
 386         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 387                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 388                       (const_int 4)
 389                       (const_int 8)))
 390    (set (attr "far_branch")
 391         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 392                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 393                       (const_int 0)
 394                       (const_int 1)))]
 395 )
 396
 397 (define_expand "casesi"
 398   [(match_operand:SI 0 "register_operand" "")   ; Index
 399    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 400    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 401    (match_operand:DI 3 "" "")                   ; Table label
 402    (match_operand:DI 4 "" "")]                  ; Out of range label
 403   ""
 404   {
 405     if (operands[1] != const0_rtx)
 406       {
 407         rtx reg = gen_reg_rtx (SImode);
 408
 409         /* Canonical RTL says that if you have:
 410
 411            (minus (X) (CONST))
 412
 413            then this should be emitted as:
 414
 415            (plus (X) (-CONST))
 416
 417            The use of trunc_int_for_mode ensures that the resulting
 418            constant can be represented in SImode, this is important
 419            for the corner case where operand[1] is INT_MIN.  */
 420
 421         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 422
 423         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 424               (operands[1], SImode))
 425           operands[1] = force_reg (SImode, operands[1]);
 426         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 427         operands[0] = reg;
 428       }
 429
 430     if (!aarch64_plus_operand (operands[2], SImode))
 431       operands[2] = force_reg (SImode, operands[2]);
 432     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 433                                                  const0_rtx),
 434                                     operands[0], operands[2], operands[4]));
 435
 436     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (VOIDmode, operands[3]));
 437     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 438                                          operands[3]));
 439     DONE;
 440   }
 441 )
 442
 443 (define_insn "casesi_dispatch"
 444   [(parallel
 445     [(set (pc)
 446           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 447                            (match_operand:SI 1 "register_operand" "r")]
 448                         UNSPEC_CASESI)))
 449      (clobber (reg:CC CC_REGNUM))
 450      (clobber (match_scratch:DI 3 "=r"))
 451      (clobber (match_scratch:DI 4 "=r"))
 452      (use (label_ref (match_operand 2 "" "")))])]
 453   ""
 454   "*
 455   return aarch64_output_casesi (operands);
 456   "
 457   [(set_attr "length" "16")
 458    (set_attr "type" "branch")]
 459 )
 460
 461 (define_insn "nop"
 462   [(unspec[(const_int 0)] UNSPEC_NOP)]
 463   ""
 464   "nop"
 465   [(set_attr "type" "no_insn")]
 466 )
 467
 468 (define_insn "prefetch"
 469   [(prefetch (match_operand:DI 0 "register_operand" "r")
 470             (match_operand:QI 1 "const_int_operand" "")
 471             (match_operand:QI 2 "const_int_operand" ""))]
 472   ""
 473   {
 474     const char * pftype[2][4] =
 475     {
 476       {"prfm\\tPLDL1STRM, %a0",
 477        "prfm\\tPLDL3KEEP, %a0",
 478        "prfm\\tPLDL2KEEP, %a0",
 479        "prfm\\tPLDL1KEEP, %a0"},
 480       {"prfm\\tPSTL1STRM, %a0",
 481        "prfm\\tPSTL3KEEP, %a0",
 482        "prfm\\tPSTL2KEEP, %a0",
 483        "prfm\\tPSTL1KEEP, %a0"},
 484     };
 485
 486     int locality = INTVAL (operands[2]);
 487
 488     gcc_assert (IN_RANGE (locality, 0, 3));
 489
 490     return pftype[INTVAL(operands[1])][locality];
 491   }
 492   [(set_attr "type" "load1")]
 493 )
 494
 495 (define_insn "trap"
 496   [(trap_if (const_int 1) (const_int 8))]
 497   ""
 498   "brk #1000"
 499   [(set_attr "type" "trap")])
 500
 501 (define_expand "prologue"
 502   [(clobber (const_int 0))]
 503   ""
 504   "
 505   aarch64_expand_prologue ();
 506   DONE;
 507   "
 508 )
 509
 510 (define_expand "epilogue"
 511   [(clobber (const_int 0))]
 512   ""
 513   "
 514   aarch64_expand_epilogue (false);
 515   DONE;
 516   "
 517 )
 518
 519 (define_expand "sibcall_epilogue"
 520   [(clobber (const_int 0))]
 521   ""
 522   "
 523   aarch64_expand_epilogue (true);
 524   DONE;
 525   "
 526 )
 527
 528 (define_insn "*do_return"
 529   [(return)]
 530   ""
 531   "ret"
 532   [(set_attr "type" "branch")]
 533 )
 534
 535 (define_expand "return"
 536   [(simple_return)]
 537   "aarch64_use_return_insn_p ()"
 538   ""
 539 )
 540
 541 (define_insn "simple_return"
 542   [(simple_return)]
 543   ""
 544   "ret"
 545   [(set_attr "type" "branch")]
 546 )
 547
 548 (define_insn "eh_return"
 549   [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")]
 550     UNSPECV_EH_RETURN)]
 551   ""
 552   "#"
 553   [(set_attr "type" "branch")]
 554
 555 )
 556
 557 (define_split
 558   [(unspec_volatile [(match_operand:DI 0 "register_operand" "")]
 559     UNSPECV_EH_RETURN)]
 560   "reload_completed"
 561   [(set (match_dup 1) (match_dup 0))]
 562   {
 563     operands[1] = aarch64_final_eh_return_addr ();
 564   }
 565 )
 566
 567 (define_insn "*cb<optab><mode>1"
 568   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 569                                 (const_int 0))
 570                            (label_ref (match_operand 1 "" ""))
 571                            (pc)))]
 572   ""
 573   {
 574     if (get_attr_length (insn) == 8)
 575       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 576     else
 577       return "<cbz>\\t%<w>0, %l1";
 578   }
 579   [(set_attr "type" "branch")
 580    (set (attr "length")
 581         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 582                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 583                       (const_int 4)
 584                       (const_int 8)))
 585    (set (attr "far_branch")
 586         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 587                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 588                       (const_int 0)
 589                       (const_int 1)))]
 590 )
 591
 592 (define_insn "*tb<optab><mode>1"
 593   [(set (pc) (if_then_else
 594               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 595                                     (const_int 1)
 596                                     (match_operand 1 "const_int_operand" "n"))
 597                    (const_int 0))
 598              (label_ref (match_operand 2 "" ""))
 599              (pc)))
 600    (clobber (reg:CC CC_REGNUM))]
 601   ""
 602   {
 603     if (get_attr_length (insn) == 8)
 604       {
 605         if (get_attr_far_branch (insn) == 1)
 606           return aarch64_gen_far_branch (operands, 2, "Ltb",
 607                                          "<inv_tb>\\t%<w>0, %1, ");
 608         else
 609           {
 610             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 611             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 612           }
 613       }
 614     else
 615       return "<tbz>\t%<w>0, %1, %l2";
 616   }
 617   [(set_attr "type" "branch")
 618    (set (attr "length")
 619         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 620                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 621                       (const_int 4)
 622                       (const_int 8)))
 623    (set (attr "far_branch")
 624         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 625                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 626                       (const_int 0)
 627                       (const_int 1)))]
 628
 629 )
 630
 631 (define_insn "*cb<optab><mode>1"
 632   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 633                                  (const_int 0))
 634                            (label_ref (match_operand 1 "" ""))
 635                            (pc)))
 636    (clobber (reg:CC CC_REGNUM))]
 637   ""
 638   {
 639     if (get_attr_length (insn) == 8)
 640       {
 641         if (get_attr_far_branch (insn) == 1)
 642           return aarch64_gen_far_branch (operands, 1, "Ltb",
 643                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 644         else
 645           {
 646             char buf[64];
 647             uint64_t val = ((uint64_t) 1)
 648                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 649             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 650             output_asm_insn (buf, operands);
 651             return "<bcond>\t%l1";
 652           }
 653       }
 654     else
 655       return "<tbz>\t%<w>0, <sizem1>, %l1";
 656   }
 657   [(set_attr "type" "branch")
 658    (set (attr "length")
 659         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 660                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 661                       (const_int 4)
 662                       (const_int 8)))
 663    (set (attr "far_branch")
 664         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 665                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 666                       (const_int 0)
 667                       (const_int 1)))]
 668 )
 669
 670 ;; -------------------------------------------------------------------
 671 ;; Subroutine calls and sibcalls
 672 ;; -------------------------------------------------------------------
 673
 674 (define_expand "call_internal"
 675   [(parallel [(call (match_operand 0 "memory_operand" "")
 676                     (match_operand 1 "general_operand" ""))
 677               (use (match_operand 2 "" ""))
 678               (clobber (reg:DI LR_REGNUM))])])
 679
 680 (define_expand "call"
 681   [(parallel [(call (match_operand 0 "memory_operand" "")
 682                     (match_operand 1 "general_operand" ""))
 683               (use (match_operand 2 "" ""))
 684               (clobber (reg:DI LR_REGNUM))])]
 685   ""
 686   "
 687   {
 688     rtx callee, pat;
 689
 690     /* In an untyped call, we can get NULL for operand 2.  */
 691     if (operands[2] == NULL)
 692       operands[2] = const0_rtx;
 693
 694     /* Decide if we should generate indirect calls by loading the
 695        64-bit address of the callee into a register before performing
 696        the branch-and-link.  */
 697     callee = XEXP (operands[0], 0);
 698     if (GET_CODE (callee) == SYMBOL_REF
 699         ? (aarch64_is_long_call_p (callee)
 700            || aarch64_is_noplt_call_p (callee))
 701         : !REG_P (callee))
 702       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 703
 704     pat = gen_call_internal (operands[0], operands[1], operands[2]);
 705     aarch64_emit_call_insn (pat);
 706     DONE;
 707   }"
 708 )
 709
 710 (define_insn "*call_reg"
 711   [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
 712          (match_operand 1 "" ""))
 713    (use (match_operand 2 "" ""))
 714    (clobber (reg:DI LR_REGNUM))]
 715   ""
 716   "blr\\t%0"
 717   [(set_attr "type" "call")]
 718 )
 719
 720 (define_insn "*call_symbol"
 721   [(call (mem:DI (match_operand:DI 0 "" ""))
 722          (match_operand 1 "" ""))
 723    (use (match_operand 2 "" ""))
 724    (clobber (reg:DI LR_REGNUM))]
 725   "GET_CODE (operands[0]) == SYMBOL_REF
 726    && !aarch64_is_long_call_p (operands[0])
 727    && !aarch64_is_noplt_call_p (operands[0])"
 728   "bl\\t%a0"
 729   [(set_attr "type" "call")]
 730 )
 731
 732 (define_expand "call_value_internal"
 733   [(parallel [(set (match_operand 0 "" "")
 734                    (call (match_operand 1 "memory_operand" "")
 735                          (match_operand 2 "general_operand" "")))
 736               (use (match_operand 3 "" ""))
 737               (clobber (reg:DI LR_REGNUM))])])
 738
 739 (define_expand "call_value"
 740   [(parallel [(set (match_operand 0 "" "")
 741                    (call (match_operand 1 "memory_operand" "")
 742                          (match_operand 2 "general_operand" "")))
 743               (use (match_operand 3 "" ""))
 744               (clobber (reg:DI LR_REGNUM))])]
 745   ""
 746   "
 747   {
 748     rtx callee, pat;
 749
 750     /* In an untyped call, we can get NULL for operand 3.  */
 751     if (operands[3] == NULL)
 752       operands[3] = const0_rtx;
 753
 754     /* Decide if we should generate indirect calls by loading the
 755        64-bit address of the callee into a register before performing
 756        the branch-and-link.  */
 757     callee = XEXP (operands[1], 0);
 758     if (GET_CODE (callee) == SYMBOL_REF
 759         ? (aarch64_is_long_call_p (callee)
 760            || aarch64_is_noplt_call_p (callee))
 761         : !REG_P (callee))
 762       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 763
 764     pat = gen_call_value_internal (operands[0], operands[1], operands[2],
 765                                    operands[3]);
 766     aarch64_emit_call_insn (pat);
 767     DONE;
 768   }"
 769 )
 770
 771 (define_insn "*call_value_reg"
 772   [(set (match_operand 0 "" "")
 773         (call (mem:DI (match_operand:DI 1 "register_operand" "r"))
 774                       (match_operand 2 "" "")))
 775    (use (match_operand 3 "" ""))
 776    (clobber (reg:DI LR_REGNUM))]
 777   ""
 778   "blr\\t%1"
 779   [(set_attr "type" "call")]
 780
 781 )
 782
 783 (define_insn "*call_value_symbol"
 784   [(set (match_operand 0 "" "")
 785         (call (mem:DI (match_operand:DI 1 "" ""))
 786               (match_operand 2 "" "")))
 787    (use (match_operand 3 "" ""))
 788    (clobber (reg:DI LR_REGNUM))]
 789   "GET_CODE (operands[1]) == SYMBOL_REF
 790    && !aarch64_is_long_call_p (operands[1])
 791    && !aarch64_is_noplt_call_p (operands[1])"
 792   "bl\\t%a1"
 793   [(set_attr "type" "call")]
 794 )
 795
 796 (define_expand "sibcall_internal"
 797   [(parallel [(call (match_operand 0 "memory_operand" "")
 798                     (match_operand 1 "general_operand" ""))
 799               (return)
 800               (use (match_operand 2 "" ""))])])
 801
 802 (define_expand "sibcall"
 803   [(parallel [(call (match_operand 0 "memory_operand" "")
 804                     (match_operand 1 "general_operand" ""))
 805               (return)
 806               (use (match_operand 2 "" ""))])]
 807   ""
 808   {
 809     rtx pat;
 810     rtx callee = XEXP (operands[0], 0);
 811     if (!REG_P (callee)
 812        && ((GET_CODE (callee) != SYMBOL_REF)
 813            || aarch64_is_noplt_call_p (callee)))
 814       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 815
 816     if (operands[2] == NULL_RTX)
 817       operands[2] = const0_rtx;
 818
 819     pat = gen_sibcall_internal (operands[0], operands[1], operands[2]);
 820     aarch64_emit_call_insn (pat);
 821     DONE;
 822   }
 823 )
 824
 825 (define_expand "sibcall_value_internal"
 826   [(parallel [(set (match_operand 0 "" "")
 827                    (call (match_operand 1 "memory_operand" "")
 828                          (match_operand 2 "general_operand" "")))
 829               (return)
 830               (use (match_operand 3 "" ""))])])
 831
 832 (define_expand "sibcall_value"
 833   [(parallel [(set (match_operand 0 "" "")
 834                    (call (match_operand 1 "memory_operand" "")
 835                          (match_operand 2 "general_operand" "")))
 836               (return)
 837               (use (match_operand 3 "" ""))])]
 838   ""
 839   {
 840     rtx pat;
 841     rtx callee = XEXP (operands[1], 0);
 842     if (!REG_P (callee)
 843        && ((GET_CODE (callee) != SYMBOL_REF)
 844            || aarch64_is_noplt_call_p (callee)))
 845       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 846
 847     if (operands[3] == NULL_RTX)
 848       operands[3] = const0_rtx;
 849
 850     pat = gen_sibcall_value_internal (operands[0], operands[1], operands[2],
 851                                       operands[3]);
 852     aarch64_emit_call_insn (pat);
 853     DONE;
 854   }
 855 )
 856
 857 (define_insn "*sibcall_insn"
 858   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 859          (match_operand 1 "" ""))
 860    (return)
 861    (use (match_operand 2 "" ""))]
 862   "SIBLING_CALL_P (insn)"
 863   "@
 864    br\\t%0
 865    b\\t%a0"
 866   [(set_attr "type" "branch, branch")]
 867 )
 868
 869 (define_insn "*sibcall_value_insn"
 870   [(set (match_operand 0 "" "")
 871         (call (mem:DI
 872                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 873               (match_operand 2 "" "")))
 874    (return)
 875    (use (match_operand 3 "" ""))]
 876   "SIBLING_CALL_P (insn)"
 877   "@
 878    br\\t%1
 879    b\\t%a1"
 880   [(set_attr "type" "branch, branch")]
 881 )
 882
 883 ;; Call subroutine returning any type.
 884
 885 (define_expand "untyped_call"
 886   [(parallel [(call (match_operand 0 "")
 887                     (const_int 0))
 888               (match_operand 1 "")
 889               (match_operand 2 "")])]
 890   ""
 891 {
 892   int i;
 893
 894   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 895
 896   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 897     {
 898       rtx set = XVECEXP (operands[2], 0, i);
 899       emit_move_insn (SET_DEST (set), SET_SRC (set));
 900     }
 901
 902   /* The optimizer does not know that the call sets the function value
 903      registers we stored in the result block.  We avoid problems by
 904      claiming that all hard registers are used and clobbered at this
 905      point.  */
 906   emit_insn (gen_blockage ());
 907   DONE;
 908 })
 909
 910 ;; -------------------------------------------------------------------
 911 ;; Moves
 912 ;; -------------------------------------------------------------------
 913
 914 (define_expand "mov<mode>"
 915   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 916         (match_operand:SHORT 1 "general_operand" ""))]
 917   ""
 918   "
 919     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 920       operands[1] = force_reg (<MODE>mode, operands[1]);
 921   "
 922 )
 923
 924 (define_insn "*mov<mode>_aarch64"
 925   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,   *w,r,*w, m, m, r,*w,*w")
 926         (match_operand:SHORT 1 "general_operand"      " r,M,D<hq>,m, m,rZ,*w,*w, r,*w"))]
 927   "(register_operand (operands[0], <MODE>mode)
 928     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 929 {
 930    switch (which_alternative)
 931      {
 932      case 0:
 933        return "mov\t%w0, %w1";
 934      case 1:
 935        return "mov\t%w0, %1";
 936      case 2:
 937        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 938                                                         <MODE>mode);
 939      case 3:
 940        return "ldr<size>\t%w0, %1";
 941      case 4:
 942        return "ldr\t%<size>0, %1";
 943      case 5:
 944        return "str<size>\t%w1, %0";
 945      case 6:
 946        return "str\t%<size>1, %0";
 947      case 7:
 948        return "umov\t%w0, %1.<v>[0]";
 949      case 8:
 950        return "dup\t%0.<Vallxd>, %w1";
 951      case 9:
 952        return "dup\t%<Vetype>0, %1.<v>[0]";
 953      default:
 954        gcc_unreachable ();
 955      }
 956 }
 957   [(set_attr "type" "mov_reg,mov_imm,neon_move,load1,load1,store1,store1,\
 958                      neon_to_gp<q>,neon_from_gp<q>,neon_dup")
 959    (set_attr "simd" "*,*,yes,*,*,*,*,yes,yes,yes")]
 960 )
 961
 962 (define_expand "mov<mode>"
 963   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
 964         (match_operand:GPI 1 "general_operand" ""))]
 965   ""
 966   "
 967     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 968       operands[1] = force_reg (<MODE>mode, operands[1]);
 969
 970     /* FIXME: RR we still need to fix up what we are doing with
 971        symbol_refs and other types of constants.  */
 972     if (CONSTANT_P (operands[1])
 973         && !CONST_INT_P (operands[1]))
 974      {
 975        aarch64_expand_mov_immediate (operands[0], operands[1]);
 976        DONE;
 977      }
 978   "
 979 )
 980
 981 (define_insn_and_split "*movsi_aarch64"
 982   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r  ,*w, r,*w")
 983         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,m, m,rZ,*w,S,Ush,rZ,*w,*w"))]
 984   "(register_operand (operands[0], SImode)
 985     || aarch64_reg_or_zero (operands[1], SImode))"
 986   "@
 987    mov\\t%w0, %w1
 988    mov\\t%w0, %w1
 989    mov\\t%w0, %w1
 990    mov\\t%w0, %1
 991    #
 992    ldr\\t%w0, %1
 993    ldr\\t%s0, %1
 994    str\\t%w1, %0
 995    str\\t%s1, %0
 996    adr\\t%x0, %a1
 997    adrp\\t%x0, %A1
 998    fmov\\t%s0, %w1
 999    fmov\\t%w0, %s1
1000    fmov\\t%s0, %s1"
1001    "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1002     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1003    [(const_int 0)]
1004    "{
1005        aarch64_expand_mov_immediate (operands[0], operands[1]);
1006        DONE;
1007     }"
1008   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1009                      adr,adr,f_mcr,f_mrc,fmov")
1010    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes")]
1011 )
1012
1013 (define_insn_and_split "*movdi_aarch64"
1014   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r,  *w, r,*w,w")
1015         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,n,m, m,rZ,*w,S,Ush,rZ,*w,*w,Dd"))]
1016   "(register_operand (operands[0], DImode)
1017     || aarch64_reg_or_zero (operands[1], DImode))"
1018   "@
1019    mov\\t%x0, %x1
1020    mov\\t%0, %x1
1021    mov\\t%x0, %1
1022    mov\\t%x0, %1
1023    #
1024    ldr\\t%x0, %1
1025    ldr\\t%d0, %1
1026    str\\t%x1, %0
1027    str\\t%d1, %0
1028    adr\\t%x0, %a1
1029    adrp\\t%x0, %A1
1030    fmov\\t%d0, %x1
1031    fmov\\t%x0, %d1
1032    fmov\\t%d0, %d1
1033    movi\\t%d0, %1"
1034    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1035     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1036    [(const_int 0)]
1037    "{
1038        aarch64_expand_mov_immediate (operands[0], operands[1]);
1039        DONE;
1040     }"
1041   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
1042                      adr,adr,f_mcr,f_mrc,fmov,neon_move")
1043    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1044    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")]
1045 )
1046
1047 (define_insn "insv_imm<mode>"
1048   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1049                           (const_int 16)
1050                           (match_operand:GPI 1 "const_int_operand" "n"))
1051         (match_operand:GPI 2 "const_int_operand" "n"))]
1052   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1053    && UINTVAL (operands[1]) % 16 == 0"
1054   "movk\\t%<w>0, %X2, lsl %1"
1055   [(set_attr "type" "mov_imm")]
1056 )
1057
1058 (define_expand "movti"
1059   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1060         (match_operand:TI 1 "general_operand" ""))]
1061   ""
1062   "
1063     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1064       operands[1] = force_reg (TImode, operands[1]);
1065   "
1066 )
1067
1068 (define_insn "*movti_aarch64"
1069   [(set (match_operand:TI 0
1070          "nonimmediate_operand"  "=r, *w,r ,*w,r  ,Ump,Ump,*w,m")
1071         (match_operand:TI 1
1072          "aarch64_movti_operand" " rn,r ,*w,*w,Ump,r  ,Z  , m,*w"))]
1073   "(register_operand (operands[0], TImode)
1074     || aarch64_reg_or_zero (operands[1], TImode))"
1075   "@
1076    #
1077    #
1078    #
1079    orr\\t%0.16b, %1.16b, %1.16b
1080    ldp\\t%0, %H0, %1
1081    stp\\t%1, %H1, %0
1082    stp\\txzr, xzr, %0
1083    ldr\\t%q0, %1
1084    str\\t%q1, %0"
1085   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1086                              load2,store2,store2,f_loadd,f_stored")
1087    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1088    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1089    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1090 )
1091
1092 ;; Split a TImode register-register or register-immediate move into
1093 ;; its component DImode pieces, taking care to handle overlapping
1094 ;; source and dest registers.
1095 (define_split
1096    [(set (match_operand:TI 0 "register_operand" "")
1097          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1098   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1099   [(const_int 0)]
1100 {
1101   aarch64_split_128bit_move (operands[0], operands[1]);
1102   DONE;
1103 })
1104
1105 (define_expand "mov<mode>"
1106   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1107         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1108   ""
1109   {
1110     if (!TARGET_FLOAT)
1111       {
1112         aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1113         FAIL;
1114       }
1115
1116     if (GET_CODE (operands[0]) == MEM
1117         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1118               && aarch64_float_const_zero_rtx_p (operands[1])))
1119       operands[1] = force_reg (<MODE>mode, operands[1]);
1120   }
1121 )
1122
1123 (define_insn "*movhf_aarch64"
1124   [(set (match_operand:HF 0 "nonimmediate_operand" "=w, ?r,w,w,m,r,m ,r")
1125         (match_operand:HF 1 "general_operand"      "?rY, w,w,m,w,m,rY,r"))]
1126   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1127     || register_operand (operands[1], HFmode))"
1128   "@
1129    mov\\t%0.h[0], %w1
1130    umov\\t%w0, %1.h[0]
1131    mov\\t%0.h[0], %1.h[0]
1132    ldr\\t%h0, %1
1133    str\\t%h1, %0
1134    ldrh\\t%w0, %1
1135    strh\\t%w1, %0
1136    mov\\t%w0, %w1"
1137   [(set_attr "type" "neon_from_gp,neon_to_gp,fmov,\
1138                      f_loads,f_stores,load1,store1,mov_reg")
1139    (set_attr "simd" "yes,yes,yes,*,*,*,*,*")
1140    (set_attr "fp"   "*,*,*,yes,yes,*,*,*")]
1141 )
1142
1143 (define_insn "*movsf_aarch64"
1144   [(set (match_operand:SF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
1145         (match_operand:SF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
1146   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1147     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1148   "@
1149    fmov\\t%s0, %w1
1150    fmov\\t%w0, %s1
1151    fmov\\t%s0, %s1
1152    fmov\\t%s0, %1
1153    ldr\\t%s0, %1
1154    str\\t%s1, %0
1155    ldr\\t%w0, %1
1156    str\\t%w1, %0
1157    mov\\t%w0, %w1"
1158   [(set_attr "type" "f_mcr,f_mrc,fmov,fconsts,\
1159                      f_loads,f_stores,load1,store1,mov_reg")]
1160 )
1161
1162 (define_insn "*movdf_aarch64"
1163   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
1164         (match_operand:DF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
1165   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1166     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1167   "@
1168    fmov\\t%d0, %x1
1169    fmov\\t%x0, %d1
1170    fmov\\t%d0, %d1
1171    fmov\\t%d0, %1
1172    ldr\\t%d0, %1
1173    str\\t%d1, %0
1174    ldr\\t%x0, %1
1175    str\\t%x1, %0
1176    mov\\t%x0, %x1"
1177   [(set_attr "type" "f_mcr,f_mrc,fmov,fconstd,\
1178                      f_loadd,f_stored,load1,store1,mov_reg")]
1179 )
1180
1181 (define_insn "*movtf_aarch64"
1182   [(set (match_operand:TF 0
1183          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r ,Ump,Ump")
1184         (match_operand:TF 1
1185          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,Ump,?r ,Y"))]
1186   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1187     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1188   "@
1189    orr\\t%0.16b, %1.16b, %1.16b
1190    #
1191    #
1192    #
1193    movi\\t%0.2d, #0
1194    fmov\\t%s0, wzr
1195    ldr\\t%q0, %1
1196    str\\t%q1, %0
1197    ldp\\t%0, %H0, %1
1198    stp\\t%1, %H1, %0
1199    stp\\txzr, xzr, %0"
1200   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,fconstd,\
1201                      f_loadd,f_stored,load2,store2,store2")
1202    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1203    (set_attr "fp" "*,*,yes,yes,*,yes,yes,yes,*,*,*")
1204    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1205 )
1206
1207 (define_split
1208    [(set (match_operand:TF 0 "register_operand" "")
1209          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1210   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1211   [(const_int 0)]
1212   {
1213     aarch64_split_128bit_move (operands[0], operands[1]);
1214     DONE;
1215   }
1216 )
1217
1218 ;; 0 is dst
1219 ;; 1 is src
1220 ;; 2 is size of move in bytes
1221 ;; 3 is alignment
1222
1223 (define_expand "movmemdi"
1224   [(match_operand:BLK 0 "memory_operand")
1225    (match_operand:BLK 1 "memory_operand")
1226    (match_operand:DI 2 "immediate_operand")
1227    (match_operand:DI 3 "immediate_operand")]
1228    "!STRICT_ALIGNMENT"
1229 {
1230   if (aarch64_expand_movmem (operands))
1231     DONE;
1232   FAIL;
1233 }
1234 )
1235
1236 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1237 ;; fairly lax checking on the second memory operation.
1238 (define_insn "load_pairsi"
1239   [(set (match_operand:SI 0 "register_operand" "=r,*w")
1240         (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1241    (set (match_operand:SI 2 "register_operand" "=r,*w")
1242         (match_operand:SI 3 "memory_operand" "m,m"))]
1243   "rtx_equal_p (XEXP (operands[3], 0),
1244                 plus_constant (Pmode,
1245                                XEXP (operands[1], 0),
1246                                GET_MODE_SIZE (SImode)))"
1247   "@
1248    ldp\\t%w0, %w2, %1
1249    ldp\\t%s0, %s2, %1"
1250   [(set_attr "type" "load2,neon_load1_2reg")
1251    (set_attr "fp" "*,yes")]
1252 )
1253
1254 (define_insn "load_pairdi"
1255   [(set (match_operand:DI 0 "register_operand" "=r,*w")
1256         (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1257    (set (match_operand:DI 2 "register_operand" "=r,*w")
1258         (match_operand:DI 3 "memory_operand" "m,m"))]
1259   "rtx_equal_p (XEXP (operands[3], 0),
1260                 plus_constant (Pmode,
1261                                XEXP (operands[1], 0),
1262                                GET_MODE_SIZE (DImode)))"
1263   "@
1264    ldp\\t%x0, %x2, %1
1265    ldp\\t%d0, %d2, %1"
1266   [(set_attr "type" "load2,neon_load1_2reg")
1267    (set_attr "fp" "*,yes")]
1268 )
1269
1270
1271 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1272 ;; fairly lax checking on the second memory operation.
1273 (define_insn "store_pairsi"
1274   [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1275         (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1276    (set (match_operand:SI 2 "memory_operand" "=m,m")
1277         (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1278   "rtx_equal_p (XEXP (operands[2], 0),
1279                 plus_constant (Pmode,
1280                                XEXP (operands[0], 0),
1281                                GET_MODE_SIZE (SImode)))"
1282   "@
1283    stp\\t%w1, %w3, %0
1284    stp\\t%s1, %s3, %0"
1285   [(set_attr "type" "store2,neon_store1_2reg")
1286    (set_attr "fp" "*,yes")]
1287 )
1288
1289 (define_insn "store_pairdi"
1290   [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1291         (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1292    (set (match_operand:DI 2 "memory_operand" "=m,m")
1293         (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1294   "rtx_equal_p (XEXP (operands[2], 0),
1295                 plus_constant (Pmode,
1296                                XEXP (operands[0], 0),
1297                                GET_MODE_SIZE (DImode)))"
1298   "@
1299    stp\\t%x1, %x3, %0
1300    stp\\t%d1, %d3, %0"
1301   [(set_attr "type" "store2,neon_store1_2reg")
1302    (set_attr "fp" "*,yes")]
1303 )
1304
1305 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1306 ;; fairly lax checking on the second memory operation.
1307 (define_insn "load_pairsf"
1308   [(set (match_operand:SF 0 "register_operand" "=w,*r")
1309         (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1310    (set (match_operand:SF 2 "register_operand" "=w,*r")
1311         (match_operand:SF 3 "memory_operand" "m,m"))]
1312   "rtx_equal_p (XEXP (operands[3], 0),
1313                 plus_constant (Pmode,
1314                                XEXP (operands[1], 0),
1315                                GET_MODE_SIZE (SFmode)))"
1316   "@
1317    ldp\\t%s0, %s2, %1
1318    ldp\\t%w0, %w2, %1"
1319   [(set_attr "type" "neon_load1_2reg,load2")
1320    (set_attr "fp" "yes,*")]
1321 )
1322
1323 (define_insn "load_pairdf"
1324   [(set (match_operand:DF 0 "register_operand" "=w,*r")
1325         (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1326    (set (match_operand:DF 2 "register_operand" "=w,*r")
1327         (match_operand:DF 3 "memory_operand" "m,m"))]
1328   "rtx_equal_p (XEXP (operands[3], 0),
1329                 plus_constant (Pmode,
1330                                XEXP (operands[1], 0),
1331                                GET_MODE_SIZE (DFmode)))"
1332   "@
1333    ldp\\t%d0, %d2, %1
1334    ldp\\t%x0, %x2, %1"
1335   [(set_attr "type" "neon_load1_2reg,load2")
1336    (set_attr "fp" "yes,*")]
1337 )
1338
1339 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1340 ;; fairly lax checking on the second memory operation.
1341 (define_insn "store_pairsf"
1342   [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1343         (match_operand:SF 1 "register_operand" "w,*r"))
1344    (set (match_operand:SF 2 "memory_operand" "=m,m")
1345         (match_operand:SF 3 "register_operand" "w,*r"))]
1346   "rtx_equal_p (XEXP (operands[2], 0),
1347                 plus_constant (Pmode,
1348                                XEXP (operands[0], 0),
1349                                GET_MODE_SIZE (SFmode)))"
1350   "@
1351    stp\\t%s1, %s3, %0
1352    stp\\t%w1, %w3, %0"
1353   [(set_attr "type" "neon_store1_2reg,store2")
1354    (set_attr "fp" "yes,*")]
1355 )
1356
1357 (define_insn "store_pairdf"
1358   [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1359         (match_operand:DF 1 "register_operand" "w,*r"))
1360    (set (match_operand:DF 2 "memory_operand" "=m,m")
1361         (match_operand:DF 3 "register_operand" "w,*r"))]
1362   "rtx_equal_p (XEXP (operands[2], 0),
1363                 plus_constant (Pmode,
1364                                XEXP (operands[0], 0),
1365                                GET_MODE_SIZE (DFmode)))"
1366   "@
1367    stp\\t%d1, %d3, %0
1368    stp\\t%x1, %x3, %0"
1369   [(set_attr "type" "neon_store1_2reg,store2")
1370    (set_attr "fp" "yes,*")]
1371 )
1372
1373 ;; Load pair with post-index writeback.  This is primarily used in function
1374 ;; epilogues.
1375 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1376   [(parallel
1377     [(set (match_operand:P 0 "register_operand" "=k")
1378           (plus:P (match_operand:P 1 "register_operand" "0")
1379                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1380      (set (match_operand:GPI 2 "register_operand" "=r")
1381           (mem:GPI (match_dup 1)))
1382      (set (match_operand:GPI 3 "register_operand" "=r")
1383           (mem:GPI (plus:P (match_dup 1)
1384                    (match_operand:P 5 "const_int_operand" "n"))))])]
1385   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1386   "ldp\\t%<w>2, %<w>3, [%1], %4"
1387   [(set_attr "type" "load2")]
1388 )
1389
1390 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1391   [(parallel
1392     [(set (match_operand:P 0 "register_operand" "=k")
1393           (plus:P (match_operand:P 1 "register_operand" "0")
1394                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1395      (set (match_operand:GPF 2 "register_operand" "=w")
1396           (mem:GPF (match_dup 1)))
1397      (set (match_operand:GPF 3 "register_operand" "=w")
1398           (mem:GPF (plus:P (match_dup 1)
1399                    (match_operand:P 5 "const_int_operand" "n"))))])]
1400   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1401   "ldp\\t%<w>2, %<w>3, [%1], %4"
1402   [(set_attr "type" "neon_load1_2reg")]
1403 )
1404
1405 ;; Store pair with pre-index writeback.  This is primarily used in function
1406 ;; prologues.
1407 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1408   [(parallel
1409     [(set (match_operand:P 0 "register_operand" "=&k")
1410           (plus:P (match_operand:P 1 "register_operand" "0")
1411                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1412      (set (mem:GPI (plus:P (match_dup 0)
1413                    (match_dup 4)))
1414           (match_operand:GPI 2 "register_operand" "r"))
1415      (set (mem:GPI (plus:P (match_dup 0)
1416                    (match_operand:P 5 "const_int_operand" "n")))
1417           (match_operand:GPI 3 "register_operand" "r"))])]
1418   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1419   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1420   [(set_attr "type" "store2")]
1421 )
1422
1423 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1424   [(parallel
1425     [(set (match_operand:P 0 "register_operand" "=&k")
1426           (plus:P (match_operand:P 1 "register_operand" "0")
1427                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1428      (set (mem:GPF (plus:P (match_dup 0)
1429                    (match_dup 4)))
1430           (match_operand:GPF 2 "register_operand" "w"))
1431      (set (mem:GPF (plus:P (match_dup 0)
1432                    (match_operand:P 5 "const_int_operand" "n")))
1433           (match_operand:GPF 3 "register_operand" "w"))])]
1434   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1435   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1436   [(set_attr "type" "neon_store1_2reg<q>")]
1437 )
1438
1439 ;; -------------------------------------------------------------------
1440 ;; Sign/Zero extension
1441 ;; -------------------------------------------------------------------
1442
1443 (define_expand "<optab>sidi2"
1444   [(set (match_operand:DI 0 "register_operand")
1445         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1446   ""
1447 )
1448
1449 (define_insn "*extendsidi2_aarch64"
1450   [(set (match_operand:DI 0 "register_operand" "=r,r")
1451         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1452   ""
1453   "@
1454    sxtw\t%0, %w1
1455    ldrsw\t%0, %1"
1456   [(set_attr "type" "extend,load1")]
1457 )
1458
1459 (define_insn "*load_pair_extendsidi2_aarch64"
1460   [(set (match_operand:DI 0 "register_operand" "=r")
1461         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1462    (set (match_operand:DI 2 "register_operand" "=r")
1463         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1464   "rtx_equal_p (XEXP (operands[3], 0),
1465                 plus_constant (Pmode,
1466                                XEXP (operands[1], 0),
1467                                GET_MODE_SIZE (SImode)))"
1468   "ldpsw\\t%0, %2, %1"
1469   [(set_attr "type" "load2")]
1470 )
1471
1472 (define_insn "*zero_extendsidi2_aarch64"
1473   [(set (match_operand:DI 0 "register_operand" "=r,r")
1474         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1475   ""
1476   "@
1477    uxtw\t%0, %w1
1478    ldr\t%w0, %1"
1479   [(set_attr "type" "extend,load1")]
1480 )
1481
1482 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1483   [(set (match_operand:DI 0 "register_operand" "=r")
1484         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1485    (set (match_operand:DI 2 "register_operand" "=r")
1486         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1487   "rtx_equal_p (XEXP (operands[3], 0),
1488                 plus_constant (Pmode,
1489                                XEXP (operands[1], 0),
1490                                GET_MODE_SIZE (SImode)))"
1491   "ldp\\t%w0, %w2, %1"
1492   [(set_attr "type" "load2")]
1493 )
1494
1495 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1496   [(set (match_operand:GPI 0 "register_operand")
1497         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1498   ""
1499 )
1500
1501 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1502   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1503         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1504   ""
1505   "@
1506    sxt<SHORT:size>\t%<GPI:w>0, %w1
1507    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1508   [(set_attr "type" "extend,load1")]
1509 )
1510
1511 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1512   [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1513         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1514   ""
1515   "@
1516    uxt<SHORT:size>\t%<GPI:w>0, %w1
1517    ldr<SHORT:size>\t%w0, %1
1518    ldr\t%<SHORT:size>0, %1"
1519   [(set_attr "type" "extend,load1,load1")]
1520 )
1521
1522 (define_expand "<optab>qihi2"
1523   [(set (match_operand:HI 0 "register_operand")
1524         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1525   ""
1526 )
1527
1528 (define_insn "*<optab>qihi2_aarch64"
1529   [(set (match_operand:HI 0 "register_operand" "=r,r")
1530         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1531   ""
1532   "@
1533    <su>xtb\t%w0, %w1
1534    <ldrxt>b\t%w0, %1"
1535   [(set_attr "type" "extend,load1")]
1536 )
1537
1538 ;; -------------------------------------------------------------------
1539 ;; Simple arithmetic
1540 ;; -------------------------------------------------------------------
1541
1542 (define_expand "add<mode>3"
1543   [(set
1544     (match_operand:GPI 0 "register_operand" "")
1545     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1546               (match_operand:GPI 2 "aarch64_pluslong_operand" "")))]
1547   ""
1548   "
1549   if (! aarch64_plus_operand (operands[2], VOIDmode))
1550     {
1551       HOST_WIDE_INT imm = INTVAL (operands[2]);
1552
1553       if (aarch64_move_imm (imm, <MODE>mode) && can_create_pseudo_p ())
1554         {
1555           rtx tmp = gen_reg_rtx (<MODE>mode);
1556           emit_move_insn (tmp, operands[2]);
1557           operands[2] = tmp;
1558         }
1559       else
1560         {
1561           rtx subtarget = ((optimize && can_create_pseudo_p ())
1562                            ? gen_reg_rtx (<MODE>mode) : operands[0]);
1563
1564           if (imm < 0)
1565             imm = -(-imm & ~0xfff);
1566           else
1567             imm &= ~0xfff;
1568
1569           emit_insn (gen_add<mode>3 (subtarget, operands[1], GEN_INT (imm)));
1570           operands[1] = subtarget;
1571           operands[2] = GEN_INT (INTVAL (operands[2]) - imm);
1572         }
1573     }
1574   "
1575 )
1576
1577 (define_insn "*addsi3_aarch64"
1578   [(set
1579     (match_operand:SI 0 "register_operand" "=rk,rk,w,rk")
1580     (plus:SI
1581      (match_operand:SI 1 "register_operand" "%rk,rk,w,rk")
1582      (match_operand:SI 2 "aarch64_plus_operand" "I,r,w,J")))]
1583   ""
1584   "@
1585   add\\t%w0, %w1, %2
1586   add\\t%w0, %w1, %w2
1587   add\\t%0.2s, %1.2s, %2.2s
1588   sub\\t%w0, %w1, #%n2"
1589   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm")
1590    (set_attr "simd" "*,*,yes,*")]
1591 )
1592
1593 ;; zero_extend version of above
1594 (define_insn "*addsi3_aarch64_uxtw"
1595   [(set
1596     (match_operand:DI 0 "register_operand" "=rk,rk,rk")
1597     (zero_extend:DI
1598      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
1599               (match_operand:SI 2 "aarch64_plus_operand" "I,r,J"))))]
1600   ""
1601   "@
1602   add\\t%w0, %w1, %2
1603   add\\t%w0, %w1, %w2
1604   sub\\t%w0, %w1, #%n2"
1605   [(set_attr "type" "alu_imm,alu_sreg,alu_imm")]
1606 )
1607
1608 (define_insn "*adddi3_aarch64"
1609   [(set
1610     (match_operand:DI 0 "register_operand" "=rk,rk,rk,w")
1611     (plus:DI
1612      (match_operand:DI 1 "register_operand" "%rk,rk,rk,w")
1613      (match_operand:DI 2 "aarch64_plus_operand" "I,r,J,w")))]
1614   ""
1615   "@
1616   add\\t%x0, %x1, %2
1617   add\\t%x0, %x1, %x2
1618   sub\\t%x0, %x1, #%n2
1619   add\\t%d0, %d1, %d2"
1620   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,neon_add")
1621    (set_attr "simd" "*,*,*,yes")]
1622 )
1623
1624 (define_expand "addti3"
1625   [(set (match_operand:TI 0 "register_operand" "")
1626         (plus:TI (match_operand:TI 1 "register_operand" "")
1627                  (match_operand:TI 2 "register_operand" "")))]
1628   ""
1629 {
1630   rtx low = gen_reg_rtx (DImode);
1631   emit_insn (gen_adddi3_compare0 (low, gen_lowpart (DImode, operands[1]),
1632                                   gen_lowpart (DImode, operands[2])));
1633
1634   rtx high = gen_reg_rtx (DImode);
1635   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1636                                  gen_highpart (DImode, operands[2])));
1637
1638   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1639   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1640   DONE;
1641 })
1642
1643 (define_insn "add<mode>3_compare0"
1644   [(set (reg:CC_NZ CC_REGNUM)
1645         (compare:CC_NZ
1646          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1647                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1648          (const_int 0)))
1649    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1650         (plus:GPI (match_dup 1) (match_dup 2)))]
1651   ""
1652   "@
1653   adds\\t%<w>0, %<w>1, %<w>2
1654   adds\\t%<w>0, %<w>1, %<w>2
1655   subs\\t%<w>0, %<w>1, #%n2"
1656   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1657 )
1658
1659 ;; zero_extend version of above
1660 (define_insn "*addsi3_compare0_uxtw"
1661   [(set (reg:CC_NZ CC_REGNUM)
1662         (compare:CC_NZ
1663          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1664                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1665          (const_int 0)))
1666    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1667         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1668   ""
1669   "@
1670   adds\\t%w0, %w1, %w2
1671   adds\\t%w0, %w1, %w2
1672   subs\\t%w0, %w1, #%n2"
1673   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1674 )
1675
1676 (define_insn "*adds_shift_imm_<mode>"
1677   [(set (reg:CC_NZ CC_REGNUM)
1678         (compare:CC_NZ
1679          (plus:GPI (ASHIFT:GPI
1680                     (match_operand:GPI 1 "register_operand" "r")
1681                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1682                    (match_operand:GPI 3 "register_operand" "r"))
1683          (const_int 0)))
1684    (set (match_operand:GPI 0 "register_operand" "=r")
1685         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1686                   (match_dup 3)))]
1687   ""
1688   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1689   [(set_attr "type" "alus_shift_imm")]
1690 )
1691
1692 (define_insn "*subs_shift_imm_<mode>"
1693   [(set (reg:CC_NZ CC_REGNUM)
1694         (compare:CC_NZ
1695          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1696                     (ASHIFT:GPI
1697                      (match_operand:GPI 2 "register_operand" "r")
1698                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1699          (const_int 0)))
1700    (set (match_operand:GPI 0 "register_operand" "=r")
1701         (minus:GPI (match_dup 1)
1702                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1703   ""
1704   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1705   [(set_attr "type" "alus_shift_imm")]
1706 )
1707
1708 (define_insn "*adds_mul_imm_<mode>"
1709   [(set (reg:CC_NZ CC_REGNUM)
1710         (compare:CC_NZ
1711          (plus:GPI (mult:GPI
1712                     (match_operand:GPI 1 "register_operand" "r")
1713                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1714                    (match_operand:GPI 3 "register_operand" "r"))
1715          (const_int 0)))
1716    (set (match_operand:GPI 0 "register_operand" "=r")
1717         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1718                   (match_dup 3)))]
1719   ""
1720   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1721   [(set_attr "type" "alus_shift_imm")]
1722 )
1723
1724 (define_insn "*subs_mul_imm_<mode>"
1725   [(set (reg:CC_NZ CC_REGNUM)
1726         (compare:CC_NZ
1727          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1728                     (mult:GPI
1729                      (match_operand:GPI 2 "register_operand" "r")
1730                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1731          (const_int 0)))
1732    (set (match_operand:GPI 0 "register_operand" "=r")
1733         (minus:GPI (match_dup 1)
1734                    (mult:GPI (match_dup 2) (match_dup 3))))]
1735   ""
1736   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1737   [(set_attr "type" "alus_shift_imm")]
1738 )
1739
1740 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1741   [(set (reg:CC_NZ CC_REGNUM)
1742         (compare:CC_NZ
1743          (plus:GPI
1744           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1745           (match_operand:GPI 2 "register_operand" "r"))
1746         (const_int 0)))
1747    (set (match_operand:GPI 0 "register_operand" "=r")
1748         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1749   ""
1750   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1751   [(set_attr "type" "alus_ext")]
1752 )
1753
1754 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1755   [(set (reg:CC_NZ CC_REGNUM)
1756         (compare:CC_NZ
1757          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1758                     (ANY_EXTEND:GPI
1759                      (match_operand:ALLX 2 "register_operand" "r")))
1760         (const_int 0)))
1761    (set (match_operand:GPI 0 "register_operand" "=r")
1762         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1763   ""
1764   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1765   [(set_attr "type" "alus_ext")]
1766 )
1767
1768 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1769   [(set (reg:CC_NZ CC_REGNUM)
1770         (compare:CC_NZ
1771          (plus:GPI (ashift:GPI
1772                     (ANY_EXTEND:GPI
1773                      (match_operand:ALLX 1 "register_operand" "r"))
1774                     (match_operand 2 "aarch64_imm3" "Ui3"))
1775                    (match_operand:GPI 3 "register_operand" "r"))
1776          (const_int 0)))
1777    (set (match_operand:GPI 0 "register_operand" "=rk")
1778         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1779                               (match_dup 2))
1780                   (match_dup 3)))]
1781   ""
1782   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1783   [(set_attr "type" "alus_ext")]
1784 )
1785
1786 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1787   [(set (reg:CC_NZ CC_REGNUM)
1788         (compare:CC_NZ
1789          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1790                     (ashift:GPI
1791                      (ANY_EXTEND:GPI
1792                       (match_operand:ALLX 2 "register_operand" "r"))
1793                      (match_operand 3 "aarch64_imm3" "Ui3")))
1794          (const_int 0)))
1795    (set (match_operand:GPI 0 "register_operand" "=rk")
1796         (minus:GPI (match_dup 1)
1797                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
1798                                (match_dup 3))))]
1799   ""
1800   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
1801   [(set_attr "type" "alus_ext")]
1802 )
1803
1804 (define_insn "*adds_<optab><mode>_multp2"
1805   [(set (reg:CC_NZ CC_REGNUM)
1806         (compare:CC_NZ
1807          (plus:GPI (ANY_EXTRACT:GPI
1808                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1809                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1810                     (match_operand 3 "const_int_operand" "n")
1811                     (const_int 0))
1812                    (match_operand:GPI 4 "register_operand" "r"))
1813         (const_int 0)))
1814    (set (match_operand:GPI 0 "register_operand" "=r")
1815         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
1816                                    (match_dup 3)
1817                                    (const_int 0))
1818                   (match_dup 4)))]
1819   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1820   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1821   [(set_attr "type" "alus_ext")]
1822 )
1823
1824 (define_insn "*subs_<optab><mode>_multp2"
1825   [(set (reg:CC_NZ CC_REGNUM)
1826         (compare:CC_NZ
1827          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
1828                     (ANY_EXTRACT:GPI
1829                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1830                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1831                      (match_operand 3 "const_int_operand" "n")
1832                      (const_int 0)))
1833         (const_int 0)))
1834    (set (match_operand:GPI 0 "register_operand" "=r")
1835         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
1836                                   (mult:GPI (match_dup 1) (match_dup 2))
1837                                   (match_dup 3)
1838                                   (const_int 0))))]
1839   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1840   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1841   [(set_attr "type" "alus_ext")]
1842 )
1843
1844 (define_insn "*add<mode>3nr_compare0"
1845   [(set (reg:CC_NZ CC_REGNUM)
1846         (compare:CC_NZ
1847          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
1848                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
1849          (const_int 0)))]
1850   ""
1851   "@
1852   cmn\\t%<w>0, %<w>1
1853   cmn\\t%<w>0, %<w>1
1854   cmp\\t%<w>0, #%n1"
1855   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1856 )
1857
1858 (define_insn "*compare_neg<mode>"
1859   [(set (reg:CC_Z CC_REGNUM)
1860         (compare:CC_Z
1861          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
1862          (match_operand:GPI 1 "register_operand" "r")))]
1863   ""
1864   "cmn\\t%<w>1, %<w>0"
1865   [(set_attr "type" "alus_sreg")]
1866 )
1867
1868 (define_insn "*add_<shift>_<mode>"
1869   [(set (match_operand:GPI 0 "register_operand" "=r")
1870         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
1871                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1872                   (match_operand:GPI 3 "register_operand" "r")))]
1873   ""
1874   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1875   [(set_attr "type" "alu_shift_imm")]
1876 )
1877
1878 ;; zero_extend version of above
1879 (define_insn "*add_<shift>_si_uxtw"
1880   [(set (match_operand:DI 0 "register_operand" "=r")
1881         (zero_extend:DI
1882          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
1883                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
1884                   (match_operand:SI 3 "register_operand" "r"))))]
1885   ""
1886   "add\\t%w0, %w3, %w1, <shift> %2"
1887   [(set_attr "type" "alu_shift_imm")]
1888 )
1889
1890 (define_insn "*add_mul_imm_<mode>"
1891   [(set (match_operand:GPI 0 "register_operand" "=r")
1892         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1893                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1894                   (match_operand:GPI 3 "register_operand" "r")))]
1895   ""
1896   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1897   [(set_attr "type" "alu_shift_imm")]
1898 )
1899
1900 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
1901   [(set (match_operand:GPI 0 "register_operand" "=rk")
1902         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1903                   (match_operand:GPI 2 "register_operand" "r")))]
1904   ""
1905   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1906   [(set_attr "type" "alu_ext")]
1907 )
1908
1909 ;; zero_extend version of above
1910 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
1911   [(set (match_operand:DI 0 "register_operand" "=rk")
1912         (zero_extend:DI
1913          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
1914                   (match_operand:GPI 2 "register_operand" "r"))))]
1915   ""
1916   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
1917   [(set_attr "type" "alu_ext")]
1918 )
1919
1920 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
1921   [(set (match_operand:GPI 0 "register_operand" "=rk")
1922         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
1923                                (match_operand:ALLX 1 "register_operand" "r"))
1924                               (match_operand 2 "aarch64_imm3" "Ui3"))
1925                   (match_operand:GPI 3 "register_operand" "r")))]
1926   ""
1927   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1928   [(set_attr "type" "alu_ext")]
1929 )
1930
1931 ;; zero_extend version of above
1932 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
1933   [(set (match_operand:DI 0 "register_operand" "=rk")
1934         (zero_extend:DI
1935          (plus:SI (ashift:SI (ANY_EXTEND:SI
1936                               (match_operand:SHORT 1 "register_operand" "r"))
1937                              (match_operand 2 "aarch64_imm3" "Ui3"))
1938                   (match_operand:SI 3 "register_operand" "r"))))]
1939   ""
1940   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
1941   [(set_attr "type" "alu_ext")]
1942 )
1943
1944 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
1945   [(set (match_operand:GPI 0 "register_operand" "=rk")
1946         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
1947                              (match_operand:ALLX 1 "register_operand" "r"))
1948                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1949                   (match_operand:GPI 3 "register_operand" "r")))]
1950   ""
1951   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
1952   [(set_attr "type" "alu_ext")]
1953 )
1954
1955 ;; zero_extend version of above
1956 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
1957   [(set (match_operand:DI 0 "register_operand" "=rk")
1958         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
1959                              (match_operand:SHORT 1 "register_operand" "r"))
1960                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1961                   (match_operand:SI 3 "register_operand" "r"))))]
1962   ""
1963   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
1964   [(set_attr "type" "alu_ext")]
1965 )
1966
1967 (define_insn "*add_<optab><mode>_multp2"
1968   [(set (match_operand:GPI 0 "register_operand" "=rk")
1969         (plus:GPI (ANY_EXTRACT:GPI
1970                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1971                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1972                    (match_operand 3 "const_int_operand" "n")
1973                    (const_int 0))
1974                   (match_operand:GPI 4 "register_operand" "r")))]
1975   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1976   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1977   [(set_attr "type" "alu_ext")]
1978 )
1979
1980 ;; zero_extend version of above
1981 (define_insn "*add_<optab>si_multp2_uxtw"
1982   [(set (match_operand:DI 0 "register_operand" "=rk")
1983         (zero_extend:DI
1984          (plus:SI (ANY_EXTRACT:SI
1985                    (mult:SI (match_operand:SI 1 "register_operand" "r")
1986                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1987                    (match_operand 3 "const_int_operand" "n")
1988                    (const_int 0))
1989                   (match_operand:SI 4 "register_operand" "r"))))]
1990   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
1991   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
1992   [(set_attr "type" "alu_ext")]
1993 )
1994
1995 (define_insn "add<mode>3_carryin"
1996   [(set
1997     (match_operand:GPI 0 "register_operand" "=r")
1998     (plus:GPI (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
1999               (plus:GPI
2000                 (match_operand:GPI 1 "register_operand" "r")
2001                 (match_operand:GPI 2 "register_operand" "r"))))]
2002    ""
2003    "adc\\t%<w>0, %<w>1, %<w>2"
2004   [(set_attr "type" "adc_reg")]
2005 )
2006
2007 ;; zero_extend version of above
2008 (define_insn "*addsi3_carryin_uxtw"
2009   [(set
2010     (match_operand:DI 0 "register_operand" "=r")
2011     (zero_extend:DI
2012      (plus:SI (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2013               (plus:SI
2014                (match_operand:SI 1 "register_operand" "r")
2015                (match_operand:SI 2 "register_operand" "r")))))]
2016    ""
2017    "adc\\t%w0, %w1, %w2"
2018   [(set_attr "type" "adc_reg")]
2019 )
2020
2021 (define_insn "*add<mode>3_carryin_alt1"
2022   [(set
2023     (match_operand:GPI 0 "register_operand" "=r")
2024     (plus:GPI (plus:GPI
2025                 (match_operand:GPI 1 "register_operand" "r")
2026                 (match_operand:GPI 2 "register_operand" "r"))
2027               (geu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2028    ""
2029    "adc\\t%<w>0, %<w>1, %<w>2"
2030   [(set_attr "type" "adc_reg")]
2031 )
2032
2033 ;; zero_extend version of above
2034 (define_insn "*addsi3_carryin_alt1_uxtw"
2035   [(set
2036     (match_operand:DI 0 "register_operand" "=r")
2037     (zero_extend:DI
2038      (plus:SI (plus:SI
2039                (match_operand:SI 1 "register_operand" "r")
2040                (match_operand:SI 2 "register_operand" "r"))
2041               (geu:SI (reg:CC CC_REGNUM) (const_int 0)))))]
2042    ""
2043    "adc\\t%w0, %w1, %w2"
2044   [(set_attr "type" "adc_reg")]
2045 )
2046
2047 (define_insn "*add<mode>3_carryin_alt2"
2048   [(set
2049     (match_operand:GPI 0 "register_operand" "=r")
2050     (plus:GPI (plus:GPI
2051                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
2052                 (match_operand:GPI 1 "register_operand" "r"))
2053               (match_operand:GPI 2 "register_operand" "r")))]
2054    ""
2055    "adc\\t%<w>0, %<w>1, %<w>2"
2056   [(set_attr "type" "adc_reg")]
2057 )
2058
2059 ;; zero_extend version of above
2060 (define_insn "*addsi3_carryin_alt2_uxtw"
2061   [(set
2062     (match_operand:DI 0 "register_operand" "=r")
2063     (zero_extend:DI
2064      (plus:SI (plus:SI
2065                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2066                (match_operand:SI 1 "register_operand" "r"))
2067               (match_operand:SI 2 "register_operand" "r"))))]
2068    ""
2069    "adc\\t%w0, %w1, %w2"
2070   [(set_attr "type" "adc_reg")]
2071 )
2072
2073 (define_insn "*add<mode>3_carryin_alt3"
2074   [(set
2075     (match_operand:GPI 0 "register_operand" "=r")
2076     (plus:GPI (plus:GPI
2077                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
2078                 (match_operand:GPI 2 "register_operand" "r"))
2079               (match_operand:GPI 1 "register_operand" "r")))]
2080    ""
2081    "adc\\t%<w>0, %<w>1, %<w>2"
2082   [(set_attr "type" "adc_reg")]
2083 )
2084
2085 ;; zero_extend version of above
2086 (define_insn "*addsi3_carryin_alt3_uxtw"
2087   [(set
2088     (match_operand:DI 0 "register_operand" "=r")
2089     (zero_extend:DI
2090      (plus:SI (plus:SI
2091                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
2092                (match_operand:SI 2 "register_operand" "r"))
2093               (match_operand:SI 1 "register_operand" "r"))))]
2094    ""
2095    "adc\\t%w0, %w1, %w2"
2096   [(set_attr "type" "adc_reg")]
2097 )
2098
2099 (define_insn "*add_uxt<mode>_shift2"
2100   [(set (match_operand:GPI 0 "register_operand" "=rk")
2101         (plus:GPI (and:GPI
2102                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2103                                (match_operand 2 "aarch64_imm3" "Ui3"))
2104                    (match_operand 3 "const_int_operand" "n"))
2105                   (match_operand:GPI 4 "register_operand" "r")))]
2106   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2107   "*
2108   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2109                                            INTVAL (operands[3])));
2110   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2111   [(set_attr "type" "alu_ext")]
2112 )
2113
2114 ;; zero_extend version of above
2115 (define_insn "*add_uxtsi_shift2_uxtw"
2116   [(set (match_operand:DI 0 "register_operand" "=rk")
2117         (zero_extend:DI
2118          (plus:SI (and:SI
2119                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2120                               (match_operand 2 "aarch64_imm3" "Ui3"))
2121                    (match_operand 3 "const_int_operand" "n"))
2122                   (match_operand:SI 4 "register_operand" "r"))))]
2123   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2124   "*
2125   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2126                                            INTVAL (operands[3])));
2127   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2128   [(set_attr "type" "alu_ext")]
2129 )
2130
2131 (define_insn "*add_uxt<mode>_multp2"
2132   [(set (match_operand:GPI 0 "register_operand" "=rk")
2133         (plus:GPI (and:GPI
2134                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2135                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2136                    (match_operand 3 "const_int_operand" "n"))
2137                   (match_operand:GPI 4 "register_operand" "r")))]
2138   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2139   "*
2140   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2141                                            INTVAL (operands[3])));
2142   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2143   [(set_attr "type" "alu_ext")]
2144 )
2145
2146 ;; zero_extend version of above
2147 (define_insn "*add_uxtsi_multp2_uxtw"
2148   [(set (match_operand:DI 0 "register_operand" "=rk")
2149         (zero_extend:DI
2150          (plus:SI (and:SI
2151                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2152                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2153                    (match_operand 3 "const_int_operand" "n"))
2154                   (match_operand:SI 4 "register_operand" "r"))))]
2155   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2156   "*
2157   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2158                                            INTVAL (operands[3])));
2159   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2160   [(set_attr "type" "alu_ext")]
2161 )
2162
2163 (define_insn "subsi3"
2164   [(set (match_operand:SI 0 "register_operand" "=rk")
2165         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2166                   (match_operand:SI 2 "register_operand" "r")))]
2167   ""
2168   "sub\\t%w0, %w1, %w2"
2169   [(set_attr "type" "alu_sreg")]
2170 )
2171
2172 ;; zero_extend version of above
2173 (define_insn "*subsi3_uxtw"
2174   [(set (match_operand:DI 0 "register_operand" "=rk")
2175         (zero_extend:DI
2176          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2177                    (match_operand:SI 2 "register_operand" "r"))))]
2178   ""
2179   "sub\\t%w0, %w1, %w2"
2180   [(set_attr "type" "alu_sreg")]
2181 )
2182
2183 (define_insn "subdi3"
2184   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2185         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2186                   (match_operand:DI 2 "register_operand" "r,w")))]
2187   ""
2188   "@
2189    sub\\t%x0, %x1, %x2
2190    sub\\t%d0, %d1, %d2"
2191   [(set_attr "type" "alu_sreg, neon_sub")
2192    (set_attr "simd" "*,yes")]
2193 )
2194
2195 (define_expand "subti3"
2196   [(set (match_operand:TI 0 "register_operand" "")
2197         (minus:TI (match_operand:TI 1 "register_operand" "")
2198                   (match_operand:TI 2 "register_operand" "")))]
2199   ""
2200 {
2201   rtx low = gen_reg_rtx (DImode);
2202   emit_insn (gen_subdi3_compare0 (low, gen_lowpart (DImode, operands[1]),
2203                                   gen_lowpart (DImode, operands[2])));
2204
2205   rtx high = gen_reg_rtx (DImode);
2206   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2207                                  gen_highpart (DImode, operands[2])));
2208
2209   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2210   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2211   DONE;
2212 })
2213
2214 (define_insn "sub<mode>3_compare0"
2215   [(set (reg:CC_NZ CC_REGNUM)
2216         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2217                                   (match_operand:GPI 2 "register_operand" "r"))
2218                        (const_int 0)))
2219    (set (match_operand:GPI 0 "register_operand" "=r")
2220         (minus:GPI (match_dup 1) (match_dup 2)))]
2221   ""
2222   "subs\\t%<w>0, %<w>1, %<w>2"
2223   [(set_attr "type" "alus_sreg")]
2224 )
2225
2226 ;; zero_extend version of above
2227 (define_insn "*subsi3_compare0_uxtw"
2228   [(set (reg:CC_NZ CC_REGNUM)
2229         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2230                                  (match_operand:SI 2 "register_operand" "r"))
2231                        (const_int 0)))
2232    (set (match_operand:DI 0 "register_operand" "=r")
2233         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2234   ""
2235   "subs\\t%w0, %w1, %w2"
2236   [(set_attr "type" "alus_sreg")]
2237 )
2238
2239 (define_insn "*sub_<shift>_<mode>"
2240   [(set (match_operand:GPI 0 "register_operand" "=r")
2241         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2242                    (ASHIFT:GPI
2243                     (match_operand:GPI 1 "register_operand" "r")
2244                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2245   ""
2246   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2247   [(set_attr "type" "alu_shift_imm")]
2248 )
2249
2250 ;; zero_extend version of above
2251 (define_insn "*sub_<shift>_si_uxtw"
2252   [(set (match_operand:DI 0 "register_operand" "=r")
2253         (zero_extend:DI
2254          (minus:SI (match_operand:SI 3 "register_operand" "r")
2255                    (ASHIFT:SI
2256                     (match_operand:SI 1 "register_operand" "r")
2257                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2258   ""
2259   "sub\\t%w0, %w3, %w1, <shift> %2"
2260   [(set_attr "type" "alu_shift_imm")]
2261 )
2262
2263 (define_insn "*sub_mul_imm_<mode>"
2264   [(set (match_operand:GPI 0 "register_operand" "=r")
2265         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2266                    (mult:GPI
2267                     (match_operand:GPI 1 "register_operand" "r")
2268                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2269   ""
2270   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2271   [(set_attr "type" "alu_shift_imm")]
2272 )
2273
2274 ;; zero_extend version of above
2275 (define_insn "*sub_mul_imm_si_uxtw"
2276   [(set (match_operand:DI 0 "register_operand" "=r")
2277         (zero_extend:DI
2278          (minus:SI (match_operand:SI 3 "register_operand" "r")
2279                    (mult:SI
2280                     (match_operand:SI 1 "register_operand" "r")
2281                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2282   ""
2283   "sub\\t%w0, %w3, %w1, lsl %p2"
2284   [(set_attr "type" "alu_shift_imm")]
2285 )
2286
2287 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2288   [(set (match_operand:GPI 0 "register_operand" "=rk")
2289         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2290                    (ANY_EXTEND:GPI
2291                     (match_operand:ALLX 2 "register_operand" "r"))))]
2292   ""
2293   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2294   [(set_attr "type" "alu_ext")]
2295 )
2296
2297 ;; zero_extend version of above
2298 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2299   [(set (match_operand:DI 0 "register_operand" "=rk")
2300         (zero_extend:DI
2301          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2302                    (ANY_EXTEND:SI
2303                     (match_operand:SHORT 2 "register_operand" "r")))))]
2304   ""
2305   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2306   [(set_attr "type" "alu_ext")]
2307 )
2308
2309 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2310   [(set (match_operand:GPI 0 "register_operand" "=rk")
2311         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2312                    (ashift:GPI (ANY_EXTEND:GPI
2313                                 (match_operand:ALLX 2 "register_operand" "r"))
2314                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2315   ""
2316   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2317   [(set_attr "type" "alu_ext")]
2318 )
2319
2320 ;; zero_extend version of above
2321 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2322   [(set (match_operand:DI 0 "register_operand" "=rk")
2323         (zero_extend:DI
2324          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2325                    (ashift:SI (ANY_EXTEND:SI
2326                                (match_operand:SHORT 2 "register_operand" "r"))
2327                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2328   ""
2329   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2330   [(set_attr "type" "alu_ext")]
2331 )
2332
2333 (define_insn "*sub_<optab><mode>_multp2"
2334   [(set (match_operand:GPI 0 "register_operand" "=rk")
2335         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2336                    (ANY_EXTRACT:GPI
2337                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2338                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2339                     (match_operand 3 "const_int_operand" "n")
2340                     (const_int 0))))]
2341   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2342   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2343   [(set_attr "type" "alu_ext")]
2344 )
2345
2346 ;; zero_extend version of above
2347 (define_insn "*sub_<optab>si_multp2_uxtw"
2348   [(set (match_operand:DI 0 "register_operand" "=rk")
2349         (zero_extend:DI
2350          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2351                    (ANY_EXTRACT:SI
2352                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2353                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2354                     (match_operand 3 "const_int_operand" "n")
2355                     (const_int 0)))))]
2356   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2357   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2358   [(set_attr "type" "alu_ext")]
2359 )
2360
2361 (define_insn "sub<mode>3_carryin"
2362   [(set
2363     (match_operand:GPI 0 "register_operand" "=r")
2364     (minus:GPI (minus:GPI
2365                 (match_operand:GPI 1 "register_operand" "r")
2366                 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2367                (match_operand:GPI 2 "register_operand" "r")))]
2368    ""
2369    "sbc\\t%<w>0, %<w>1, %<w>2"
2370   [(set_attr "type" "adc_reg")]
2371 )
2372
2373 ;; zero_extend version of the above
2374 (define_insn "*subsi3_carryin_uxtw"
2375   [(set
2376     (match_operand:DI 0 "register_operand" "=r")
2377     (zero_extend:DI
2378      (minus:SI (minus:SI
2379                 (match_operand:SI 1 "register_operand" "r")
2380                 (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2381                (match_operand:SI 2 "register_operand" "r"))))]
2382    ""
2383    "sbc\\t%w0, %w1, %w2"
2384   [(set_attr "type" "adc_reg")]
2385 )
2386
2387 (define_insn "*sub_uxt<mode>_shift2"
2388   [(set (match_operand:GPI 0 "register_operand" "=rk")
2389         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2390                    (and:GPI
2391                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2392                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2393                     (match_operand 3 "const_int_operand" "n"))))]
2394   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2395   "*
2396   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2397                                            INTVAL (operands[3])));
2398   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2399   [(set_attr "type" "alu_ext")]
2400 )
2401
2402 ;; zero_extend version of above
2403 (define_insn "*sub_uxtsi_shift2_uxtw"
2404   [(set (match_operand:DI 0 "register_operand" "=rk")
2405         (zero_extend:DI
2406          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2407                    (and:SI
2408                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2409                                (match_operand 2 "aarch64_imm3" "Ui3"))
2410                     (match_operand 3 "const_int_operand" "n")))))]
2411   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2412   "*
2413   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2414                                            INTVAL (operands[3])));
2415   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2416   [(set_attr "type" "alu_ext")]
2417 )
2418
2419 (define_insn "*sub_uxt<mode>_multp2"
2420   [(set (match_operand:GPI 0 "register_operand" "=rk")
2421         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2422                    (and:GPI
2423                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2424                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2425                     (match_operand 3 "const_int_operand" "n"))))]
2426   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2427   "*
2428   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2429                                            INTVAL (operands[3])));
2430   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2431   [(set_attr "type" "alu_ext")]
2432 )
2433
2434 ;; zero_extend version of above
2435 (define_insn "*sub_uxtsi_multp2_uxtw"
2436   [(set (match_operand:DI 0 "register_operand" "=rk")
2437         (zero_extend:DI
2438          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2439                    (and:SI
2440                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2441                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2442                     (match_operand 3 "const_int_operand" "n")))))]
2443   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2444   "*
2445   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2446                                            INTVAL (operands[3])));
2447   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2448   [(set_attr "type" "alu_ext")]
2449 )
2450
2451 (define_expand "abs<mode>2"
2452   [(match_operand:GPI 0 "register_operand" "")
2453    (match_operand:GPI 1 "register_operand" "")]
2454   ""
2455   {
2456     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2457     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2458     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2459     DONE;
2460   }
2461 )
2462
2463 (define_insn "neg<mode>2"
2464   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2465         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2466   ""
2467   "@
2468    neg\\t%<w>0, %<w>1
2469    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2470   [(set_attr "type" "alu_sreg, neon_neg<q>")
2471    (set_attr "simd" "*,yes")]
2472 )
2473
2474 ;; zero_extend version of above
2475 (define_insn "*negsi2_uxtw"
2476   [(set (match_operand:DI 0 "register_operand" "=r")
2477         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2478   ""
2479   "neg\\t%w0, %w1"
2480   [(set_attr "type" "alu_sreg")]
2481 )
2482
2483 (define_insn "*ngc<mode>"
2484   [(set (match_operand:GPI 0 "register_operand" "=r")
2485         (minus:GPI (neg:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2486                    (match_operand:GPI 1 "register_operand" "r")))]
2487   ""
2488   "ngc\\t%<w>0, %<w>1"
2489   [(set_attr "type" "adc_reg")]
2490 )
2491
2492 (define_insn "*ngcsi_uxtw"
2493   [(set (match_operand:DI 0 "register_operand" "=r")
2494         (zero_extend:DI
2495          (minus:SI (neg:SI (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2496                    (match_operand:SI 1 "register_operand" "r"))))]
2497   ""
2498   "ngc\\t%w0, %w1"
2499   [(set_attr "type" "adc_reg")]
2500 )
2501
2502 (define_insn "neg<mode>2_compare0"
2503   [(set (reg:CC_NZ CC_REGNUM)
2504         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2505                        (const_int 0)))
2506    (set (match_operand:GPI 0 "register_operand" "=r")
2507         (neg:GPI (match_dup 1)))]
2508   ""
2509   "negs\\t%<w>0, %<w>1"
2510   [(set_attr "type" "alus_sreg")]
2511 )
2512
2513 ;; zero_extend version of above
2514 (define_insn "*negsi2_compare0_uxtw"
2515   [(set (reg:CC_NZ CC_REGNUM)
2516         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2517                        (const_int 0)))
2518    (set (match_operand:DI 0 "register_operand" "=r")
2519         (zero_extend:DI (neg:SI (match_dup 1))))]
2520   ""
2521   "negs\\t%w0, %w1"
2522   [(set_attr "type" "alus_sreg")]
2523 )
2524
2525 (define_insn "*neg_<shift><mode>3_compare0"
2526   [(set (reg:CC_NZ CC_REGNUM)
2527         (compare:CC_NZ
2528          (neg:GPI (ASHIFT:GPI
2529                    (match_operand:GPI 1 "register_operand" "r")
2530                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2531          (const_int 0)))
2532    (set (match_operand:GPI 0 "register_operand" "=r")
2533         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2534   ""
2535   "negs\\t%<w>0, %<w>1, <shift> %2"
2536   [(set_attr "type" "alus_shift_imm")]
2537 )
2538
2539 (define_insn "*neg_<shift>_<mode>2"
2540   [(set (match_operand:GPI 0 "register_operand" "=r")
2541         (neg:GPI (ASHIFT:GPI
2542                   (match_operand:GPI 1 "register_operand" "r")
2543                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2544   ""
2545   "neg\\t%<w>0, %<w>1, <shift> %2"
2546   [(set_attr "type" "alu_shift_imm")]
2547 )
2548
2549 ;; zero_extend version of above
2550 (define_insn "*neg_<shift>_si2_uxtw"
2551   [(set (match_operand:DI 0 "register_operand" "=r")
2552         (zero_extend:DI
2553          (neg:SI (ASHIFT:SI
2554                   (match_operand:SI 1 "register_operand" "r")
2555                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2556   ""
2557   "neg\\t%w0, %w1, <shift> %2"
2558   [(set_attr "type" "alu_shift_imm")]
2559 )
2560
2561 (define_insn "*neg_mul_imm_<mode>2"
2562   [(set (match_operand:GPI 0 "register_operand" "=r")
2563         (neg:GPI (mult:GPI
2564                   (match_operand:GPI 1 "register_operand" "r")
2565                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2566   ""
2567   "neg\\t%<w>0, %<w>1, lsl %p2"
2568   [(set_attr "type" "alu_shift_imm")]
2569 )
2570
2571 ;; zero_extend version of above
2572 (define_insn "*neg_mul_imm_si2_uxtw"
2573   [(set (match_operand:DI 0 "register_operand" "=r")
2574         (zero_extend:DI
2575          (neg:SI (mult:SI
2576                   (match_operand:SI 1 "register_operand" "r")
2577                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2578   ""
2579   "neg\\t%w0, %w1, lsl %p2"
2580   [(set_attr "type" "alu_shift_imm")]
2581 )
2582
2583 (define_insn "mul<mode>3"
2584   [(set (match_operand:GPI 0 "register_operand" "=r")
2585         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2586                   (match_operand:GPI 2 "register_operand" "r")))]
2587   ""
2588   "mul\\t%<w>0, %<w>1, %<w>2"
2589   [(set_attr "type" "mul")]
2590 )
2591
2592 ;; zero_extend version of above
2593 (define_insn "*mulsi3_uxtw"
2594   [(set (match_operand:DI 0 "register_operand" "=r")
2595         (zero_extend:DI
2596          (mult:SI (match_operand:SI 1 "register_operand" "r")
2597                   (match_operand:SI 2 "register_operand" "r"))))]
2598   ""
2599   "mul\\t%w0, %w1, %w2"
2600   [(set_attr "type" "mul")]
2601 )
2602
2603 (define_insn "madd<mode>"
2604   [(set (match_operand:GPI 0 "register_operand" "=r")
2605         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2606                             (match_operand:GPI 2 "register_operand" "r"))
2607                   (match_operand:GPI 3 "register_operand" "r")))]
2608   ""
2609   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2610   [(set_attr "type" "mla")]
2611 )
2612
2613 ;; zero_extend version of above
2614 (define_insn "*maddsi_uxtw"
2615   [(set (match_operand:DI 0 "register_operand" "=r")
2616         (zero_extend:DI
2617          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2618                            (match_operand:SI 2 "register_operand" "r"))
2619                   (match_operand:SI 3 "register_operand" "r"))))]
2620   ""
2621   "madd\\t%w0, %w1, %w2, %w3"
2622   [(set_attr "type" "mla")]
2623 )
2624
2625 (define_insn "*msub<mode>"
2626   [(set (match_operand:GPI 0 "register_operand" "=r")
2627         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2628                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2629                              (match_operand:GPI 2 "register_operand" "r"))))]
2630
2631   ""
2632   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2633   [(set_attr "type" "mla")]
2634 )
2635
2636 ;; zero_extend version of above
2637 (define_insn "*msubsi_uxtw"
2638   [(set (match_operand:DI 0 "register_operand" "=r")
2639         (zero_extend:DI
2640          (minus:SI (match_operand:SI 3 "register_operand" "r")
2641                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2642                             (match_operand:SI 2 "register_operand" "r")))))]
2643
2644   ""
2645   "msub\\t%w0, %w1, %w2, %w3"
2646   [(set_attr "type" "mla")]
2647 )
2648
2649 (define_insn "*mul<mode>_neg"
2650   [(set (match_operand:GPI 0 "register_operand" "=r")
2651         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2652                   (match_operand:GPI 2 "register_operand" "r")))]
2653
2654   ""
2655   "mneg\\t%<w>0, %<w>1, %<w>2"
2656   [(set_attr "type" "mul")]
2657 )
2658
2659 ;; zero_extend version of above
2660 (define_insn "*mulsi_neg_uxtw"
2661   [(set (match_operand:DI 0 "register_operand" "=r")
2662         (zero_extend:DI
2663          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2664                   (match_operand:SI 2 "register_operand" "r"))))]
2665
2666   ""
2667   "mneg\\t%w0, %w1, %w2"
2668   [(set_attr "type" "mul")]
2669 )
2670
2671 (define_insn "<su_optab>mulsidi3"
2672   [(set (match_operand:DI 0 "register_operand" "=r")
2673         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2674                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2675   ""
2676   "<su>mull\\t%0, %w1, %w2"
2677   [(set_attr "type" "<su>mull")]
2678 )
2679
2680 (define_insn "<su_optab>maddsidi4"
2681   [(set (match_operand:DI 0 "register_operand" "=r")
2682         (plus:DI (mult:DI
2683                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2684                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
2685                  (match_operand:DI 3 "register_operand" "r")))]
2686   ""
2687   "<su>maddl\\t%0, %w1, %w2, %3"
2688   [(set_attr "type" "<su>mlal")]
2689 )
2690
2691 (define_insn "<su_optab>msubsidi4"
2692   [(set (match_operand:DI 0 "register_operand" "=r")
2693         (minus:DI
2694          (match_operand:DI 3 "register_operand" "r")
2695          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2696                   (ANY_EXTEND:DI
2697                    (match_operand:SI 2 "register_operand" "r")))))]
2698   ""
2699   "<su>msubl\\t%0, %w1, %w2, %3"
2700   [(set_attr "type" "<su>mlal")]
2701 )
2702
2703 (define_insn "*<su_optab>mulsidi_neg"
2704   [(set (match_operand:DI 0 "register_operand" "=r")
2705         (mult:DI (neg:DI
2706                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
2707                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2708   ""
2709   "<su>mnegl\\t%0, %w1, %w2"
2710   [(set_attr "type" "<su>mull")]
2711 )
2712
2713 (define_expand "<su_optab>mulditi3"
2714   [(set (match_operand:TI 0 "register_operand")
2715         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
2716                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
2717   ""
2718 {
2719   rtx low = gen_reg_rtx (DImode);
2720   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
2721
2722   rtx high = gen_reg_rtx (DImode);
2723   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
2724
2725   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2726   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2727   DONE;
2728 })
2729
2730 ;; The default expansion of multi3 using umuldi3_highpart will perform
2731 ;; the additions in an order that fails to combine into two madd insns.
2732 (define_expand "multi3"
2733   [(set (match_operand:TI 0 "register_operand")
2734         (mult:TI (match_operand:TI 1 "register_operand")
2735                  (match_operand:TI 2 "register_operand")))]
2736   ""
2737 {
2738   rtx l0 = gen_reg_rtx (DImode);
2739   rtx l1 = gen_lowpart (DImode, operands[1]);
2740   rtx l2 = gen_lowpart (DImode, operands[2]);
2741   rtx h0 = gen_reg_rtx (DImode);
2742   rtx h1 = gen_highpart (DImode, operands[1]);
2743   rtx h2 = gen_highpart (DImode, operands[2]);
2744
2745   emit_insn (gen_muldi3 (l0, l1, l2));
2746   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
2747   emit_insn (gen_madddi (h0, h1, l2, h0));
2748   emit_insn (gen_madddi (h0, l1, h2, h0));
2749
2750   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
2751   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
2752   DONE;
2753 })
2754
2755 (define_insn "<su>muldi3_highpart"
2756   [(set (match_operand:DI 0 "register_operand" "=r")
2757         (truncate:DI
2758          (lshiftrt:TI
2759           (mult:TI
2760            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
2761            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
2762           (const_int 64))))]
2763   ""
2764   "<su>mulh\\t%0, %1, %2"
2765   [(set_attr "type" "<su>mull")]
2766 )
2767
2768 (define_insn "<su_optab>div<mode>3"
2769   [(set (match_operand:GPI 0 "register_operand" "=r")
2770         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
2771                      (match_operand:GPI 2 "register_operand" "r")))]
2772   ""
2773   "<su>div\\t%<w>0, %<w>1, %<w>2"
2774   [(set_attr "type" "<su>div")]
2775 )
2776
2777 ;; zero_extend version of above
2778 (define_insn "*<su_optab>divsi3_uxtw"
2779   [(set (match_operand:DI 0 "register_operand" "=r")
2780         (zero_extend:DI
2781          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
2782                      (match_operand:SI 2 "register_operand" "r"))))]
2783   ""
2784   "<su>div\\t%w0, %w1, %w2"
2785   [(set_attr "type" "<su>div")]
2786 )
2787
2788 ;; -------------------------------------------------------------------
2789 ;; Comparison insns
2790 ;; -------------------------------------------------------------------
2791
2792 (define_insn "*cmp<mode>"
2793   [(set (reg:CC CC_REGNUM)
2794         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
2795                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
2796   ""
2797   "@
2798    cmp\\t%<w>0, %<w>1
2799    cmp\\t%<w>0, %<w>1
2800    cmn\\t%<w>0, #%n1"
2801   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2802 )
2803
2804 (define_insn "*cmp<mode>"
2805   [(set (reg:CCFP CC_REGNUM)
2806         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
2807                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2808    "TARGET_FLOAT"
2809    "@
2810     fcmp\\t%<s>0, #0.0
2811     fcmp\\t%<s>0, %<s>1"
2812   [(set_attr "type" "fcmp<s>")]
2813 )
2814
2815 (define_insn "*cmpe<mode>"
2816   [(set (reg:CCFPE CC_REGNUM)
2817         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
2818                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2819    "TARGET_FLOAT"
2820    "@
2821     fcmpe\\t%<s>0, #0.0
2822     fcmpe\\t%<s>0, %<s>1"
2823   [(set_attr "type" "fcmp<s>")]
2824 )
2825
2826 (define_insn "*cmp_swp_<shift>_reg<mode>"
2827   [(set (reg:CC_SWP CC_REGNUM)
2828         (compare:CC_SWP (ASHIFT:GPI
2829                          (match_operand:GPI 0 "register_operand" "r")
2830                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
2831                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
2832   ""
2833   "cmp\\t%<w>2, %<w>0, <shift> %1"
2834   [(set_attr "type" "alus_shift_imm")]
2835 )
2836
2837 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
2838   [(set (reg:CC_SWP CC_REGNUM)
2839         (compare:CC_SWP (ANY_EXTEND:GPI
2840                          (match_operand:ALLX 0 "register_operand" "r"))
2841                         (match_operand:GPI 1 "register_operand" "r")))]
2842   ""
2843   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
2844   [(set_attr "type" "alus_ext")]
2845 )
2846
2847 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
2848   [(set (reg:CC_SWP CC_REGNUM)
2849         (compare:CC_SWP (ashift:GPI
2850                          (ANY_EXTEND:GPI
2851                           (match_operand:ALLX 0 "register_operand" "r"))
2852                          (match_operand 1 "aarch64_imm3" "Ui3"))
2853         (match_operand:GPI 2 "register_operand" "r")))]
2854   ""
2855   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
2856   [(set_attr "type" "alus_ext")]
2857 )
2858
2859 ;; -------------------------------------------------------------------
2860 ;; Store-flag and conditional select insns
2861 ;; -------------------------------------------------------------------
2862
2863 (define_expand "cstore<mode>4"
2864   [(set (match_operand:SI 0 "register_operand" "")
2865         (match_operator:SI 1 "aarch64_comparison_operator"
2866          [(match_operand:GPI 2 "register_operand" "")
2867           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
2868   ""
2869   "
2870   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2871                                       operands[3]);
2872   operands[3] = const0_rtx;
2873   "
2874 )
2875
2876 (define_expand "cstorecc4"
2877   [(set (match_operand:SI 0 "register_operand")
2878        (match_operator 1 "aarch64_comparison_operator"
2879         [(match_operand 2 "ccmp_cc_register")
2880          (match_operand 3 "const0_operand")]))]
2881   ""
2882 "{
2883   emit_insn (gen_rtx_SET (operands[0], operands[1]));
2884   DONE;
2885 }")
2886
2887
2888 (define_expand "cstore<mode>4"
2889   [(set (match_operand:SI 0 "register_operand" "")
2890         (match_operator:SI 1 "aarch64_comparison_operator"
2891          [(match_operand:GPF 2 "register_operand" "")
2892           (match_operand:GPF 3 "register_operand" "")]))]
2893   ""
2894   "
2895   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2896                                       operands[3]);
2897   operands[3] = const0_rtx;
2898   "
2899 )
2900
2901 (define_insn "*cstore<mode>_insn"
2902   [(set (match_operand:ALLI 0 "register_operand" "=r")
2903         (match_operator:ALLI 1 "aarch64_comparison_operator"
2904          [(match_operand 2 "cc_register" "") (const_int 0)]))]
2905   ""
2906   "cset\\t%<w>0, %m1"
2907   [(set_attr "type" "csel")]
2908 )
2909
2910 ;; zero_extend version of the above
2911 (define_insn "*cstoresi_insn_uxtw"
2912   [(set (match_operand:DI 0 "register_operand" "=r")
2913         (zero_extend:DI
2914          (match_operator:SI 1 "aarch64_comparison_operator"
2915           [(match_operand 2 "cc_register" "") (const_int 0)])))]
2916   ""
2917   "cset\\t%w0, %m1"
2918   [(set_attr "type" "csel")]
2919 )
2920
2921 (define_insn "cstore<mode>_neg"
2922   [(set (match_operand:ALLI 0 "register_operand" "=r")
2923         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator"
2924                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
2925   ""
2926   "csetm\\t%<w>0, %m1"
2927   [(set_attr "type" "csel")]
2928 )
2929
2930 ;; zero_extend version of the above
2931 (define_insn "*cstoresi_neg_uxtw"
2932   [(set (match_operand:DI 0 "register_operand" "=r")
2933         (zero_extend:DI
2934          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator"
2935                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
2936   ""
2937   "csetm\\t%w0, %m1"
2938   [(set_attr "type" "csel")]
2939 )
2940
2941 (define_expand "cmov<mode>6"
2942   [(set (match_operand:GPI 0 "register_operand" "")
2943         (if_then_else:GPI
2944          (match_operator 1 "aarch64_comparison_operator"
2945           [(match_operand:GPI 2 "register_operand" "")
2946            (match_operand:GPI 3 "aarch64_plus_operand" "")])
2947          (match_operand:GPI 4 "register_operand" "")
2948          (match_operand:GPI 5 "register_operand" "")))]
2949   ""
2950   "
2951   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2952                                       operands[3]);
2953   operands[3] = const0_rtx;
2954   "
2955 )
2956
2957 (define_expand "cmov<mode>6"
2958   [(set (match_operand:GPF 0 "register_operand" "")
2959         (if_then_else:GPF
2960          (match_operator 1 "aarch64_comparison_operator"
2961           [(match_operand:GPF 2 "register_operand" "")
2962            (match_operand:GPF 3 "register_operand" "")])
2963          (match_operand:GPF 4 "register_operand" "")
2964          (match_operand:GPF 5 "register_operand" "")))]
2965   ""
2966   "
2967   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2968                                       operands[3]);
2969   operands[3] = const0_rtx;
2970   "
2971 )
2972
2973 (define_insn "*cmov<mode>_insn"
2974   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
2975         (if_then_else:ALLI
2976          (match_operator 1 "aarch64_comparison_operator"
2977           [(match_operand 2 "cc_register" "") (const_int 0)])
2978          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
2979          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
2980   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
2981      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
2982   ;; Final two alternatives should be unreachable, but included for completeness
2983   "@
2984    csel\\t%<w>0, %<w>3, %<w>4, %m1
2985    csinv\\t%<w>0, %<w>3, <w>zr, %m1
2986    csinv\\t%<w>0, %<w>4, <w>zr, %M1
2987    csinc\\t%<w>0, %<w>3, <w>zr, %m1
2988    csinc\\t%<w>0, %<w>4, <w>zr, %M1
2989    mov\\t%<w>0, -1
2990    mov\\t%<w>0, 1"
2991   [(set_attr "type" "csel")]
2992 )
2993
2994 ;; zero_extend version of above
2995 (define_insn "*cmovsi_insn_uxtw"
2996   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
2997         (zero_extend:DI
2998          (if_then_else:SI
2999           (match_operator 1 "aarch64_comparison_operator"
3000            [(match_operand 2 "cc_register" "") (const_int 0)])
3001           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3002           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3003   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3004      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3005   ;; Final two alternatives should be unreachable, but included for completeness
3006   "@
3007    csel\\t%w0, %w3, %w4, %m1
3008    csinv\\t%w0, %w3, wzr, %m1
3009    csinv\\t%w0, %w4, wzr, %M1
3010    csinc\\t%w0, %w3, wzr, %m1
3011    csinc\\t%w0, %w4, wzr, %M1
3012    mov\\t%w0, -1
3013    mov\\t%w0, 1"
3014   [(set_attr "type" "csel")]
3015 )
3016
3017 (define_insn "*cmovdi_insn_uxtw"
3018   [(set (match_operand:DI 0 "register_operand" "=r")
3019         (if_then_else:DI
3020          (match_operator 1 "aarch64_comparison_operator"
3021           [(match_operand 2 "cc_register" "") (const_int 0)])
3022          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3023          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3024   ""
3025   "csel\\t%w0, %w3, %w4, %m1"
3026   [(set_attr "type" "csel")]
3027 )
3028
3029 (define_insn "*cmov<mode>_insn"
3030   [(set (match_operand:GPF 0 "register_operand" "=w")
3031         (if_then_else:GPF
3032          (match_operator 1 "aarch64_comparison_operator"
3033           [(match_operand 2 "cc_register" "") (const_int 0)])
3034          (match_operand:GPF 3 "register_operand" "w")
3035          (match_operand:GPF 4 "register_operand" "w")))]
3036   "TARGET_FLOAT"
3037   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3038   [(set_attr "type" "fcsel")]
3039 )
3040
3041 (define_expand "mov<mode>cc"
3042   [(set (match_operand:ALLI 0 "register_operand" "")
3043         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3044                            (match_operand:ALLI 2 "register_operand" "")
3045                            (match_operand:ALLI 3 "register_operand" "")))]
3046   ""
3047   {
3048     enum rtx_code code = GET_CODE (operands[1]);
3049
3050     if (code == UNEQ || code == LTGT)
3051       FAIL;
3052
3053     if (!ccmp_cc_register (XEXP (operands[1], 0),
3054                            GET_MODE (XEXP (operands[1], 0))))
3055       {
3056         rtx ccreg;
3057         ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3058                                          XEXP (operands[1], 1));
3059         operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3060       }
3061   }
3062 )
3063
3064 (define_expand "mov<GPF:mode><GPI:mode>cc"
3065   [(set (match_operand:GPI 0 "register_operand" "")
3066         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3067                           (match_operand:GPF 2 "register_operand" "")
3068                           (match_operand:GPF 3 "register_operand" "")))]
3069   ""
3070   {
3071     rtx ccreg;
3072     enum rtx_code code = GET_CODE (operands[1]);
3073
3074     if (code == UNEQ || code == LTGT)
3075       FAIL;
3076
3077     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3078                                   XEXP (operands[1], 1));
3079     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3080   }
3081 )
3082
3083 (define_expand "mov<mode>cc"
3084   [(set (match_operand:GPF 0 "register_operand" "")
3085         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3086                           (match_operand:GPF 2 "register_operand" "")
3087                           (match_operand:GPF 3 "register_operand" "")))]
3088   ""
3089   {
3090     rtx ccreg;
3091     enum rtx_code code = GET_CODE (operands[1]);
3092
3093     if (code == UNEQ || code == LTGT)
3094       FAIL;
3095
3096     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3097                                   XEXP (operands[1], 1));
3098     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3099   }
3100 )
3101
3102
3103 ;; CRC32 instructions.
3104 (define_insn "aarch64_<crc_variant>"
3105   [(set (match_operand:SI 0 "register_operand" "=r")
3106         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3107                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3108          CRC))]
3109   "TARGET_CRC32"
3110   {
3111     if (GET_MODE_BITSIZE (GET_MODE (operands[2])) >= 64)
3112       return "<crc_variant>\\t%w0, %w1, %x2";
3113     else
3114       return "<crc_variant>\\t%w0, %w1, %w2";
3115   }
3116   [(set_attr "type" "crc")]
3117 )
3118
3119 (define_insn "*csinc2<mode>_insn"
3120   [(set (match_operand:GPI 0 "register_operand" "=r")
3121         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3122                   (match_operand:GPI 1 "register_operand" "r")))]
3123   ""
3124   "cinc\\t%<w>0, %<w>1, %m2"
3125   [(set_attr "type" "csel")]
3126 )
3127
3128 (define_insn "csinc3<mode>_insn"
3129   [(set (match_operand:GPI 0 "register_operand" "=r")
3130         (if_then_else:GPI
3131           (match_operand 1 "aarch64_comparison_operation" "")
3132           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3133                     (const_int 1))
3134           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3135   ""
3136   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3137   [(set_attr "type" "csel")]
3138 )
3139
3140 (define_insn "*csinv3<mode>_insn"
3141   [(set (match_operand:GPI 0 "register_operand" "=r")
3142         (if_then_else:GPI
3143           (match_operand 1 "aarch64_comparison_operation" "")
3144           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3145           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3146   ""
3147   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3148   [(set_attr "type" "csel")]
3149 )
3150
3151 (define_insn "csneg3_uxtw_insn"
3152   [(set (match_operand:DI 0 "register_operand" "=r")
3153         (zero_extend:DI
3154           (if_then_else:SI
3155             (match_operand 1 "aarch64_comparison_operation" "")
3156             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3157             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3158   ""
3159   "csneg\\t%w0, %w3, %w2, %M1"
3160   [(set_attr "type" "csel")]
3161 )
3162
3163 (define_insn "csneg3<mode>_insn"
3164   [(set (match_operand:GPI 0 "register_operand" "=r")
3165         (if_then_else:GPI
3166           (match_operand 1 "aarch64_comparison_operation" "")
3167           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3168           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3169   ""
3170   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3171   [(set_attr "type" "csel")]
3172 )
3173
3174 ;; -------------------------------------------------------------------
3175 ;; Logical operations
3176 ;; -------------------------------------------------------------------
3177
3178 (define_insn "<optab><mode>3"
3179   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3180         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3181                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3182   ""
3183   "@
3184   <logical>\\t%<w>0, %<w>1, %<w>2
3185   <logical>\\t%<w>0, %<w>1, %<w>2
3186   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3187   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3188    (set_attr "simd" "*,*,yes")]
3189 )
3190
3191 ;; zero_extend version of above
3192 (define_insn "*<optab>si3_uxtw"
3193   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3194         (zero_extend:DI
3195          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3196                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3197   ""
3198   "<logical>\\t%w0, %w1, %w2"
3199   [(set_attr "type" "logic_reg,logic_imm")]
3200 )
3201
3202 (define_insn "*and<mode>3_compare0"
3203   [(set (reg:CC_NZ CC_REGNUM)
3204         (compare:CC_NZ
3205          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3206                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3207          (const_int 0)))
3208    (set (match_operand:GPI 0 "register_operand" "=r,r")
3209         (and:GPI (match_dup 1) (match_dup 2)))]
3210   ""
3211   "ands\\t%<w>0, %<w>1, %<w>2"
3212   [(set_attr "type" "logics_reg,logics_imm")]
3213 )
3214
3215 ;; zero_extend version of above
3216 (define_insn "*andsi3_compare0_uxtw"
3217   [(set (reg:CC_NZ CC_REGNUM)
3218         (compare:CC_NZ
3219          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3220                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3221          (const_int 0)))
3222    (set (match_operand:DI 0 "register_operand" "=r,r")
3223         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3224   ""
3225   "ands\\t%w0, %w1, %w2"
3226   [(set_attr "type" "logics_reg,logics_imm")]
3227 )
3228
3229 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3230   [(set (reg:CC_NZ CC_REGNUM)
3231         (compare:CC_NZ
3232          (and:GPI (SHIFT:GPI
3233                    (match_operand:GPI 1 "register_operand" "r")
3234                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3235                   (match_operand:GPI 3 "register_operand" "r"))
3236          (const_int 0)))
3237    (set (match_operand:GPI 0 "register_operand" "=r")
3238         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3239   ""
3240   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3241   [(set_attr "type" "logics_shift_imm")]
3242 )
3243
3244 ;; zero_extend version of above
3245 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3246   [(set (reg:CC_NZ CC_REGNUM)
3247         (compare:CC_NZ
3248          (and:SI (SHIFT:SI
3249                   (match_operand:SI 1 "register_operand" "r")
3250                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3251                  (match_operand:SI 3 "register_operand" "r"))
3252          (const_int 0)))
3253    (set (match_operand:DI 0 "register_operand" "=r")
3254         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3255                                 (match_dup 3))))]
3256   ""
3257   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3258   [(set_attr "type" "logics_shift_imm")]
3259 )
3260
3261 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3262   [(set (match_operand:GPI 0 "register_operand" "=r")
3263         (LOGICAL:GPI (SHIFT:GPI
3264                       (match_operand:GPI 1 "register_operand" "r")
3265                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3266                      (match_operand:GPI 3 "register_operand" "r")))]
3267   ""
3268   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3269   [(set_attr "type" "logic_shift_imm")]
3270 )
3271
3272 (define_insn "*<optab>_rol<mode>3"
3273   [(set (match_operand:GPI 0 "register_operand" "=r")
3274         (LOGICAL:GPI (rotate:GPI
3275                       (match_operand:GPI 1 "register_operand" "r")
3276                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3277                      (match_operand:GPI 3 "register_operand" "r")))]
3278   ""
3279   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3280   [(set_attr "type" "logic_shift_imm")]
3281 )
3282
3283 ;; zero_extend versions of above
3284 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3285   [(set (match_operand:DI 0 "register_operand" "=r")
3286         (zero_extend:DI
3287          (LOGICAL:SI (SHIFT:SI
3288                       (match_operand:SI 1 "register_operand" "r")
3289                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3290                      (match_operand:SI 3 "register_operand" "r"))))]
3291   ""
3292   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3293   [(set_attr "type" "logic_shift_imm")]
3294 )
3295
3296 (define_insn "*<optab>_rolsi3_uxtw"
3297   [(set (match_operand:DI 0 "register_operand" "=r")
3298         (zero_extend:DI
3299          (LOGICAL:SI (rotate:SI
3300                       (match_operand:SI 1 "register_operand" "r")
3301                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3302                      (match_operand:SI 3 "register_operand" "r"))))]
3303   ""
3304   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3305   [(set_attr "type" "logic_shift_imm")]
3306 )
3307
3308 (define_insn "one_cmpl<mode>2"
3309   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3310         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3311   ""
3312   "@
3313   mvn\\t%<w>0, %<w>1
3314   mvn\\t%0.8b, %1.8b"
3315   [(set_attr "type" "logic_reg,neon_logic")
3316    (set_attr "simd" "*,yes")]
3317 )
3318
3319 (define_insn "*one_cmpl_<optab><mode>2"
3320   [(set (match_operand:GPI 0 "register_operand" "=r")
3321         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3322                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3323   ""
3324   "mvn\\t%<w>0, %<w>1, <shift> %2"
3325   [(set_attr "type" "logic_shift_imm")]
3326 )
3327
3328 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3329
3330 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3331   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3332         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3333                      (match_operand:GPI 2 "register_operand" "r,w")))]
3334   ""
3335   "@
3336   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3337   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3338   [(set_attr "type" "logic_reg,neon_logic")
3339    (set_attr "simd" "*,yes")]
3340 )
3341
3342 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3343   [(set (match_operand:DI 0 "register_operand" "=r")
3344         (zero_extend:DI
3345           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3346                        (match_operand:SI 2 "register_operand" "r"))))]
3347   ""
3348   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3349   [(set_attr "type" "logic_reg")]
3350 )
3351
3352 (define_insn "*xor_one_cmplsidi3_ze"
3353   [(set (match_operand:DI 0 "register_operand" "=r")
3354         (zero_extend:DI
3355           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3356                           (match_operand:SI 2 "register_operand" "r")))))]
3357   ""
3358   "eon\\t%w0, %w1, %w2"
3359   [(set_attr "type" "logic_reg")]
3360 )
3361
3362 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3363 ;; eon does not operate on SIMD registers so the vector variant must be split.
3364 (define_insn_and_split "*xor_one_cmpl<mode>3"
3365   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3366         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3367                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3368   ""
3369   "@
3370   eon\\t%<w>0, %<w>1, %<w>2
3371   #"
3372   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3373   [(set (match_operand:GPI 0 "register_operand" "=w")
3374         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3375                  (match_operand:GPI 2 "register_operand" "w")))
3376    (set (match_dup 0) (not:GPI (match_dup 0)))]
3377   ""
3378   [(set_attr "type" "logic_reg,multiple")
3379    (set_attr "simd" "*,yes")]
3380 )
3381
3382 (define_insn "*and_one_cmpl<mode>3_compare0"
3383   [(set (reg:CC_NZ CC_REGNUM)
3384         (compare:CC_NZ
3385          (and:GPI (not:GPI
3386                    (match_operand:GPI 1 "register_operand" "r"))
3387                   (match_operand:GPI 2 "register_operand" "r"))
3388          (const_int 0)))
3389    (set (match_operand:GPI 0 "register_operand" "=r")
3390         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3391   ""
3392   "bics\\t%<w>0, %<w>2, %<w>1"
3393   [(set_attr "type" "logics_reg")]
3394 )
3395
3396 ;; zero_extend version of above
3397 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3398   [(set (reg:CC_NZ CC_REGNUM)
3399         (compare:CC_NZ
3400          (and:SI (not:SI
3401                   (match_operand:SI 1 "register_operand" "r"))
3402                  (match_operand:SI 2 "register_operand" "r"))
3403          (const_int 0)))
3404    (set (match_operand:DI 0 "register_operand" "=r")
3405         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3406   ""
3407   "bics\\t%w0, %w2, %w1"
3408   [(set_attr "type" "logics_reg")]
3409 )
3410
3411 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3412   [(set (reg:CC_NZ CC_REGNUM)
3413     (compare:CC_NZ
3414      (and:GPI (not:GPI
3415            (match_operand:GPI 0 "register_operand" "r"))
3416           (match_operand:GPI 1 "register_operand" "r"))
3417      (const_int 0)))]
3418   ""
3419   "bics\\t<w>zr, %<w>1, %<w>0"
3420   [(set_attr "type" "logics_reg")]
3421 )
3422
3423 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3424   [(set (match_operand:GPI 0 "register_operand" "=r")
3425         (LOGICAL:GPI (not:GPI
3426                       (SHIFT:GPI
3427                        (match_operand:GPI 1 "register_operand" "r")
3428                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3429                      (match_operand:GPI 3 "register_operand" "r")))]
3430   ""
3431   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3432   [(set_attr "type" "logic_shift_imm")]
3433 )
3434
3435 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3436   [(set (match_operand:GPI 0 "register_operand" "=r")
3437         (not:GPI (xor:GPI
3438                       (SHIFT:GPI
3439                        (match_operand:GPI 1 "register_operand" "r")
3440                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3441                      (match_operand:GPI 3 "register_operand" "r"))))]
3442   ""
3443   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3444   [(set_attr "type" "logic_shift_imm")]
3445 )
3446
3447 ;; Zero-extend version of the above.
3448 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3449   [(set (match_operand:DI 0 "register_operand" "=r")
3450         (zero_extend:DI
3451           (not:SI (xor:SI
3452                     (SHIFT:SI
3453                       (match_operand:SI 1 "register_operand" "r")
3454                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3455                     (match_operand:SI 3 "register_operand" "r")))))]
3456   ""
3457   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3458   [(set_attr "type" "logic_shift_imm")]
3459 )
3460
3461 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3462   [(set (reg:CC_NZ CC_REGNUM)
3463         (compare:CC_NZ
3464          (and:GPI (not:GPI
3465                    (SHIFT:GPI
3466                     (match_operand:GPI 1 "register_operand" "r")
3467                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3468                   (match_operand:GPI 3 "register_operand" "r"))
3469          (const_int 0)))
3470    (set (match_operand:GPI 0 "register_operand" "=r")
3471         (and:GPI (not:GPI
3472                   (SHIFT:GPI
3473                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3474   ""
3475   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3476   [(set_attr "type" "logics_shift_imm")]
3477 )
3478
3479 ;; zero_extend version of above
3480 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3481   [(set (reg:CC_NZ CC_REGNUM)
3482         (compare:CC_NZ
3483          (and:SI (not:SI
3484                   (SHIFT:SI
3485                    (match_operand:SI 1 "register_operand" "r")
3486                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3487                  (match_operand:SI 3 "register_operand" "r"))
3488          (const_int 0)))
3489    (set (match_operand:DI 0 "register_operand" "=r")
3490         (zero_extend:DI (and:SI
3491                          (not:SI
3492                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3493   ""
3494   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3495   [(set_attr "type" "logics_shift_imm")]
3496 )
3497
3498 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3499   [(set (reg:CC_NZ CC_REGNUM)
3500     (compare:CC_NZ
3501      (and:GPI (not:GPI
3502            (SHIFT:GPI
3503             (match_operand:GPI 0 "register_operand" "r")
3504             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3505           (match_operand:GPI 2 "register_operand" "r"))
3506      (const_int 0)))]
3507   ""
3508   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3509   [(set_attr "type" "logics_shift_imm")]
3510 )
3511
3512 (define_insn "clz<mode>2"
3513   [(set (match_operand:GPI 0 "register_operand" "=r")
3514         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3515   ""
3516   "clz\\t%<w>0, %<w>1"
3517   [(set_attr "type" "clz")]
3518 )
3519
3520 (define_expand "ffs<mode>2"
3521   [(match_operand:GPI 0 "register_operand")
3522    (match_operand:GPI 1 "register_operand")]
3523   ""
3524   {
3525     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3526     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3527
3528     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3529     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3530     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3531     DONE;
3532   }
3533 )
3534
3535 (define_insn "clrsb<mode>2"
3536   [(set (match_operand:GPI 0 "register_operand" "=r")
3537         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
3538   ""
3539   "cls\\t%<w>0, %<w>1"
3540   [(set_attr "type" "clz")]
3541 )
3542
3543 (define_insn "rbit<mode>2"
3544   [(set (match_operand:GPI 0 "register_operand" "=r")
3545         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
3546   ""
3547   "rbit\\t%<w>0, %<w>1"
3548   [(set_attr "type" "rbit")]
3549 )
3550
3551 (define_expand "ctz<mode>2"
3552   [(match_operand:GPI 0 "register_operand")
3553    (match_operand:GPI 1 "register_operand")]
3554   ""
3555   {
3556     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3557     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3558     DONE;
3559   }
3560 )
3561
3562 (define_insn "*and<mode>3nr_compare0"
3563   [(set (reg:CC_NZ CC_REGNUM)
3564         (compare:CC_NZ
3565          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
3566                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
3567          (const_int 0)))]
3568   ""
3569   "tst\\t%<w>0, %<w>1"
3570   [(set_attr "type" "logics_reg,logics_imm")]
3571 )
3572
3573 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
3574   [(set (reg:CC_NZ CC_REGNUM)
3575         (compare:CC_NZ
3576          (and:GPI (SHIFT:GPI
3577                    (match_operand:GPI 0 "register_operand" "r")
3578                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3579                   (match_operand:GPI 2 "register_operand" "r"))
3580         (const_int 0)))]
3581   ""
3582   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
3583   [(set_attr "type" "logics_shift_imm")]
3584 )
3585
3586 ;; -------------------------------------------------------------------
3587 ;; Shifts
3588 ;; -------------------------------------------------------------------
3589
3590 (define_expand "<optab><mode>3"
3591   [(set (match_operand:GPI 0 "register_operand")
3592         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
3593                     (match_operand:QI 2 "nonmemory_operand")))]
3594   ""
3595   {
3596     if (CONST_INT_P (operands[2]))
3597       {
3598         operands[2] = GEN_INT (INTVAL (operands[2])
3599                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3600
3601         if (operands[2] == const0_rtx)
3602           {
3603             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3604             DONE;
3605           }
3606       }
3607   }
3608 )
3609
3610 (define_expand "ashl<mode>3"
3611   [(set (match_operand:SHORT 0 "register_operand")
3612         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
3613                       (match_operand:QI 2 "nonmemory_operand")))]
3614   ""
3615   {
3616     if (CONST_INT_P (operands[2]))
3617       {
3618         operands[2] = GEN_INT (INTVAL (operands[2])
3619                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3620
3621         if (operands[2] == const0_rtx)
3622           {
3623             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3624             DONE;
3625           }
3626       }
3627     else
3628       FAIL;
3629   }
3630 )
3631
3632 (define_expand "rotr<mode>3"
3633   [(set (match_operand:GPI 0 "register_operand")
3634         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3635                       (match_operand:QI 2 "nonmemory_operand")))]
3636   ""
3637   {
3638     if (CONST_INT_P (operands[2]))
3639       {
3640         operands[2] = GEN_INT (INTVAL (operands[2])
3641                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3642
3643         if (operands[2] == const0_rtx)
3644           {
3645             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3646             DONE;
3647           }
3648       }
3649   }
3650 )
3651
3652 (define_expand "rotl<mode>3"
3653   [(set (match_operand:GPI 0 "register_operand")
3654         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3655                       (match_operand:QI 2 "nonmemory_operand")))]
3656   ""
3657   {
3658     /* (SZ - cnt) % SZ == -cnt % SZ */
3659     if (CONST_INT_P (operands[2]))
3660       {
3661         operands[2] = GEN_INT ((-INTVAL (operands[2]))
3662                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3663         if (operands[2] == const0_rtx)
3664           {
3665             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3666             DONE;
3667           }
3668       }
3669     else
3670       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
3671                                         NULL_RTX, 1);
3672   }
3673 )
3674
3675 ;; Logical left shift using SISD or Integer instruction
3676 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3677   [(set (match_operand:GPI 0 "register_operand" "=r,w,w")
3678         (ashift:GPI
3679           (match_operand:GPI 1 "register_operand" "r,w,w")
3680           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "rUs<cmode>,Us<cmode>,w")))]
3681   ""
3682   "@
3683    lsl\t%<w>0, %<w>1, %<w>2
3684    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3685    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
3686   [(set_attr "simd" "no,yes,yes")
3687    (set_attr "type" "shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
3688 )
3689
3690 ;; Logical right shift using SISD or Integer instruction
3691 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3692   [(set (match_operand:GPI 0 "register_operand" "=r,w,&w,&w")
3693         (lshiftrt:GPI
3694           (match_operand:GPI 1 "register_operand" "r,w,w,w")
3695           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "rUs<cmode>,Us<cmode>,w,0")))]
3696   ""
3697   "@
3698    lsr\t%<w>0, %<w>1, %<w>2
3699    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3700    #
3701    #"
3702   [(set_attr "simd" "no,yes,yes,yes")
3703    (set_attr "type" "shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
3704 )
3705
3706 (define_split
3707   [(set (match_operand:DI 0 "aarch64_simd_register")
3708         (lshiftrt:DI
3709            (match_operand:DI 1 "aarch64_simd_register")
3710            (match_operand:QI 2 "aarch64_simd_register")))]
3711   "TARGET_SIMD && reload_completed"
3712   [(set (match_dup 3)
3713         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3714    (set (match_dup 0)
3715         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
3716   {
3717     operands[3] = gen_lowpart (QImode, operands[0]);
3718   }
3719 )
3720
3721 (define_split
3722   [(set (match_operand:SI 0 "aarch64_simd_register")
3723         (lshiftrt:SI
3724            (match_operand:SI 1 "aarch64_simd_register")
3725            (match_operand:QI 2 "aarch64_simd_register")))]
3726   "TARGET_SIMD && reload_completed"
3727   [(set (match_dup 3)
3728         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3729    (set (match_dup 0)
3730         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
3731   {
3732     operands[3] = gen_lowpart (QImode, operands[0]);
3733   }
3734 )
3735
3736 ;; Arithmetic right shift using SISD or Integer instruction
3737 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3738   [(set (match_operand:GPI 0 "register_operand" "=r,w,&w,&w")
3739         (ashiftrt:GPI
3740           (match_operand:GPI 1 "register_operand" "r,w,w,w")
3741           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "rUs<cmode>,Us<cmode>,w,0")))]
3742   ""
3743   "@
3744    asr\t%<w>0, %<w>1, %<w>2
3745    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3746    #
3747    #"
3748   [(set_attr "simd" "no,yes,yes,yes")
3749    (set_attr "type" "shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
3750 )
3751
3752 (define_split
3753   [(set (match_operand:DI 0 "aarch64_simd_register")
3754         (ashiftrt:DI
3755            (match_operand:DI 1 "aarch64_simd_register")
3756            (match_operand:QI 2 "aarch64_simd_register")))]
3757   "TARGET_SIMD && reload_completed"
3758   [(set (match_dup 3)
3759         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3760    (set (match_dup 0)
3761         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
3762 {
3763   operands[3] = gen_lowpart (QImode, operands[0]);
3764 }
3765 )
3766
3767 (define_split
3768   [(set (match_operand:SI 0 "aarch64_simd_register")
3769         (ashiftrt:SI
3770            (match_operand:SI 1 "aarch64_simd_register")
3771            (match_operand:QI 2 "aarch64_simd_register")))]
3772   "TARGET_SIMD && reload_completed"
3773   [(set (match_dup 3)
3774         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3775    (set (match_dup 0)
3776         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
3777 {
3778   operands[3] = gen_lowpart (QImode, operands[0]);
3779 }
3780 )
3781
3782 (define_insn "*aarch64_sisd_ushl"
3783   [(set (match_operand:DI 0 "register_operand" "=w")
3784         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3785                     (match_operand:QI 2 "register_operand" "w")]
3786                    UNSPEC_SISD_USHL))]
3787   "TARGET_SIMD"
3788   "ushl\t%d0, %d1, %d2"
3789   [(set_attr "simd" "yes")
3790    (set_attr "type" "neon_shift_reg")]
3791 )
3792
3793 (define_insn "*aarch64_ushl_2s"
3794   [(set (match_operand:SI 0 "register_operand" "=w")
3795         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3796                     (match_operand:QI 2 "register_operand" "w")]
3797                    UNSPEC_USHL_2S))]
3798   "TARGET_SIMD"
3799   "ushl\t%0.2s, %1.2s, %2.2s"
3800   [(set_attr "simd" "yes")
3801    (set_attr "type" "neon_shift_reg")]
3802 )
3803
3804 (define_insn "*aarch64_sisd_sshl"
3805   [(set (match_operand:DI 0 "register_operand" "=w")
3806         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3807                     (match_operand:QI 2 "register_operand" "w")]
3808                    UNSPEC_SISD_SSHL))]
3809   "TARGET_SIMD"
3810   "sshl\t%d0, %d1, %d2"
3811   [(set_attr "simd" "yes")
3812    (set_attr "type" "neon_shift_reg")]
3813 )
3814
3815 (define_insn "*aarch64_sshl_2s"
3816   [(set (match_operand:SI 0 "register_operand" "=w")
3817         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3818                     (match_operand:QI 2 "register_operand" "w")]
3819                    UNSPEC_SSHL_2S))]
3820   "TARGET_SIMD"
3821   "sshl\t%0.2s, %1.2s, %2.2s"
3822   [(set_attr "simd" "yes")
3823    (set_attr "type" "neon_shift_reg")]
3824 )
3825
3826 (define_insn "*aarch64_sisd_neg_qi"
3827   [(set (match_operand:QI 0 "register_operand" "=w")
3828         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
3829                    UNSPEC_SISD_NEG))]
3830   "TARGET_SIMD"
3831   "neg\t%d0, %d1"
3832   [(set_attr "simd" "yes")
3833    (set_attr "type" "neon_neg")]
3834 )
3835
3836 ;; Rotate right
3837 (define_insn "*ror<mode>3_insn"
3838   [(set (match_operand:GPI 0 "register_operand" "=r,r")
3839      (rotatert:GPI
3840        (match_operand:GPI 1 "register_operand" "r,r")
3841        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "r,Us<cmode>")))]
3842   ""
3843   "ror\\t%<w>0, %<w>1, %<w>2"
3844   [(set_attr "type" "shift_reg, rotate_imm")]
3845 )
3846
3847 ;; zero_extend version of above
3848 (define_insn "*<optab>si3_insn_uxtw"
3849   [(set (match_operand:DI 0 "register_operand" "=r")
3850         (zero_extend:DI (SHIFT:SI
3851          (match_operand:SI 1 "register_operand" "r")
3852          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "rUss"))))]
3853   ""
3854   "<shift>\\t%w0, %w1, %w2"
3855   [(set_attr "type" "shift_reg")]
3856 )
3857
3858 (define_insn "*<optab><mode>3_insn"
3859   [(set (match_operand:SHORT 0 "register_operand" "=r")
3860         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
3861                       (match_operand 2 "const_int_operand" "n")))]
3862   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
3863 {
3864   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
3865   return "<bfshift>\t%w0, %w1, %2, %3";
3866 }
3867   [(set_attr "type" "bfm")]
3868 )
3869
3870 (define_insn "*extr<mode>5_insn"
3871   [(set (match_operand:GPI 0 "register_operand" "=r")
3872         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3873                              (match_operand 3 "const_int_operand" "n"))
3874                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
3875                                (match_operand 4 "const_int_operand" "n"))))]
3876   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
3877    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
3878   "extr\\t%<w>0, %<w>1, %<w>2, %4"
3879   [(set_attr "type" "shift_imm")]
3880 )
3881
3882 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
3883 ;; so we have to match both orderings.
3884 (define_insn "*extr<mode>5_insn_alt"
3885   [(set (match_operand:GPI 0 "register_operand" "=r")
3886         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
3887                                 (match_operand 4 "const_int_operand" "n"))
3888                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3889                               (match_operand 3 "const_int_operand" "n"))))]
3890   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
3891    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
3892        == GET_MODE_BITSIZE (<MODE>mode))"
3893   "extr\\t%<w>0, %<w>1, %<w>2, %4"
3894   [(set_attr "type" "shift_imm")]
3895 )
3896
3897 ;; zero_extend version of the above
3898 (define_insn "*extrsi5_insn_uxtw"
3899   [(set (match_operand:DI 0 "register_operand" "=r")
3900         (zero_extend:DI
3901          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
3902                             (match_operand 3 "const_int_operand" "n"))
3903                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
3904                               (match_operand 4 "const_int_operand" "n")))))]
3905   "UINTVAL (operands[3]) < 32 &&
3906    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
3907   "extr\\t%w0, %w1, %w2, %4"
3908   [(set_attr "type" "shift_imm")]
3909 )
3910
3911 (define_insn "*extrsi5_insn_uxtw_alt"
3912   [(set (match_operand:DI 0 "register_operand" "=r")
3913         (zero_extend:DI
3914          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
3915                                (match_operand 4 "const_int_operand" "n"))
3916                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
3917                             (match_operand 3 "const_int_operand" "n")))))]
3918   "UINTVAL (operands[3]) < 32 &&
3919    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
3920   "extr\\t%w0, %w1, %w2, %4"
3921   [(set_attr "type" "shift_imm")]
3922 )
3923
3924 (define_insn "*ror<mode>3_insn"
3925   [(set (match_operand:GPI 0 "register_operand" "=r")
3926         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
3927                     (match_operand 2 "const_int_operand" "n")))]
3928   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
3929 {
3930   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
3931   return "ror\\t%<w>0, %<w>1, %3";
3932 }
3933   [(set_attr "type" "rotate_imm")]
3934 )
3935
3936 ;; zero_extend version of the above
3937 (define_insn "*rorsi3_insn_uxtw"
3938   [(set (match_operand:DI 0 "register_operand" "=r")
3939         (zero_extend:DI
3940          (rotate:SI (match_operand:SI 1 "register_operand" "r")
3941                     (match_operand 2 "const_int_operand" "n"))))]
3942   "UINTVAL (operands[2]) < 32"
3943 {
3944   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
3945   return "ror\\t%w0, %w1, %3";
3946 }
3947   [(set_attr "type" "rotate_imm")]
3948 )
3949
3950 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
3951   [(set (match_operand:GPI 0 "register_operand" "=r")
3952         (ANY_EXTEND:GPI
3953          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
3954                        (match_operand 2 "const_int_operand" "n"))))]
3955   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3956 {
3957   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3958   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3959 }
3960   [(set_attr "type" "bfm")]
3961 )
3962
3963 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
3964   [(set (match_operand:GPI 0 "register_operand" "=r")
3965         (zero_extend:GPI
3966          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
3967                          (match_operand 2 "const_int_operand" "n"))))]
3968   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3969 {
3970   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3971   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3972 }
3973   [(set_attr "type" "bfm")]
3974 )
3975
3976 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
3977   [(set (match_operand:GPI 0 "register_operand" "=r")
3978         (sign_extend:GPI
3979          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
3980                          (match_operand 2 "const_int_operand" "n"))))]
3981   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3982 {
3983   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3984   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3985 }
3986   [(set_attr "type" "bfm")]
3987 )
3988
3989 ;; -------------------------------------------------------------------
3990 ;; Bitfields
3991 ;; -------------------------------------------------------------------
3992
3993 (define_expand "<optab>"
3994   [(set (match_operand:DI 0 "register_operand" "=r")
3995         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand" "r")
3996                         (match_operand 2 "const_int_operand" "n")
3997                         (match_operand 3 "const_int_operand" "n")))]
3998   ""
3999   ""
4000 )
4001
4002 (define_insn "*<optab><mode>"
4003   [(set (match_operand:GPI 0 "register_operand" "=r")
4004         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4005                          (match_operand 2 "const_int_operand" "n")
4006                          (match_operand 3 "const_int_operand" "n")))]
4007   ""
4008   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4009   [(set_attr "type" "bfm")]
4010 )
4011
4012 ;; Bitfield Insert (insv)
4013 (define_expand "insv<mode>"
4014   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4015                           (match_operand 1 "const_int_operand")
4016                           (match_operand 2 "const_int_operand"))
4017         (match_operand:GPI 3 "general_operand"))]
4018   ""
4019 {
4020   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4021   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4022   rtx value = operands[3];
4023
4024   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4025     FAIL;
4026
4027   if (CONST_INT_P (value))
4028     {
4029       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4030
4031       /* Prefer AND/OR for inserting all zeros or all ones.  */
4032       if ((UINTVAL (value) & mask) == 0
4033            || (UINTVAL (value) & mask) == mask)
4034         FAIL;
4035
4036       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
4037       if (width == 16 && (pos % 16) == 0)
4038         DONE;
4039     }
4040   operands[3] = force_reg (<MODE>mode, value);
4041 })
4042
4043 (define_insn "*insv_reg<mode>"
4044   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4045                           (match_operand 1 "const_int_operand" "n")
4046                           (match_operand 2 "const_int_operand" "n"))
4047         (match_operand:GPI 3 "register_operand" "r"))]
4048   "!(UINTVAL (operands[1]) == 0
4049      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4050          > GET_MODE_BITSIZE (<MODE>mode)))"
4051   "bfi\\t%<w>0, %<w>3, %2, %1"
4052   [(set_attr "type" "bfm")]
4053 )
4054
4055 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4056   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4057                           (match_operand 1 "const_int_operand" "n")
4058                           (match_operand 2 "const_int_operand" "n"))
4059         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
4060   "UINTVAL (operands[1]) <= <ALLX:sizen>"
4061   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4062   [(set_attr "type" "bfm")]
4063 )
4064
4065 (define_insn "*extr_insv_lower_reg<mode>"
4066   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4067                           (match_operand 1 "const_int_operand" "n")
4068                           (const_int 0))
4069         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4070                           (match_dup 1)
4071                           (match_operand 3 "const_int_operand" "n")))]
4072   "!(UINTVAL (operands[1]) == 0
4073      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4074          > GET_MODE_BITSIZE (<MODE>mode)))"
4075   "bfxil\\t%<w>0, %<w>2, %3, %1"
4076   [(set_attr "type" "bfm")]
4077 )
4078
4079 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4080   [(set (match_operand:GPI 0 "register_operand" "=r")
4081         (ashift:GPI (ANY_EXTEND:GPI
4082                      (match_operand:ALLX 1 "register_operand" "r"))
4083                     (match_operand 2 "const_int_operand" "n")))]
4084   "UINTVAL (operands[2]) < <GPI:sizen>"
4085 {
4086   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4087               ? GEN_INT (<ALLX:sizen>)
4088               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4089   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4090 }
4091   [(set_attr "type" "bfm")]
4092 )
4093
4094 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4095
4096 (define_insn "*andim_ashift<mode>_bfiz"
4097   [(set (match_operand:GPI 0 "register_operand" "=r")
4098         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4099                              (match_operand 2 "const_int_operand" "n"))
4100                  (match_operand 3 "const_int_operand" "n")))]
4101   "(INTVAL (operands[2]) < (<GPI:sizen>))
4102    && exact_log2 ((INTVAL (operands[3]) >> INTVAL (operands[2])) + 1) >= 0
4103    && (INTVAL (operands[3]) & ((1 << INTVAL (operands[2])) - 1)) == 0"
4104   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4105   [(set_attr "type" "bfm")]
4106 )
4107
4108 (define_insn "bswap<mode>2"
4109   [(set (match_operand:GPI 0 "register_operand" "=r")
4110         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4111   ""
4112   "rev\\t%<w>0, %<w>1"
4113   [(set_attr "type" "rev")]
4114 )
4115
4116 (define_insn "bswaphi2"
4117   [(set (match_operand:HI 0 "register_operand" "=r")
4118         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4119   ""
4120   "rev16\\t%w0, %w1"
4121   [(set_attr "type" "rev")]
4122 )
4123
4124 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4125 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4126 ;; each valid permutation.
4127
4128 (define_insn "rev16<mode>2"
4129   [(set (match_operand:GPI 0 "register_operand" "=r")
4130         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4131                                       (const_int 8))
4132                           (match_operand:GPI 3 "const_int_operand" "n"))
4133                  (and:GPI (lshiftrt:GPI (match_dup 1)
4134                                         (const_int 8))
4135                           (match_operand:GPI 2 "const_int_operand" "n"))))]
4136   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4137    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4138   "rev16\\t%<w>0, %<w>1"
4139   [(set_attr "type" "rev")]
4140 )
4141
4142 (define_insn "rev16<mode>2_alt"
4143   [(set (match_operand:GPI 0 "register_operand" "=r")
4144         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4145                                         (const_int 8))
4146                           (match_operand:GPI 2 "const_int_operand" "n"))
4147                  (and:GPI (ashift:GPI (match_dup 1)
4148                                       (const_int 8))
4149                           (match_operand:GPI 3 "const_int_operand" "n"))))]
4150   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4151    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4152   "rev16\\t%<w>0, %<w>1"
4153   [(set_attr "type" "rev")]
4154 )
4155
4156 ;; zero_extend version of above
4157 (define_insn "*bswapsi2_uxtw"
4158   [(set (match_operand:DI 0 "register_operand" "=r")
4159         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4160   ""
4161   "rev\\t%w0, %w1"
4162   [(set_attr "type" "rev")]
4163 )
4164
4165 ;; -------------------------------------------------------------------
4166 ;; Floating-point intrinsics
4167 ;; -------------------------------------------------------------------
4168
4169 ;; frint floating-point round to integral standard patterns.
4170 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4171
4172 (define_insn "<frint_pattern><mode>2"
4173   [(set (match_operand:GPF 0 "register_operand" "=w")
4174         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4175          FRINT))]
4176   "TARGET_FLOAT"
4177   "frint<frint_suffix>\\t%<s>0, %<s>1"
4178   [(set_attr "type" "f_rint<s>")]
4179 )
4180
4181 ;; frcvt floating-point round to integer and convert standard patterns.
4182 ;; Expands to lbtrunc, lceil, lfloor, lround.
4183 (define_insn "l<fcvt_pattern><su_optab><GPF:mode><GPI:mode>2"
4184   [(set (match_operand:GPI 0 "register_operand" "=r")
4185         (FIXUORS:GPI (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4186                       FCVT)))]
4187   "TARGET_FLOAT"
4188   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF:s>1"
4189   [(set_attr "type" "f_cvtf2i")]
4190 )
4191
4192 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4193   [(set (match_operand:GPI 0 "register_operand" "=r")
4194         (FIXUORS:GPI
4195           (mult:GPF
4196             (match_operand:GPF 1 "register_operand" "w")
4197             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4198   "TARGET_FLOAT
4199    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4200                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4201   {
4202     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4203     char buf[64];
4204     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4205     output_asm_insn (buf, operands);
4206     return "";
4207   }
4208   [(set_attr "type" "f_cvtf2i")]
4209 )
4210
4211 ;; fma - no throw
4212
4213 (define_insn "fma<mode>4"
4214   [(set (match_operand:GPF 0 "register_operand" "=w")
4215         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4216                  (match_operand:GPF 2 "register_operand" "w")
4217                  (match_operand:GPF 3 "register_operand" "w")))]
4218   "TARGET_FLOAT"
4219   "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4220   [(set_attr "type" "fmac<s>")]
4221 )
4222
4223 (define_insn "fnma<mode>4"
4224   [(set (match_operand:GPF 0 "register_operand" "=w")
4225         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4226                  (match_operand:GPF 2 "register_operand" "w")
4227                  (match_operand:GPF 3 "register_operand" "w")))]
4228   "TARGET_FLOAT"
4229   "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4230   [(set_attr "type" "fmac<s>")]
4231 )
4232
4233 (define_insn "fms<mode>4"
4234   [(set (match_operand:GPF 0 "register_operand" "=w")
4235         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4236                  (match_operand:GPF 2 "register_operand" "w")
4237                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4238   "TARGET_FLOAT"
4239   "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4240   [(set_attr "type" "fmac<s>")]
4241 )
4242
4243 (define_insn "fnms<mode>4"
4244   [(set (match_operand:GPF 0 "register_operand" "=w")
4245         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4246                  (match_operand:GPF 2 "register_operand" "w")
4247                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4248   "TARGET_FLOAT"
4249   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4250   [(set_attr "type" "fmac<s>")]
4251 )
4252
4253 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4254 (define_insn "*fnmadd<mode>4"
4255   [(set (match_operand:GPF 0 "register_operand" "=w")
4256         (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4257                           (match_operand:GPF 2 "register_operand" "w")
4258                           (match_operand:GPF 3 "register_operand" "w"))))]
4259   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
4260   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4261   [(set_attr "type" "fmac<s>")]
4262 )
4263
4264 ;; -------------------------------------------------------------------
4265 ;; Floating-point conversions
4266 ;; -------------------------------------------------------------------
4267
4268 (define_insn "extendsfdf2"
4269   [(set (match_operand:DF 0 "register_operand" "=w")
4270         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
4271   "TARGET_FLOAT"
4272   "fcvt\\t%d0, %s1"
4273   [(set_attr "type" "f_cvt")]
4274 )
4275
4276 (define_insn "extendhfsf2"
4277   [(set (match_operand:SF 0 "register_operand" "=w")
4278         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
4279   "TARGET_FLOAT"
4280   "fcvt\\t%s0, %h1"
4281   [(set_attr "type" "f_cvt")]
4282 )
4283
4284 (define_insn "extendhfdf2"
4285   [(set (match_operand:DF 0 "register_operand" "=w")
4286         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
4287   "TARGET_FLOAT"
4288   "fcvt\\t%d0, %h1"
4289   [(set_attr "type" "f_cvt")]
4290 )
4291
4292 (define_insn "truncdfsf2"
4293   [(set (match_operand:SF 0 "register_operand" "=w")
4294         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
4295   "TARGET_FLOAT"
4296   "fcvt\\t%s0, %d1"
4297   [(set_attr "type" "f_cvt")]
4298 )
4299
4300 (define_insn "truncsfhf2"
4301   [(set (match_operand:HF 0 "register_operand" "=w")
4302         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
4303   "TARGET_FLOAT"
4304   "fcvt\\t%h0, %s1"
4305   [(set_attr "type" "f_cvt")]
4306 )
4307
4308 (define_insn "truncdfhf2"
4309   [(set (match_operand:HF 0 "register_operand" "=w")
4310         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
4311   "TARGET_FLOAT"
4312   "fcvt\\t%h0, %d1"
4313   [(set_attr "type" "f_cvt")]
4314 )
4315
4316 (define_insn "fix_trunc<GPF:mode><GPI:mode>2"
4317   [(set (match_operand:GPI 0 "register_operand" "=r")
4318         (fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4319   "TARGET_FLOAT"
4320   "fcvtzs\\t%<GPI:w>0, %<GPF:s>1"
4321   [(set_attr "type" "f_cvtf2i")]
4322 )
4323
4324 (define_insn "fixuns_trunc<GPF:mode><GPI:mode>2"
4325   [(set (match_operand:GPI 0 "register_operand" "=r")
4326         (unsigned_fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4327   "TARGET_FLOAT"
4328   "fcvtzu\\t%<GPI:w>0, %<GPF:s>1"
4329   [(set_attr "type" "f_cvtf2i")]
4330 )
4331
4332 (define_insn "<optab><fcvt_target><GPF:mode>2"
4333   [(set (match_operand:GPF 0 "register_operand" "=w,w")
4334         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
4335   "TARGET_FLOAT"
4336   "@
4337    <su_optab>cvtf\t%<GPF:s>0, %<s>1
4338    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
4339   [(set_attr "simd" "yes,no")
4340    (set_attr "fp" "no,yes")
4341    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
4342 )
4343
4344 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
4345   [(set (match_operand:GPF 0 "register_operand" "=w")
4346         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
4347   "TARGET_FLOAT"
4348   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
4349   [(set_attr "type" "f_cvti2f")]
4350 )
4351
4352 ;; -------------------------------------------------------------------
4353 ;; Floating-point arithmetic
4354 ;; -------------------------------------------------------------------
4355
4356 (define_insn "add<mode>3"
4357   [(set (match_operand:GPF 0 "register_operand" "=w")
4358         (plus:GPF
4359          (match_operand:GPF 1 "register_operand" "w")
4360          (match_operand:GPF 2 "register_operand" "w")))]
4361   "TARGET_FLOAT"
4362   "fadd\\t%<s>0, %<s>1, %<s>2"
4363   [(set_attr "type" "fadd<s>")]
4364 )
4365
4366 (define_insn "sub<mode>3"
4367   [(set (match_operand:GPF 0 "register_operand" "=w")
4368         (minus:GPF
4369          (match_operand:GPF 1 "register_operand" "w")
4370          (match_operand:GPF 2 "register_operand" "w")))]
4371   "TARGET_FLOAT"
4372   "fsub\\t%<s>0, %<s>1, %<s>2"
4373   [(set_attr "type" "fadd<s>")]
4374 )
4375
4376 (define_insn "mul<mode>3"
4377   [(set (match_operand:GPF 0 "register_operand" "=w")
4378         (mult:GPF
4379          (match_operand:GPF 1 "register_operand" "w")
4380          (match_operand:GPF 2 "register_operand" "w")))]
4381   "TARGET_FLOAT"
4382   "fmul\\t%<s>0, %<s>1, %<s>2"
4383   [(set_attr "type" "fmul<s>")]
4384 )
4385
4386 (define_insn "*fnmul<mode>3"
4387   [(set (match_operand:GPF 0 "register_operand" "=w")
4388         (mult:GPF
4389                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4390                  (match_operand:GPF 2 "register_operand" "w")))]
4391   "TARGET_FLOAT && !flag_rounding_math"
4392   "fnmul\\t%<s>0, %<s>1, %<s>2"
4393   [(set_attr "type" "fmul<s>")]
4394 )
4395
4396 (define_insn "*fnmul<mode>3"
4397   [(set (match_operand:GPF 0 "register_operand" "=w")
4398         (neg:GPF (mult:GPF
4399                  (match_operand:GPF 1 "register_operand" "w")
4400                  (match_operand:GPF 2 "register_operand" "w"))))]
4401   "TARGET_FLOAT"
4402   "fnmul\\t%<s>0, %<s>1, %<s>2"
4403   [(set_attr "type" "fmul<s>")]
4404 )
4405
4406 (define_insn "div<mode>3"
4407   [(set (match_operand:GPF 0 "register_operand" "=w")
4408         (div:GPF
4409          (match_operand:GPF 1 "register_operand" "w")
4410          (match_operand:GPF 2 "register_operand" "w")))]
4411   "TARGET_FLOAT"
4412   "fdiv\\t%<s>0, %<s>1, %<s>2"
4413   [(set_attr "type" "fdiv<s>")]
4414 )
4415
4416 (define_insn "neg<mode>2"
4417   [(set (match_operand:GPF 0 "register_operand" "=w")
4418         (neg:GPF (match_operand:GPF 1 "register_operand" "w")))]
4419   "TARGET_FLOAT"
4420   "fneg\\t%<s>0, %<s>1"
4421   [(set_attr "type" "ffarith<s>")]
4422 )
4423
4424 (define_insn "sqrt<mode>2"
4425   [(set (match_operand:GPF 0 "register_operand" "=w")
4426         (sqrt:GPF (match_operand:GPF 1 "register_operand" "w")))]
4427   "TARGET_FLOAT"
4428   "fsqrt\\t%<s>0, %<s>1"
4429   [(set_attr "type" "fsqrt<s>")]
4430 )
4431
4432 (define_insn "abs<mode>2"
4433   [(set (match_operand:GPF 0 "register_operand" "=w")
4434         (abs:GPF (match_operand:GPF 1 "register_operand" "w")))]
4435   "TARGET_FLOAT"
4436   "fabs\\t%<s>0, %<s>1"
4437   [(set_attr "type" "ffarith<s>")]
4438 )
4439
4440 ;; Given that smax/smin do not specify the result when either input is NaN,
4441 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
4442 ;; for smin.
4443
4444 (define_insn "smax<mode>3"
4445   [(set (match_operand:GPF 0 "register_operand" "=w")
4446         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
4447                   (match_operand:GPF 2 "register_operand" "w")))]
4448   "TARGET_FLOAT"
4449   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
4450   [(set_attr "type" "f_minmax<s>")]
4451 )
4452
4453 (define_insn "smin<mode>3"
4454   [(set (match_operand:GPF 0 "register_operand" "=w")
4455         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
4456                   (match_operand:GPF 2 "register_operand" "w")))]
4457   "TARGET_FLOAT"
4458   "fminnm\\t%<s>0, %<s>1, %<s>2"
4459   [(set_attr "type" "f_minmax<s>")]
4460 )
4461
4462 ;; For copysign (x, y), we want to generate:
4463 ;;
4464 ;;   LDR d2, #(1 << 63)
4465 ;;   BSL v2.8b, [y], [x]
4466 ;;
4467 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
4468 ;; aarch64_simd_bsldf will select the best suited of these instructions
4469 ;; to generate based on register allocation, and knows how to partially
4470 ;; constant fold based on the values of X and Y, so expand through that.
4471
4472 (define_expand "copysigndf3"
4473   [(match_operand:DF 0 "register_operand")
4474    (match_operand:DF 1 "register_operand")
4475    (match_operand:DF 2 "register_operand")]
4476   "TARGET_FLOAT && TARGET_SIMD"
4477 {
4478   rtx mask = gen_reg_rtx (DImode);
4479   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
4480   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
4481                                      operands[2], operands[1]));
4482   DONE;
4483 }
4484 )
4485
4486 ;; As above, but we must first get to a 64-bit value if we wish to use
4487 ;; aarch64_simd_bslv2sf.
4488
4489 (define_expand "copysignsf3"
4490   [(match_operand:SF 0 "register_operand")
4491    (match_operand:SF 1 "register_operand")
4492    (match_operand:SF 2 "register_operand")]
4493   "TARGET_FLOAT && TARGET_SIMD"
4494 {
4495   rtx mask = gen_reg_rtx (DImode);
4496
4497   /* Juggle modes to get us in to a vector mode for BSL.  */
4498   rtx op1 = lowpart_subreg (V2SFmode, operands[1], SFmode);
4499   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
4500   rtx tmp = gen_reg_rtx (V2SFmode);
4501   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 31));
4502   emit_insn (gen_aarch64_simd_bslv2sf (tmp, mask, op2, op1));
4503   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
4504   DONE;
4505 }
4506 )
4507
4508 ;; -------------------------------------------------------------------
4509 ;; Reload support
4510 ;; -------------------------------------------------------------------
4511 ;; Reload Scalar Floating point modes from constant pool.
4512 ;; The AArch64 port doesn't have __int128 constant move support.
4513 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
4514  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
4515        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
4516   (clobber (match_operand:P 2 "register_operand" "=&r"))]
4517  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
4518  {
4519    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
4520    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
4521    DONE;
4522  }
4523 )
4524
4525 ;; Reload Vector modes from constant pool.
4526 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
4527  [(set (match_operand:VALL 0 "register_operand" "=w")
4528        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
4529   (clobber (match_operand:P 2 "register_operand" "=&r"))]
4530  "TARGET_FLOAT && aarch64_nopcrelative_literal_loads"
4531  {
4532    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
4533    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
4534    DONE;
4535  }
4536 )
4537
4538 (define_expand "aarch64_reload_mov<mode>"
4539   [(set (match_operand:TX 0 "register_operand" "=w")
4540         (match_operand:TX 1 "register_operand" "w"))
4541    (clobber (match_operand:DI 2 "register_operand" "=&r"))
4542   ]
4543   "TARGET_FLOAT"
4544   {
4545     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
4546     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
4547     gen_aarch64_movtilow_tilow (op0, op1);
4548     gen_aarch64_movdi_tihigh (operands[2], op1);
4549     gen_aarch64_movtihigh_di (op0, operands[2]);
4550     DONE;
4551   }
4552 )
4553
4554 ;; The following secondary reload helpers patterns are invoked
4555 ;; after or during reload as we don't want these patterns to start
4556 ;; kicking in during the combiner.
4557
4558 (define_insn "aarch64_movdi_<mode>low"
4559   [(set (match_operand:DI 0 "register_operand" "=r")
4560         (truncate:DI (match_operand:TX 1 "register_operand" "w")))]
4561   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4562   "fmov\\t%x0, %d1"
4563   [(set_attr "type" "f_mrc")
4564    (set_attr "length" "4")
4565   ])
4566
4567 (define_insn "aarch64_movdi_<mode>high"
4568   [(set (match_operand:DI 0 "register_operand" "=r")
4569         (truncate:DI
4570           (lshiftrt:TX (match_operand:TX 1 "register_operand" "w")
4571                        (const_int 64))))]
4572   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4573   "fmov\\t%x0, %1.d[1]"
4574   [(set_attr "type" "f_mrc")
4575    (set_attr "length" "4")
4576   ])
4577
4578 (define_insn "aarch64_mov<mode>high_di"
4579   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
4580                          (const_int 64) (const_int 64))
4581         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4582   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4583   "fmov\\t%0.d[1], %x1"
4584   [(set_attr "type" "f_mcr")
4585    (set_attr "length" "4")
4586   ])
4587
4588 (define_insn "aarch64_mov<mode>low_di"
4589   [(set (match_operand:TX 0 "register_operand" "=w")
4590         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4591   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4592   "fmov\\t%d0, %x1"
4593   [(set_attr "type" "f_mcr")
4594    (set_attr "length" "4")
4595   ])
4596
4597 (define_insn "aarch64_movtilow_tilow"
4598   [(set (match_operand:TI 0 "register_operand" "=w")
4599         (zero_extend:TI
4600           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
4601   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4602   "fmov\\t%d0, %d1"
4603   [(set_attr "type" "fmov")
4604    (set_attr "length" "4")
4605   ])
4606
4607 ;; There is a deliberate reason why the parameters of high and lo_sum's
4608 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
4609 ;; and lo_sum's to be used with the labels defining the jump tables in
4610 ;; rodata section.
4611
4612 (define_expand "add_losym"
4613   [(set (match_operand 0 "register_operand" "=r")
4614         (lo_sum (match_operand 1 "register_operand" "r")
4615                 (match_operand 2 "aarch64_valid_symref" "S")))]
4616   ""
4617 {
4618   machine_mode mode = GET_MODE (operands[0]);
4619
4620   emit_insn ((mode == DImode
4621               ? gen_add_losym_di
4622               : gen_add_losym_si) (operands[0],
4623                                    operands[1],
4624                                    operands[2]));
4625   DONE;
4626 })
4627
4628 (define_insn "add_losym_<mode>"
4629   [(set (match_operand:P 0 "register_operand" "=r")
4630         (lo_sum:P (match_operand:P 1 "register_operand" "r")
4631                   (match_operand 2 "aarch64_valid_symref" "S")))]
4632   ""
4633   "add\\t%<w>0, %<w>1, :lo12:%a2"
4634   [(set_attr "type" "alu_imm")]
4635 )
4636
4637 (define_insn "ldr_got_small_<mode>"
4638   [(set (match_operand:PTR 0 "register_operand" "=r")
4639         (unspec:PTR [(mem:PTR (lo_sum:PTR
4640                               (match_operand:PTR 1 "register_operand" "r")
4641                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
4642                     UNSPEC_GOTSMALLPIC))]
4643   ""
4644   "ldr\\t%<w>0, [%1, #:got_lo12:%a2]"
4645   [(set_attr "type" "load1")]
4646 )
4647
4648 (define_insn "ldr_got_small_sidi"
4649   [(set (match_operand:DI 0 "register_operand" "=r")
4650         (zero_extend:DI
4651          (unspec:SI [(mem:SI (lo_sum:DI
4652                              (match_operand:DI 1 "register_operand" "r")
4653                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
4654                     UNSPEC_GOTSMALLPIC)))]
4655   "TARGET_ILP32"
4656   "ldr\\t%w0, [%1, #:got_lo12:%a2]"
4657   [(set_attr "type" "load1")]
4658 )
4659
4660 (define_insn "ldr_got_small_28k_<mode>"
4661   [(set (match_operand:PTR 0 "register_operand" "=r")
4662         (unspec:PTR [(mem:PTR (lo_sum:PTR
4663                               (match_operand:PTR 1 "register_operand" "r")
4664                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
4665                     UNSPEC_GOTSMALLPIC28K))]
4666   ""
4667   "ldr\\t%<w>0, [%1, #:<got_modifier>:%a2]"
4668   [(set_attr "type" "load1")]
4669 )
4670
4671 (define_insn "ldr_got_small_28k_sidi"
4672   [(set (match_operand:DI 0 "register_operand" "=r")
4673         (zero_extend:DI
4674          (unspec:SI [(mem:SI (lo_sum:DI
4675                              (match_operand:DI 1 "register_operand" "r")
4676                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
4677                     UNSPEC_GOTSMALLPIC28K)))]
4678   "TARGET_ILP32"
4679   "ldr\\t%w0, [%1, #:gotpage_lo14:%a2]"
4680   [(set_attr "type" "load1")]
4681 )
4682
4683 (define_insn "ldr_got_tiny"
4684   [(set (match_operand:DI 0 "register_operand" "=r")
4685         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
4686                    UNSPEC_GOTTINYPIC))]
4687   ""
4688   "ldr\\t%0, %L1"
4689   [(set_attr "type" "load1")]
4690 )
4691
4692 (define_insn "aarch64_load_tp_hard"
4693   [(set (match_operand:DI 0 "register_operand" "=r")
4694         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
4695   ""
4696   "mrs\\t%0, tpidr_el0"
4697   [(set_attr "type" "mrs")]
4698 )
4699
4700 ;; The TLS ABI specifically requires that the compiler does not schedule
4701 ;; instructions in the TLS stubs, in order to enable linker relaxation.
4702 ;; Therefore we treat the stubs as an atomic sequence.
4703 (define_expand "tlsgd_small"
4704  [(parallel [(set (match_operand 0 "register_operand" "")
4705                   (call (mem:DI (match_dup 2)) (const_int 1)))
4706              (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
4707              (clobber (reg:DI LR_REGNUM))])]
4708  ""
4709 {
4710   operands[2] = aarch64_tls_get_addr ();
4711 })
4712
4713 (define_insn "*tlsgd_small"
4714   [(set (match_operand 0 "register_operand" "")
4715         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
4716    (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
4717    (clobber (reg:DI LR_REGNUM))
4718   ]
4719   ""
4720   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
4721   [(set_attr "type" "call")
4722    (set_attr "length" "16")])
4723
4724 (define_insn "tlsie_small_<mode>"
4725   [(set (match_operand:PTR 0 "register_operand" "=r")
4726         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4727                    UNSPEC_GOTSMALLTLS))]
4728   ""
4729   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
4730   [(set_attr "type" "load1")
4731    (set_attr "length" "8")]
4732 )
4733
4734 (define_insn "tlsie_small_sidi"
4735   [(set (match_operand:DI 0 "register_operand" "=r")
4736         (zero_extend:DI
4737           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4738                       UNSPEC_GOTSMALLTLS)))]
4739   ""
4740   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
4741   [(set_attr "type" "load1")
4742    (set_attr "length" "8")]
4743 )
4744
4745 (define_insn "tlsie_tiny_<mode>"
4746   [(set (match_operand:PTR 0 "register_operand" "=&r")
4747         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
4748                      (match_operand:PTR 2 "register_operand" "r")]
4749                    UNSPEC_GOTTINYTLS))]
4750   ""
4751   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
4752   [(set_attr "type" "multiple")
4753    (set_attr "length" "8")]
4754 )
4755
4756 (define_insn "tlsie_tiny_sidi"
4757   [(set (match_operand:DI 0 "register_operand" "=&r")
4758         (zero_extend:DI
4759           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
4760                       (match_operand:DI 2 "register_operand" "r")
4761                       ]
4762                       UNSPEC_GOTTINYTLS)))]
4763   ""
4764   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
4765   [(set_attr "type" "multiple")
4766    (set_attr "length" "8")]
4767 )
4768
4769 (define_insn "tlsle12_<mode>"
4770   [(set (match_operand:P 0 "register_operand" "=r")
4771         (unspec:P [(match_operand:P 1 "register_operand" "r")
4772                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4773                    UNSPEC_TLSLE12))]
4774   ""
4775   "add\\t%<w>0, %<w>1, #%L2";
4776   [(set_attr "type" "alu_sreg")
4777    (set_attr "length" "4")]
4778 )
4779
4780 (define_insn "tlsle24_<mode>"
4781   [(set (match_operand:P 0 "register_operand" "=r")
4782         (unspec:P [(match_operand:P 1 "register_operand" "r")
4783                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4784                    UNSPEC_TLSLE24))]
4785   ""
4786   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
4787   [(set_attr "type" "multiple")
4788    (set_attr "length" "8")]
4789 )
4790
4791 (define_insn "tlsle32_<mode>"
4792   [(set (match_operand:P 0 "register_operand" "=r")
4793         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
4794                    UNSPEC_TLSLE32))]
4795   ""
4796   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
4797   [(set_attr "type" "multiple")
4798    (set_attr "length" "8")]
4799 )
4800
4801 (define_insn "tlsle48_<mode>"
4802   [(set (match_operand:P 0 "register_operand" "=r")
4803         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
4804                    UNSPEC_TLSLE48))]
4805   ""
4806   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
4807   [(set_attr "type" "multiple")
4808    (set_attr "length" "12")]
4809 )
4810
4811 (define_insn "tlsdesc_small_<mode>"
4812   [(set (reg:PTR R0_REGNUM)
4813         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
4814                    UNSPEC_TLSDESC))
4815    (clobber (reg:DI LR_REGNUM))
4816    (clobber (reg:CC CC_REGNUM))
4817    (clobber (match_scratch:DI 1 "=r"))]
4818   "TARGET_TLS_DESC"
4819   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
4820   [(set_attr "type" "call")
4821    (set_attr "length" "16")])
4822
4823 (define_insn "stack_tie"
4824   [(set (mem:BLK (scratch))
4825         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
4826                      (match_operand:DI 1 "register_operand" "rk")]
4827                     UNSPEC_PRLG_STK))]
4828   ""
4829   ""
4830   [(set_attr "length" "0")]
4831 )
4832
4833 ;; Named pattern for expanding thread pointer reference.
4834 (define_expand "get_thread_pointerdi"
4835   [(match_operand:DI 0 "register_operand" "=r")]
4836   ""
4837 {
4838   rtx tmp = aarch64_load_tp (operands[0]);
4839   if (tmp != operands[0])
4840     emit_move_insn (operands[0], tmp);
4841   DONE;
4842 })
4843
4844 ;; Named patterns for stack smashing protection.
4845 (define_expand "stack_protect_set"
4846   [(match_operand 0 "memory_operand")
4847    (match_operand 1 "memory_operand")]
4848   ""
4849 {
4850   machine_mode mode = GET_MODE (operands[0]);
4851
4852   emit_insn ((mode == DImode
4853               ? gen_stack_protect_set_di
4854               : gen_stack_protect_set_si) (operands[0], operands[1]));
4855   DONE;
4856 })
4857
4858 (define_insn "stack_protect_set_<mode>"
4859   [(set (match_operand:PTR 0 "memory_operand" "=m")
4860         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
4861          UNSPEC_SP_SET))
4862    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
4863   ""
4864   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
4865   [(set_attr "length" "12")
4866    (set_attr "type" "multiple")])
4867
4868 (define_expand "stack_protect_test"
4869   [(match_operand 0 "memory_operand")
4870    (match_operand 1 "memory_operand")
4871    (match_operand 2)]
4872   ""
4873 {
4874   rtx result;
4875   machine_mode mode = GET_MODE (operands[0]);
4876
4877   result = gen_reg_rtx(mode);
4878
4879   emit_insn ((mode == DImode
4880               ? gen_stack_protect_test_di
4881               : gen_stack_protect_test_si) (result,
4882                                             operands[0],
4883                                             operands[1]));
4884
4885   if (mode == DImode)
4886     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
4887                                     result, const0_rtx, operands[2]));
4888   else
4889     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
4890                                     result, const0_rtx, operands[2]));
4891   DONE;
4892 })
4893
4894 (define_insn "stack_protect_test_<mode>"
4895   [(set (match_operand:PTR 0 "register_operand" "=r")
4896         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
4897                      (match_operand:PTR 2 "memory_operand" "m")]
4898          UNSPEC_SP_TEST))
4899    (clobber (match_scratch:PTR 3 "=&r"))]
4900   ""
4901   "ldr\t%<w>3, %x1\;ldr\t%<w>0, %x2\;eor\t%<w>0, %<w>3, %<w>0"
4902   [(set_attr "length" "12")
4903    (set_attr "type" "multiple")])
4904
4905 ;; Write Floating-point Control Register.
4906 (define_insn "set_fpcr"
4907   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
4908   ""
4909   "msr\\tfpcr, %0"
4910   [(set_attr "type" "mrs")])
4911
4912 ;; Read Floating-point Control Register.
4913 (define_insn "get_fpcr"
4914   [(set (match_operand:SI 0 "register_operand" "=r")
4915         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
4916   ""
4917   "mrs\\t%0, fpcr"
4918   [(set_attr "type" "mrs")])
4919
4920 ;; Write Floating-point Status Register.
4921 (define_insn "set_fpsr"
4922   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
4923   ""
4924   "msr\\tfpsr, %0"
4925   [(set_attr "type" "mrs")])
4926
4927 ;; Read Floating-point Status Register.
4928 (define_insn "get_fpsr"
4929   [(set (match_operand:SI 0 "register_operand" "=r")
4930         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
4931   ""
4932   "mrs\\t%0, fpsr"
4933   [(set_attr "type" "mrs")])
4934
4935
4936 ;; Define the subtract-one-and-jump insns so loop.c
4937 ;; knows what to generate.
4938 (define_expand "doloop_end"
4939   [(use (match_operand 0 "" ""))      ; loop pseudo
4940    (use (match_operand 1 "" ""))]     ; label
4941   "optimize > 0 && flag_modulo_sched"
4942 {
4943   rtx s0;
4944   rtx bcomp;
4945   rtx loc_ref;
4946   rtx cc_reg;
4947   rtx insn;
4948   rtx cmp;
4949
4950   /* Currently SMS relies on the do-loop pattern to recognize loops
4951      where (1) the control part consists of all insns defining and/or
4952      using a certain 'count' register and (2) the loop count can be
4953      adjusted by modifying this register prior to the loop.
4954      ??? The possible introduction of a new block to initialize the
4955      new IV can potentially affect branch optimizations.  */
4956
4957   if (GET_MODE (operands[0]) != DImode)
4958     FAIL;
4959
4960   s0 = operands [0];
4961   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
4962
4963   cmp = XVECEXP (PATTERN (insn), 0, 0);
4964   cc_reg = SET_DEST (cmp);
4965   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
4966   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
4967   emit_jump_insn (gen_rtx_SET (pc_rtx,
4968                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
4969                                                      loc_ref, pc_rtx)));
4970   DONE;
4971 })
4972
4973 ;; AdvSIMD Stuff
4974 (include "aarch64-simd.md")
4975
4976 ;; Atomic Operations
4977 (include "atomics.md")
4978
4979 ;; ldp/stp peephole patterns
4980 (include "aarch64-ldpstp.md")