gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (R17_REGNUM         17)
  42     (R18_REGNUM         18)
  43     (R19_REGNUM         19)
  44     (R20_REGNUM         20)
  45     (R21_REGNUM         21)
  46     (R22_REGNUM         22)
  47     (R23_REGNUM         23)
  48     (R24_REGNUM         24)
  49     (R25_REGNUM         25)
  50     (R26_REGNUM         26)
  51     (R27_REGNUM         27)
  52     (R28_REGNUM         28)
  53     (R29_REGNUM         29)
  54     (R30_REGNUM         30)
  55     (SP_REGNUM          31)
  56     (V0_REGNUM          32)
  57     (V1_REGNUM          33)
  58     (V2_REGNUM          34)
  59     (V3_REGNUM          35)
  60     (V4_REGNUM          36)
  61     (V5_REGNUM          37)
  62     (V6_REGNUM          38)
  63     (V7_REGNUM          39)
  64     (V8_REGNUM          40)
  65     (V9_REGNUM          41)
  66     (V10_REGNUM         42)
  67     (V11_REGNUM         43)
  68     (V12_REGNUM         44)
  69     (V13_REGNUM         45)
  70     (V14_REGNUM         46)
  71     (V15_REGNUM         47)
  72     (V16_REGNUM         48)
  73     (V17_REGNUM         49)
  74     (V18_REGNUM         50)
  75     (V19_REGNUM         51)
  76     (V20_REGNUM         52)
  77     (V21_REGNUM         53)
  78     (V22_REGNUM         54)
  79     (V23_REGNUM         55)
  80     (V24_REGNUM         56)
  81     (V25_REGNUM         57)
  82     (V26_REGNUM         58)
  83     (V27_REGNUM         59)
  84     (V28_REGNUM         60)
  85     (V29_REGNUM         61)
  86     (V30_REGNUM         62)
  87     (V31_REGNUM         63)
  88     (LAST_SAVED_REGNUM  63)
  89     (SFP_REGNUM         64)
  90     (AP_REGNUM          65)
  91     (CC_REGNUM          66)
  92     ;; Defined only to make the DWARF description simpler.
  93     (VG_REGNUM          67)
  94     (P0_REGNUM          68)
  95     (P1_REGNUM          69)
  96     (P2_REGNUM          70)
  97     (P3_REGNUM          71)
  98     (P4_REGNUM          72)
  99     (P5_REGNUM          73)
 100     (P6_REGNUM          74)
 101     (P7_REGNUM          75)
 102     (P8_REGNUM          76)
 103     (P9_REGNUM          77)
 104     (P10_REGNUM         78)
 105     (P11_REGNUM         79)
 106     (P12_REGNUM         80)
 107     (P13_REGNUM         81)
 108     (P14_REGNUM         82)
 109     (P15_REGNUM         83)
 110     ;; Scratch register used by stack clash protection to calculate
 111     ;; SVE CFA offsets during probing.
 112     (STACK_CLASH_SVE_CFA_REGNUM 11)
 113     ;; Scratch registers for prologue/epilogue use.
 114     (EP0_REGNUM         12)
 115     (EP1_REGNUM         13)
 116     ;; A couple of call-clobbered registers that we need to reserve when
 117     ;; tracking speculation this is not ABI, so is subject to change.
 118     (SPECULATION_SCRATCH_REGNUM 14)
 119     (SPECULATION_TRACKER_REGNUM 15)
 120     ;; Scratch registers used in frame layout.
 121     (IP0_REGNUM         16)
 122     (IP1_REGNUM         17)
 123     (LR_REGNUM          30)
 124   ]
 125 )
 126
 127 (define_c_enum "unspec" [
 128     UNSPEC_AUTIA1716
 129     UNSPEC_AUTIB1716
 130     UNSPEC_AUTIASP
 131     UNSPEC_AUTIBSP
 132     UNSPEC_CASESI
 133     UNSPEC_CRC32B
 134     UNSPEC_CRC32CB
 135     UNSPEC_CRC32CH
 136     UNSPEC_CRC32CW
 137     UNSPEC_CRC32CX
 138     UNSPEC_CRC32H
 139     UNSPEC_CRC32W
 140     UNSPEC_CRC32X
 141     UNSPEC_FCVTZS
 142     UNSPEC_FCVTZU
 143     UNSPEC_URECPE
 144     UNSPEC_FRECPE
 145     UNSPEC_FRECPS
 146     UNSPEC_FRECPX
 147     UNSPEC_FRINTA
 148     UNSPEC_FRINTI
 149     UNSPEC_FRINTM
 150     UNSPEC_FRINTN
 151     UNSPEC_FRINTP
 152     UNSPEC_FRINTX
 153     UNSPEC_FRINTZ
 154     UNSPEC_GOTSMALLPIC
 155     UNSPEC_GOTSMALLPIC28K
 156     UNSPEC_GOTSMALLTLS
 157     UNSPEC_GOTTINYPIC
 158     UNSPEC_GOTTINYTLS
 159     UNSPEC_LD1
 160     UNSPEC_LD2
 161     UNSPEC_LD2_DREG
 162     UNSPEC_LD2_DUP
 163     UNSPEC_LD3
 164     UNSPEC_LD3_DREG
 165     UNSPEC_LD3_DUP
 166     UNSPEC_LD4
 167     UNSPEC_LD4_DREG
 168     UNSPEC_LD4_DUP
 169     UNSPEC_LD2_LANE
 170     UNSPEC_LD3_LANE
 171     UNSPEC_LD4_LANE
 172     UNSPEC_MB
 173     UNSPEC_NOP
 174     UNSPEC_PACIA1716
 175     UNSPEC_PACIB1716
 176     UNSPEC_PACIASP
 177     UNSPEC_PACIBSP
 178     UNSPEC_PRLG_STK
 179     UNSPEC_REV
 180     UNSPEC_RBIT
 181     UNSPEC_SABAL
 182     UNSPEC_SABDL2
 183     UNSPEC_SADALP
 184     UNSPEC_SCVTF
 185     UNSPEC_SISD_NEG
 186     UNSPEC_SISD_SSHL
 187     UNSPEC_SISD_USHL
 188     UNSPEC_SSHL_2S
 189     UNSPEC_ST1
 190     UNSPEC_ST2
 191     UNSPEC_ST3
 192     UNSPEC_ST4
 193     UNSPEC_ST2_LANE
 194     UNSPEC_ST3_LANE
 195     UNSPEC_ST4_LANE
 196     UNSPEC_TLS
 197     UNSPEC_TLSDESC
 198     UNSPEC_TLSLE12
 199     UNSPEC_TLSLE24
 200     UNSPEC_TLSLE32
 201     UNSPEC_TLSLE48
 202     UNSPEC_UABAL
 203     UNSPEC_UABDL2
 204     UNSPEC_UADALP
 205     UNSPEC_UCVTF
 206     UNSPEC_USHL_2S
 207     UNSPEC_VSTRUCTDUMMY
 208     UNSPEC_SSP_SYSREG
 209     UNSPEC_SP_SET
 210     UNSPEC_SP_TEST
 211     UNSPEC_RSQRT
 212     UNSPEC_RSQRTE
 213     UNSPEC_RSQRTS
 214     UNSPEC_NZCV
 215     UNSPEC_XPACLRI
 216     UNSPEC_LD1_SVE
 217     UNSPEC_ST1_SVE
 218     UNSPEC_LD1RQ
 219     UNSPEC_LD1_GATHER
 220     UNSPEC_ST1_SCATTER
 221     UNSPEC_MERGE_PTRUE
 222     UNSPEC_PTEST_PTRUE
 223     UNSPEC_UNPACKSHI
 224     UNSPEC_UNPACKUHI
 225     UNSPEC_UNPACKSLO
 226     UNSPEC_UNPACKULO
 227     UNSPEC_PACK
 228     UNSPEC_FLOAT_CONVERT
 229     UNSPEC_WHILE_LO
 230     UNSPEC_LDN
 231     UNSPEC_STN
 232     UNSPEC_INSR
 233     UNSPEC_CLASTB
 234     UNSPEC_FADDA
 235     UNSPEC_REV_SUBREG
 236     UNSPEC_SPECULATION_TRACKER
 237     UNSPEC_COPYSIGN
 238 ])
 239
 240 (define_c_enum "unspecv" [
 241     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 242     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 243     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 244     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 245     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 246     UNSPECV_BLOCKAGE            ; Represent a blockage
 247     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 248     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 249     UNSPECV_BTI_NOARG           ; Represent BTI.
 250     UNSPECV_BTI_C               ; Represent BTI c.
 251     UNSPECV_BTI_J               ; Represent BTI j.
 252     UNSPECV_BTI_JC              ; Represent BTI jc.
 253   ]
 254 )
 255
 256 ;; If further include files are added the defintion of MD_INCLUDES
 257 ;; must be updated.
 258
 259 (include "constraints.md")
 260 (include "predicates.md")
 261 (include "iterators.md")
 262
 263 ;; -------------------------------------------------------------------
 264 ;; Instruction types and attributes
 265 ;; -------------------------------------------------------------------
 266
 267 ; The "type" attribute is included here from AArch32 backend to be able
 268 ; to share pipeline descriptions.
 269 (include "../arm/types.md")
 270
 271 ;; It is important to set the fp or simd attributes to yes when a pattern
 272 ;; alternative uses the FP or SIMD register files, usually signified by use of
 273 ;; the 'w' constraint.  This will ensure that the alternative will be
 274 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 275 ;; architecture extensions.  If all the alternatives in a pattern use the
 276 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 277 ;; or TARGET_SIMD.
 278
 279 ;; Attributes of the architecture required to support the instruction (or
 280 ;; alternative). This attribute is used to compute attribute "enabled", use type
 281 ;; "any" to enable an alternative in all cases.
 282
 283 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 284
 285 (define_enum_attr "arch" "arches" (const_string "any"))
 286
 287 ;; [For compatibility with Arm in pipeline models]
 288 ;; Attribute that specifies whether or not the instruction touches fp
 289 ;; registers.
 290 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 291 ;; backends except in the scheduling description for xgene1.  In that
 292 ;; scheduling description this attribute is used to subclass the load_4 and
 293 ;; load_8 types.
 294 (define_attr "fp" "no,yes"
 295   (if_then_else
 296     (eq_attr "arch" "fp")
 297     (const_string "yes")
 298     (const_string "no")))
 299
 300 (define_attr "arch_enabled" "no,yes"
 301   (if_then_else
 302     (ior
 303         (eq_attr "arch" "any")
 304
 305         (and (eq_attr "arch" "rcpc8_4")
 306              (match_test "AARCH64_ISA_RCPC8_4"))
 307
 308         (and (eq_attr "arch" "fp")
 309              (match_test "TARGET_FLOAT"))
 310
 311         (and (eq_attr "arch" "simd")
 312              (match_test "TARGET_SIMD"))
 313
 314         (and (eq_attr "arch" "fp16")
 315              (match_test "TARGET_FP_F16INST"))
 316
 317         (and (eq_attr "arch" "sve")
 318              (match_test "TARGET_SVE")))
 319     (const_string "yes")
 320     (const_string "no")))
 321
 322 ;; Attribute that controls whether an alternative is enabled or not.
 323 ;; Currently it is only used to disable alternatives which touch fp or simd
 324 ;; registers when -mgeneral-regs-only is specified or to require a special
 325 ;; architecture support.
 326 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 327
 328 ;; Attribute that specifies whether we are dealing with a branch to a
 329 ;; label that is far away, i.e. further away than the maximum/minimum
 330 ;; representable in a signed 21-bits number.
 331 ;; 0 :=: no
 332 ;; 1 :=: yes
 333 (define_attr "far_branch" "" (const_int 0))
 334
 335 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 336 (define_attr "movprfx" "no,yes" (const_string "no"))
 337
 338 (define_attr "length" ""
 339   (cond [(eq_attr "movprfx" "yes")
 340            (const_int 8)
 341         ] (const_int 4)))
 342
 343 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 344 ;; no predicated insns.
 345 (define_attr "predicated" "yes,no" (const_string "no"))
 346
 347 ;; Set to true on an insn that requires the speculation tracking state to be
 348 ;; in the tracking register before the insn issues.  Otherwise the compiler
 349 ;; may chose to hold the tracking state encoded in SP.
 350 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 351
 352 ;; -------------------------------------------------------------------
 353 ;; Pipeline descriptions and scheduling
 354 ;; -------------------------------------------------------------------
 355
 356 ;; Processor types.
 357 (include "aarch64-tune.md")
 358
 359 ;; Scheduling
 360 (include "../arm/cortex-a53.md")
 361 (include "../arm/cortex-a57.md")
 362 (include "../arm/exynos-m1.md")
 363 (include "falkor.md")
 364 (include "saphira.md")
 365 (include "thunderx.md")
 366 (include "../arm/xgene1.md")
 367 (include "thunderx2t99.md")
 368 (include "tsv110.md")
 369
 370 ;; -------------------------------------------------------------------
 371 ;; Jumps and other miscellaneous insns
 372 ;; -------------------------------------------------------------------
 373
 374 (define_insn "indirect_jump"
 375   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 376   ""
 377   "br\\t%0"
 378   [(set_attr "type" "branch")]
 379 )
 380
 381 (define_insn "jump"
 382   [(set (pc) (label_ref (match_operand 0 "" "")))]
 383   ""
 384   "b\\t%l0"
 385   [(set_attr "type" "branch")]
 386 )
 387
 388 (define_expand "cbranch<mode>4"
 389   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 390                             [(match_operand:GPI 1 "register_operand" "")
 391                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 392                            (label_ref (match_operand 3 "" ""))
 393                            (pc)))]
 394   ""
 395   "
 396   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 397                                          operands[2]);
 398   operands[2] = const0_rtx;
 399   "
 400 )
 401
 402 (define_expand "cbranch<mode>4"
 403   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 404                             [(match_operand:GPF 1 "register_operand" "")
 405                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 406                            (label_ref (match_operand 3 "" ""))
 407                            (pc)))]
 408   ""
 409   "
 410   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 411                                          operands[2]);
 412   operands[2] = const0_rtx;
 413   "
 414 )
 415
 416 (define_expand "cbranchcc4"
 417   [(set (pc) (if_then_else
 418               (match_operator 0 "aarch64_comparison_operator"
 419                [(match_operand 1 "cc_register" "")
 420                 (match_operand 2 "const0_operand")])
 421               (label_ref (match_operand 3 "" ""))
 422               (pc)))]
 423   ""
 424   "")
 425
 426 (define_insn "ccmp<mode>"
 427   [(set (match_operand:CC 1 "cc_register" "")
 428         (if_then_else:CC
 429           (match_operator 4 "aarch64_comparison_operator"
 430            [(match_operand 0 "cc_register" "")
 431             (const_int 0)])
 432           (compare:CC
 433             (match_operand:GPI 2 "register_operand" "r,r,r")
 434             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 435           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 436   ""
 437   "@
 438    ccmp\\t%<w>2, %<w>3, %k5, %m4
 439    ccmp\\t%<w>2, %3, %k5, %m4
 440    ccmn\\t%<w>2, #%n3, %k5, %m4"
 441   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 442 )
 443
 444 (define_insn "fccmp<mode>"
 445   [(set (match_operand:CCFP 1 "cc_register" "")
 446         (if_then_else:CCFP
 447           (match_operator 4 "aarch64_comparison_operator"
 448            [(match_operand 0 "cc_register" "")
 449             (const_int 0)])
 450           (compare:CCFP
 451             (match_operand:GPF 2 "register_operand" "w")
 452             (match_operand:GPF 3 "register_operand" "w"))
 453           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 454   "TARGET_FLOAT"
 455   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 456   [(set_attr "type" "fccmp<s>")]
 457 )
 458
 459 (define_insn "fccmpe<mode>"
 460   [(set (match_operand:CCFPE 1 "cc_register" "")
 461          (if_then_else:CCFPE
 462           (match_operator 4 "aarch64_comparison_operator"
 463            [(match_operand 0 "cc_register" "")
 464           (const_int 0)])
 465            (compare:CCFPE
 466             (match_operand:GPF 2 "register_operand" "w")
 467             (match_operand:GPF 3 "register_operand" "w"))
 468           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 469   "TARGET_FLOAT"
 470   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 471   [(set_attr "type" "fccmp<s>")]
 472 )
 473
 474 ;; Expansion of signed mod by a power of 2 using CSNEG.
 475 ;; For x0 % n where n is a power of 2 produce:
 476 ;; negs   x1, x0
 477 ;; and    x0, x0, #(n - 1)
 478 ;; and    x1, x1, #(n - 1)
 479 ;; csneg  x0, x0, x1, mi
 480
 481 (define_expand "mod<mode>3"
 482   [(match_operand:GPI 0 "register_operand" "")
 483    (match_operand:GPI 1 "register_operand" "")
 484    (match_operand:GPI 2 "const_int_operand" "")]
 485   ""
 486   {
 487     HOST_WIDE_INT val = INTVAL (operands[2]);
 488
 489     if (val <= 0
 490        || exact_log2 (val) <= 0
 491        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 492       FAIL;
 493
 494     rtx mask = GEN_INT (val - 1);
 495
 496     /* In the special case of x0 % 2 we can do the even shorter:
 497         cmp    x0, xzr
 498         and    x0, x0, 1
 499         cneg   x0, x0, lt.  */
 500     if (val == 2)
 501       {
 502         rtx masked = gen_reg_rtx (<MODE>mode);
 503         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 504         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 505         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 506         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 507         DONE;
 508       }
 509
 510     rtx neg_op = gen_reg_rtx (<MODE>mode);
 511     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 512
 513     /* Extract the condition register and mode.  */
 514     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 515     rtx cc_reg = SET_DEST (cmp);
 516     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 517
 518     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 519     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 520
 521     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 522     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 523
 524     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 525                                        masked_neg, masked_pos));
 526     DONE;
 527   }
 528 )
 529
 530 (define_insn "condjump"
 531   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 532                             [(match_operand 1 "cc_register" "") (const_int 0)])
 533                            (label_ref (match_operand 2 "" ""))
 534                            (pc)))]
 535   ""
 536   {
 537     /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
 538        but the "." is required for SVE conditions.  */
 539     bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
 540     if (get_attr_length (insn) == 8)
 541       return aarch64_gen_far_branch (operands, 2, "Lbcond",
 542                                      use_dot_p ? "b.%M0\\t" : "b%M0\\t");
 543     else
 544       return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
 545   }
 546   [(set_attr "type" "branch")
 547    (set (attr "length")
 548         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 549                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 550                       (const_int 4)
 551                       (const_int 8)))
 552    (set (attr "far_branch")
 553         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 554                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 555                       (const_int 0)
 556                       (const_int 1)))]
 557 )
 558
 559 ;; For a 24-bit immediate CST we can optimize the compare for equality
 560 ;; and branch sequence from:
 561 ;;      mov     x0, #imm1
 562 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 563 ;;      cmp     x1, x0
 564 ;;      b<ne,eq> .Label
 565 ;; into the shorter:
 566 ;;      sub     x0, x1, #(CST & 0xfff000)
 567 ;;      subs    x0, x0, #(CST & 0x000fff)
 568 ;;      b<ne,eq> .Label
 569 (define_insn_and_split "*compare_condjump<mode>"
 570   [(set (pc) (if_then_else (EQL
 571                               (match_operand:GPI 0 "register_operand" "r")
 572                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 573                            (label_ref:P (match_operand 2 "" ""))
 574                            (pc)))]
 575   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 576    && !aarch64_plus_operand (operands[1], <MODE>mode)
 577    && !reload_completed"
 578   "#"
 579   "&& true"
 580   [(const_int 0)]
 581   {
 582     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 583     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 584     rtx tmp = gen_reg_rtx (<MODE>mode);
 585     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 586     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 587     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 588     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 589     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 590     DONE;
 591   }
 592 )
 593
 594 (define_expand "casesi"
 595   [(match_operand:SI 0 "register_operand" "")   ; Index
 596    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 597    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 598    (match_operand:DI 3 "" "")                   ; Table label
 599    (match_operand:DI 4 "" "")]                  ; Out of range label
 600   ""
 601   {
 602     if (operands[1] != const0_rtx)
 603       {
 604         rtx reg = gen_reg_rtx (SImode);
 605
 606         /* Canonical RTL says that if you have:
 607
 608            (minus (X) (CONST))
 609
 610            then this should be emitted as:
 611
 612            (plus (X) (-CONST))
 613
 614            The use of trunc_int_for_mode ensures that the resulting
 615            constant can be represented in SImode, this is important
 616            for the corner case where operand[1] is INT_MIN.  */
 617
 618         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 619
 620         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 621               (operands[1], SImode))
 622           operands[1] = force_reg (SImode, operands[1]);
 623         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 624         operands[0] = reg;
 625       }
 626
 627     if (!aarch64_plus_operand (operands[2], SImode))
 628       operands[2] = force_reg (SImode, operands[2]);
 629     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 630                                                  const0_rtx),
 631                                     operands[0], operands[2], operands[4]));
 632
 633     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 634     operands[2]
 635       = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
 636                         UNSPEC_CASESI);
 637     operands[2] = gen_rtx_MEM (DImode, operands[2]);
 638     MEM_READONLY_P (operands[2]) = 1;
 639     MEM_NOTRAP_P (operands[2]) = 1;
 640     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
 641     DONE;
 642   }
 643 )
 644
 645 (define_expand "casesi_dispatch"
 646   [(parallel
 647     [(set (pc) (match_operand:DI 0 ""))
 648      (clobber (reg:CC CC_REGNUM))
 649      (clobber (match_scratch:DI 2))
 650      (clobber (match_scratch:DI 3))
 651      (use (label_ref:DI (match_operand 1 "")))])]
 652   "")
 653
 654 (define_insn "*casesi_dispatch"
 655   [(parallel
 656     [(set (pc)
 657           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 658                            (match_operand:SI 1 "register_operand" "r")]
 659                         UNSPEC_CASESI)))
 660      (clobber (reg:CC CC_REGNUM))
 661      (clobber (match_scratch:DI 3 "=r"))
 662      (clobber (match_scratch:DI 4 "=r"))
 663      (use (label_ref:DI (match_operand 2 "" "")))])]
 664   ""
 665   "*
 666   return aarch64_output_casesi (operands);
 667   "
 668   [(set_attr "length" "16")
 669    (set_attr "type" "branch")]
 670 )
 671
 672 (define_insn "nop"
 673   [(unspec[(const_int 0)] UNSPEC_NOP)]
 674   ""
 675   "nop"
 676   [(set_attr "type" "no_insn")]
 677 )
 678
 679 (define_insn "prefetch"
 680   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 681             (match_operand:QI 1 "const_int_operand" "")
 682             (match_operand:QI 2 "const_int_operand" ""))]
 683   ""
 684   {
 685     const char * pftype[2][4] =
 686     {
 687       {"prfm\\tPLDL1STRM, %0",
 688        "prfm\\tPLDL3KEEP, %0",
 689        "prfm\\tPLDL2KEEP, %0",
 690        "prfm\\tPLDL1KEEP, %0"},
 691       {"prfm\\tPSTL1STRM, %0",
 692        "prfm\\tPSTL3KEEP, %0",
 693        "prfm\\tPSTL2KEEP, %0",
 694        "prfm\\tPSTL1KEEP, %0"},
 695     };
 696
 697     int locality = INTVAL (operands[2]);
 698
 699     gcc_assert (IN_RANGE (locality, 0, 3));
 700
 701     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 702        the address into a DImode MEM so that aarch64_print_operand knows
 703        how to print it.  */
 704     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 705     return pftype[INTVAL(operands[1])][locality];
 706   }
 707   [(set_attr "type" "load_4")]
 708 )
 709
 710 (define_insn "trap"
 711   [(trap_if (const_int 1) (const_int 8))]
 712   ""
 713   "brk #1000"
 714   [(set_attr "type" "trap")])
 715
 716 (define_expand "prologue"
 717   [(clobber (const_int 0))]
 718   ""
 719   "
 720   aarch64_expand_prologue ();
 721   DONE;
 722   "
 723 )
 724
 725 (define_expand "epilogue"
 726   [(clobber (const_int 0))]
 727   ""
 728   "
 729   aarch64_expand_epilogue (false);
 730   DONE;
 731   "
 732 )
 733
 734 (define_expand "sibcall_epilogue"
 735   [(clobber (const_int 0))]
 736   ""
 737   "
 738   aarch64_expand_epilogue (true);
 739   DONE;
 740   "
 741 )
 742
 743 (define_insn "*do_return"
 744   [(return)]
 745   ""
 746   {
 747     if (aarch64_return_address_signing_enabled ()
 748         && TARGET_ARMV8_3
 749         && !crtl->calls_eh_return)
 750       {
 751         if (aarch64_ra_sign_key == AARCH64_KEY_B)
 752           return "retab";
 753         else
 754           return "retaa";
 755       }
 756     return "ret";
 757   }
 758   [(set_attr "type" "branch")]
 759 )
 760
 761 (define_expand "return"
 762   [(simple_return)]
 763   "aarch64_use_return_insn_p ()"
 764   ""
 765 )
 766
 767 (define_insn "simple_return"
 768   [(simple_return)]
 769   "aarch64_use_simple_return_insn_p ()"
 770   "ret"
 771   [(set_attr "type" "branch")]
 772 )
 773
 774 (define_insn "*cb<optab><mode>1"
 775   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 776                                 (const_int 0))
 777                            (label_ref (match_operand 1 "" ""))
 778                            (pc)))]
 779   "!aarch64_track_speculation"
 780   {
 781     if (get_attr_length (insn) == 8)
 782       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 783     else
 784       return "<cbz>\\t%<w>0, %l1";
 785   }
 786   [(set_attr "type" "branch")
 787    (set (attr "length")
 788         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 789                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 790                       (const_int 4)
 791                       (const_int 8)))
 792    (set (attr "far_branch")
 793         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 794                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 795                       (const_int 0)
 796                       (const_int 1)))]
 797 )
 798
 799 (define_insn "*tb<optab><mode>1"
 800   [(set (pc) (if_then_else
 801               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 802                                     (const_int 1)
 803                                     (match_operand 1
 804                                       "aarch64_simd_shift_imm_<mode>" "n"))
 805                    (const_int 0))
 806              (label_ref (match_operand 2 "" ""))
 807              (pc)))
 808    (clobber (reg:CC CC_REGNUM))]
 809   "!aarch64_track_speculation"
 810   {
 811     if (get_attr_length (insn) == 8)
 812       {
 813         if (get_attr_far_branch (insn) == 1)
 814           return aarch64_gen_far_branch (operands, 2, "Ltb",
 815                                          "<inv_tb>\\t%<w>0, %1, ");
 816         else
 817           {
 818             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 819             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 820           }
 821       }
 822     else
 823       return "<tbz>\t%<w>0, %1, %l2";
 824   }
 825   [(set_attr "type" "branch")
 826    (set (attr "length")
 827         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 828                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 829                       (const_int 4)
 830                       (const_int 8)))
 831    (set (attr "far_branch")
 832         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 833                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 834                       (const_int 0)
 835                       (const_int 1)))]
 836
 837 )
 838
 839 (define_insn "*cb<optab><mode>1"
 840   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 841                                  (const_int 0))
 842                            (label_ref (match_operand 1 "" ""))
 843                            (pc)))
 844    (clobber (reg:CC CC_REGNUM))]
 845   "!aarch64_track_speculation"
 846   {
 847     if (get_attr_length (insn) == 8)
 848       {
 849         if (get_attr_far_branch (insn) == 1)
 850           return aarch64_gen_far_branch (operands, 1, "Ltb",
 851                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 852         else
 853           {
 854             char buf[64];
 855             uint64_t val = ((uint64_t) 1)
 856                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 857             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 858             output_asm_insn (buf, operands);
 859             return "<bcond>\t%l1";
 860           }
 861       }
 862     else
 863       return "<tbz>\t%<w>0, <sizem1>, %l1";
 864   }
 865   [(set_attr "type" "branch")
 866    (set (attr "length")
 867         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 868                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 869                       (const_int 4)
 870                       (const_int 8)))
 871    (set (attr "far_branch")
 872         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 873                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 874                       (const_int 0)
 875                       (const_int 1)))]
 876 )
 877
 878 ;; -------------------------------------------------------------------
 879 ;; Subroutine calls and sibcalls
 880 ;; -------------------------------------------------------------------
 881
 882 (define_expand "call"
 883   [(parallel [(call (match_operand 0 "memory_operand" "")
 884                     (match_operand 1 "general_operand" ""))
 885               (use (match_operand 2 "" ""))
 886               (clobber (reg:DI LR_REGNUM))])]
 887   ""
 888   "
 889   {
 890     aarch64_expand_call (NULL_RTX, operands[0], false);
 891     DONE;
 892   }"
 893 )
 894
 895 (define_insn "*call_insn"
 896   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 897          (match_operand 1 "" ""))
 898    (clobber (reg:DI LR_REGNUM))]
 899   ""
 900   "@
 901   blr\\t%0
 902   bl\\t%c0"
 903   [(set_attr "type" "call, call")]
 904 )
 905
 906 (define_expand "call_value"
 907   [(parallel [(set (match_operand 0 "" "")
 908                    (call (match_operand 1 "memory_operand" "")
 909                          (match_operand 2 "general_operand" "")))
 910               (use (match_operand 3 "" ""))
 911               (clobber (reg:DI LR_REGNUM))])]
 912   ""
 913   "
 914   {
 915     aarch64_expand_call (operands[0], operands[1], false);
 916     DONE;
 917   }"
 918 )
 919
 920 (define_insn "*call_value_insn"
 921   [(set (match_operand 0 "" "")
 922         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 923                       (match_operand 2 "" "")))
 924    (clobber (reg:DI LR_REGNUM))]
 925   ""
 926   "@
 927   blr\\t%1
 928   bl\\t%c1"
 929   [(set_attr "type" "call, call")]
 930 )
 931
 932 (define_expand "sibcall"
 933   [(parallel [(call (match_operand 0 "memory_operand" "")
 934                     (match_operand 1 "general_operand" ""))
 935               (return)
 936               (use (match_operand 2 "" ""))])]
 937   ""
 938   {
 939     aarch64_expand_call (NULL_RTX, operands[0], true);
 940     DONE;
 941   }
 942 )
 943
 944 (define_expand "sibcall_value"
 945   [(parallel [(set (match_operand 0 "" "")
 946                    (call (match_operand 1 "memory_operand" "")
 947                          (match_operand 2 "general_operand" "")))
 948               (return)
 949               (use (match_operand 3 "" ""))])]
 950   ""
 951   {
 952     aarch64_expand_call (operands[0], operands[1], true);
 953     DONE;
 954   }
 955 )
 956
 957 (define_insn "*sibcall_insn"
 958   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 959          (match_operand 1 "" ""))
 960    (return)]
 961   "SIBLING_CALL_P (insn)"
 962   "@
 963    br\\t%0
 964    b\\t%c0"
 965   [(set_attr "type" "branch, branch")]
 966 )
 967
 968 (define_insn "*sibcall_value_insn"
 969   [(set (match_operand 0 "" "")
 970         (call (mem:DI
 971                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 972               (match_operand 2 "" "")))
 973    (return)]
 974   "SIBLING_CALL_P (insn)"
 975   "@
 976    br\\t%1
 977    b\\t%c1"
 978   [(set_attr "type" "branch, branch")]
 979 )
 980
 981 ;; Call subroutine returning any type.
 982
 983 (define_expand "untyped_call"
 984   [(parallel [(call (match_operand 0 "")
 985                     (const_int 0))
 986               (match_operand 1 "")
 987               (match_operand 2 "")])]
 988   ""
 989 {
 990   int i;
 991
 992   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 993
 994   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 995     {
 996       rtx set = XVECEXP (operands[2], 0, i);
 997       emit_move_insn (SET_DEST (set), SET_SRC (set));
 998     }
 999
1000   /* The optimizer does not know that the call sets the function value
1001      registers we stored in the result block.  We avoid problems by
1002      claiming that all hard registers are used and clobbered at this
1003      point.  */
1004   emit_insn (gen_blockage ());
1005   DONE;
1006 })
1007
1008 ;; -------------------------------------------------------------------
1009 ;; Moves
1010 ;; -------------------------------------------------------------------
1011
1012 (define_expand "mov<mode>"
1013   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
1014         (match_operand:SHORT 1 "general_operand" ""))]
1015   ""
1016   "
1017     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1018       operands[1] = force_reg (<MODE>mode, operands[1]);
1019
1020     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1021       {
1022         aarch64_expand_mov_immediate (operands[0], operands[1]);
1023         DONE;
1024       }
1025   "
1026 )
1027
1028 (define_insn "*mov<mode>_aarch64"
1029   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
1030         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1031   "(register_operand (operands[0], <MODE>mode)
1032     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1033 {
1034    switch (which_alternative)
1035      {
1036      case 0:
1037        return "mov\t%w0, %w1";
1038      case 1:
1039        return "mov\t%w0, %1";
1040      case 2:
1041        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1042                                                         <MODE>mode);
1043      case 3:
1044        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1045      case 4:
1046        return "ldr<size>\t%w0, %1";
1047      case 5:
1048        return "ldr\t%<size>0, %1";
1049      case 6:
1050        return "str<size>\t%w1, %0";
1051      case 7:
1052        return "str\t%<size>1, %0";
1053      case 8:
1054        return "umov\t%w0, %1.<v>[0]";
1055      case 9:
1056        return "dup\t%0.<Vallxd>, %w1";
1057      case 10:
1058        return "dup\t%<Vetype>0, %1.<v>[0]";
1059      default:
1060        gcc_unreachable ();
1061      }
1062 }
1063   ;; The "mov_imm" type for CNT is just a placeholder.
1064   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1065                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1066    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1067 )
1068
1069 (define_expand "mov<mode>"
1070   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1071         (match_operand:GPI 1 "general_operand" ""))]
1072   ""
1073   "
1074     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1075         && <MODE>mode == DImode
1076         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1077       DONE;
1078
1079     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1080       operands[1] = force_reg (<MODE>mode, operands[1]);
1081
1082     /* FIXME: RR we still need to fix up what we are doing with
1083        symbol_refs and other types of constants.  */
1084     if (CONSTANT_P (operands[1])
1085         && !CONST_INT_P (operands[1]))
1086      {
1087        aarch64_expand_mov_immediate (operands[0], operands[1]);
1088        DONE;
1089      }
1090   "
1091 )
1092
1093 (define_insn_and_split "*movsi_aarch64"
1094   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1095         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1096   "(register_operand (operands[0], SImode)
1097     || aarch64_reg_or_zero (operands[1], SImode))"
1098   "@
1099    mov\\t%w0, %w1
1100    mov\\t%w0, %w1
1101    mov\\t%w0, %w1
1102    mov\\t%w0, %1
1103    #
1104    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1105    ldr\\t%w0, %1
1106    ldr\\t%s0, %1
1107    str\\t%w1, %0
1108    str\\t%s1, %0
1109    adr\\t%x0, %c1
1110    adrp\\t%x0, %A1
1111    fmov\\t%s0, %w1
1112    fmov\\t%w0, %s1
1113    fmov\\t%s0, %s1
1114    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1115   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1116     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1117    [(const_int 0)]
1118    "{
1119        aarch64_expand_mov_immediate (operands[0], operands[1]);
1120        DONE;
1121     }"
1122   ;; The "mov_imm" type for CNT is just a placeholder.
1123   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1124                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1125    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1126 )
1127
1128 (define_insn_and_split "*movdi_aarch64"
1129   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1130         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1131   "(register_operand (operands[0], DImode)
1132     || aarch64_reg_or_zero (operands[1], DImode))"
1133   "@
1134    mov\\t%x0, %x1
1135    mov\\t%0, %x1
1136    mov\\t%x0, %1
1137    mov\\t%x0, %1
1138    mov\\t%w0, %1
1139    #
1140    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1141    ldr\\t%x0, %1
1142    ldr\\t%d0, %1
1143    str\\t%x1, %0
1144    str\\t%d1, %0
1145    adr\\t%x0, %c1
1146    adrp\\t%x0, %A1
1147    fmov\\t%d0, %x1
1148    fmov\\t%x0, %d1
1149    fmov\\t%d0, %d1
1150    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1151    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1152     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1153    [(const_int 0)]
1154    "{
1155        aarch64_expand_mov_immediate (operands[0], operands[1]);
1156        DONE;
1157     }"
1158   ;; The "mov_imm" type for CNTD is just a placeholder.
1159   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1160                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1161                      neon_move")
1162    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1163 )
1164
1165 (define_insn "insv_imm<mode>"
1166   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1167                           (const_int 16)
1168                           (match_operand:GPI 1 "const_int_operand" "n"))
1169         (match_operand:GPI 2 "const_int_operand" "n"))]
1170   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1171    && UINTVAL (operands[1]) % 16 == 0"
1172   "movk\\t%<w>0, %X2, lsl %1"
1173   [(set_attr "type" "mov_imm")]
1174 )
1175
1176 (define_expand "movti"
1177   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1178         (match_operand:TI 1 "general_operand" ""))]
1179   ""
1180   "
1181     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1182       operands[1] = force_reg (TImode, operands[1]);
1183
1184     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1185       {
1186         emit_move_insn (gen_lowpart (DImode, operands[0]),
1187                         gen_lowpart (DImode, operands[1]));
1188         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1189         DONE;
1190       }
1191   "
1192 )
1193
1194 (define_insn "*movti_aarch64"
1195   [(set (match_operand:TI 0
1196          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1197         (match_operand:TI 1
1198          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1199   "(register_operand (operands[0], TImode)
1200     || aarch64_reg_or_zero (operands[1], TImode))"
1201   "@
1202    #
1203    #
1204    #
1205    mov\\t%0.16b, %1.16b
1206    ldp\\t%0, %H0, %1
1207    stp\\t%1, %H1, %0
1208    stp\\txzr, xzr, %0
1209    ldr\\t%q0, %1
1210    str\\t%q1, %0"
1211   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1212                              load_16,store_16,store_16,\
1213                              load_16,store_16")
1214    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1215    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1216 )
1217
1218 ;; Split a TImode register-register or register-immediate move into
1219 ;; its component DImode pieces, taking care to handle overlapping
1220 ;; source and dest registers.
1221 (define_split
1222    [(set (match_operand:TI 0 "register_operand" "")
1223          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1224   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1225   [(const_int 0)]
1226 {
1227   aarch64_split_128bit_move (operands[0], operands[1]);
1228   DONE;
1229 })
1230
1231 (define_expand "mov<mode>"
1232   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1233         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1234   ""
1235   {
1236     if (!TARGET_FLOAT)
1237       {
1238         aarch64_err_no_fpadvsimd (<MODE>mode);
1239         FAIL;
1240       }
1241
1242     if (GET_CODE (operands[0]) == MEM
1243         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1244               && aarch64_float_const_zero_rtx_p (operands[1])))
1245       operands[1] = force_reg (<MODE>mode, operands[1]);
1246   }
1247 )
1248
1249 (define_insn "*movhf_aarch64"
1250   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1251         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1252   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1253     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1254   "@
1255    movi\\t%0.4h, #0
1256    fmov\\t%h0, %w1
1257    dup\\t%w0.4h, %w1
1258    umov\\t%w0, %1.h[0]
1259    mov\\t%0.h[0], %1.h[0]
1260    fmov\\t%h0, %1
1261    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1262    ldr\\t%h0, %1
1263    str\\t%h1, %0
1264    ldrh\\t%w0, %1
1265    strh\\t%w1, %0
1266    mov\\t%w0, %w1"
1267   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1268                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1269    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1270 )
1271
1272 (define_insn "*movsf_aarch64"
1273   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1274         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1275   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1276     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1277   "@
1278    movi\\t%0.2s, #0
1279    fmov\\t%s0, %w1
1280    fmov\\t%w0, %s1
1281    fmov\\t%s0, %s1
1282    fmov\\t%s0, %1
1283    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1284    ldr\\t%s0, %1
1285    str\\t%s1, %0
1286    ldr\\t%w0, %1
1287    str\\t%w1, %0
1288    mov\\t%w0, %w1
1289    mov\\t%w0, %1"
1290   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1291                      f_loads,f_stores,load_4,store_4,mov_reg,\
1292                      fconsts")
1293    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1294 )
1295
1296 (define_insn "*movdf_aarch64"
1297   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1298         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1299   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1300     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1301   "@
1302    movi\\t%d0, #0
1303    fmov\\t%d0, %x1
1304    fmov\\t%x0, %d1
1305    fmov\\t%d0, %d1
1306    fmov\\t%d0, %1
1307    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1308    ldr\\t%d0, %1
1309    str\\t%d1, %0
1310    ldr\\t%x0, %1
1311    str\\t%x1, %0
1312    mov\\t%x0, %x1
1313    mov\\t%x0, %1"
1314   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1315                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1316                      fconstd")
1317    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1318 )
1319
1320 (define_split
1321   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1322         (match_operand:GPF_HF 1 "general_operand"))]
1323   "can_create_pseudo_p ()
1324    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1325    && !aarch64_float_const_representable_p (operands[1])
1326    &&  aarch64_float_const_rtx_p (operands[1])"
1327   [(const_int 0)]
1328   {
1329     unsigned HOST_WIDE_INT ival;
1330     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1331       FAIL;
1332
1333     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1334     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1335     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1336     DONE;
1337   }
1338 )
1339
1340 (define_insn "*movtf_aarch64"
1341   [(set (match_operand:TF 0
1342          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1343         (match_operand:TF 1
1344          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1345   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1346     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1347   "@
1348    mov\\t%0.16b, %1.16b
1349    #
1350    #
1351    #
1352    movi\\t%0.2d, #0
1353    fmov\\t%s0, wzr
1354    ldr\\t%q0, %1
1355    str\\t%q1, %0
1356    ldp\\t%0, %H0, %1
1357    stp\\t%1, %H1, %0
1358    stp\\txzr, xzr, %0"
1359   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1360                      f_loadd,f_stored,load_16,store_16,store_16")
1361    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1362    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1363 )
1364
1365 (define_split
1366    [(set (match_operand:TF 0 "register_operand" "")
1367          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1368   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1369   [(const_int 0)]
1370   {
1371     aarch64_split_128bit_move (operands[0], operands[1]);
1372     DONE;
1373   }
1374 )
1375
1376 ;; 0 is dst
1377 ;; 1 is src
1378 ;; 2 is size of copy in bytes
1379 ;; 3 is alignment
1380
1381 (define_expand "cpymemdi"
1382   [(match_operand:BLK 0 "memory_operand")
1383    (match_operand:BLK 1 "memory_operand")
1384    (match_operand:DI 2 "immediate_operand")
1385    (match_operand:DI 3 "immediate_operand")]
1386    "!STRICT_ALIGNMENT"
1387 {
1388   if (aarch64_expand_cpymem (operands))
1389     DONE;
1390   FAIL;
1391 }
1392 )
1393
1394 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1395 ;; fairly lax checking on the second memory operation.
1396 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1397   [(set (match_operand:SX 0 "register_operand" "=r,w")
1398         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1399    (set (match_operand:SX2 2 "register_operand" "=r,w")
1400         (match_operand:SX2 3 "memory_operand" "m,m"))]
1401    "rtx_equal_p (XEXP (operands[3], 0),
1402                  plus_constant (Pmode,
1403                                 XEXP (operands[1], 0),
1404                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1405   "@
1406    ldp\\t%w0, %w2, %1
1407    ldp\\t%s0, %s2, %1"
1408   [(set_attr "type" "load_8,neon_load1_2reg")
1409    (set_attr "arch" "*,fp")]
1410 )
1411
1412 ;; Storing different modes that can still be merged
1413 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1414   [(set (match_operand:DX 0 "register_operand" "=r,w")
1415         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1416    (set (match_operand:DX2 2 "register_operand" "=r,w")
1417         (match_operand:DX2 3 "memory_operand" "m,m"))]
1418    "rtx_equal_p (XEXP (operands[3], 0),
1419                  plus_constant (Pmode,
1420                                 XEXP (operands[1], 0),
1421                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1422   "@
1423    ldp\\t%x0, %x2, %1
1424    ldp\\t%d0, %d2, %1"
1425   [(set_attr "type" "load_16,neon_load1_2reg")
1426    (set_attr "arch" "*,fp")]
1427 )
1428
1429 (define_insn "load_pair_dw_tftf"
1430   [(set (match_operand:TF 0 "register_operand" "=w")
1431         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1432    (set (match_operand:TF 2 "register_operand" "=w")
1433         (match_operand:TF 3 "memory_operand" "m"))]
1434    "TARGET_SIMD
1435     && rtx_equal_p (XEXP (operands[3], 0),
1436                     plus_constant (Pmode,
1437                                    XEXP (operands[1], 0),
1438                                    GET_MODE_SIZE (TFmode)))"
1439   "ldp\\t%q0, %q2, %1"
1440   [(set_attr "type" "neon_ldp_q")
1441    (set_attr "fp" "yes")]
1442 )
1443
1444 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1445 ;; fairly lax checking on the second memory operation.
1446 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1447   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1448         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1449    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1450         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1451    "rtx_equal_p (XEXP (operands[2], 0),
1452                  plus_constant (Pmode,
1453                                 XEXP (operands[0], 0),
1454                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1455   "@
1456    stp\\t%w1, %w3, %0
1457    stp\\t%s1, %s3, %0"
1458   [(set_attr "type" "store_8,neon_store1_2reg")
1459    (set_attr "arch" "*,fp")]
1460 )
1461
1462 ;; Storing different modes that can still be merged
1463 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1464   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1465         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1466    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1467         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1468    "rtx_equal_p (XEXP (operands[2], 0),
1469                  plus_constant (Pmode,
1470                                 XEXP (operands[0], 0),
1471                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1472   "@
1473    stp\\t%x1, %x3, %0
1474    stp\\t%d1, %d3, %0"
1475   [(set_attr "type" "store_16,neon_store1_2reg")
1476    (set_attr "arch" "*,fp")]
1477 )
1478
1479 (define_insn "store_pair_dw_tftf"
1480   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1481         (match_operand:TF 1 "register_operand" "w"))
1482    (set (match_operand:TF 2 "memory_operand" "=m")
1483         (match_operand:TF 3 "register_operand" "w"))]
1484    "TARGET_SIMD &&
1485     rtx_equal_p (XEXP (operands[2], 0),
1486                  plus_constant (Pmode,
1487                                 XEXP (operands[0], 0),
1488                                 GET_MODE_SIZE (TFmode)))"
1489   "stp\\t%q1, %q3, %0"
1490   [(set_attr "type" "neon_stp_q")
1491    (set_attr "fp" "yes")]
1492 )
1493
1494 ;; Load pair with post-index writeback.  This is primarily used in function
1495 ;; epilogues.
1496 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1497   [(parallel
1498     [(set (match_operand:P 0 "register_operand" "=k")
1499           (plus:P (match_operand:P 1 "register_operand" "0")
1500                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1501      (set (match_operand:GPI 2 "register_operand" "=r")
1502           (mem:GPI (match_dup 1)))
1503      (set (match_operand:GPI 3 "register_operand" "=r")
1504           (mem:GPI (plus:P (match_dup 1)
1505                    (match_operand:P 5 "const_int_operand" "n"))))])]
1506   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1507   "ldp\\t%<w>2, %<w>3, [%1], %4"
1508   [(set_attr "type" "load_<ldpstp_sz>")]
1509 )
1510
1511 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1512   [(parallel
1513     [(set (match_operand:P 0 "register_operand" "=k")
1514           (plus:P (match_operand:P 1 "register_operand" "0")
1515                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1516      (set (match_operand:GPF 2 "register_operand" "=w")
1517           (mem:GPF (match_dup 1)))
1518      (set (match_operand:GPF 3 "register_operand" "=w")
1519           (mem:GPF (plus:P (match_dup 1)
1520                    (match_operand:P 5 "const_int_operand" "n"))))])]
1521   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1522   "ldp\\t%<w>2, %<w>3, [%1], %4"
1523   [(set_attr "type" "neon_load1_2reg")]
1524 )
1525
1526 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1527   [(parallel
1528     [(set (match_operand:P 0 "register_operand" "=k")
1529           (plus:P (match_operand:P 1 "register_operand" "0")
1530                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1531      (set (match_operand:TX 2 "register_operand" "=w")
1532           (mem:TX (match_dup 1)))
1533      (set (match_operand:TX 3 "register_operand" "=w")
1534           (mem:TX (plus:P (match_dup 1)
1535                           (match_operand:P 5 "const_int_operand" "n"))))])]
1536   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1537   "ldp\\t%q2, %q3, [%1], %4"
1538   [(set_attr "type" "neon_ldp_q")]
1539 )
1540
1541 ;; Store pair with pre-index writeback.  This is primarily used in function
1542 ;; prologues.
1543 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1544   [(parallel
1545     [(set (match_operand:P 0 "register_operand" "=&k")
1546           (plus:P (match_operand:P 1 "register_operand" "0")
1547                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1548      (set (mem:GPI (plus:P (match_dup 0)
1549                    (match_dup 4)))
1550           (match_operand:GPI 2 "register_operand" "r"))
1551      (set (mem:GPI (plus:P (match_dup 0)
1552                    (match_operand:P 5 "const_int_operand" "n")))
1553           (match_operand:GPI 3 "register_operand" "r"))])]
1554   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1555   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1556   [(set_attr "type" "store_<ldpstp_sz>")]
1557 )
1558
1559 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1560   [(parallel
1561     [(set (match_operand:P 0 "register_operand" "=&k")
1562           (plus:P (match_operand:P 1 "register_operand" "0")
1563                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1564      (set (mem:GPF (plus:P (match_dup 0)
1565                    (match_dup 4)))
1566           (match_operand:GPF 2 "register_operand" "w"))
1567      (set (mem:GPF (plus:P (match_dup 0)
1568                    (match_operand:P 5 "const_int_operand" "n")))
1569           (match_operand:GPF 3 "register_operand" "w"))])]
1570   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1571   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1572   [(set_attr "type" "neon_store1_2reg<q>")]
1573 )
1574
1575 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1576   [(parallel
1577     [(set (match_operand:P 0 "register_operand" "=&k")
1578           (plus:P (match_operand:P 1 "register_operand" "0")
1579                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1580      (set (mem:TX (plus:P (match_dup 0)
1581                           (match_dup 4)))
1582           (match_operand:TX 2 "register_operand" "w"))
1583      (set (mem:TX (plus:P (match_dup 0)
1584                           (match_operand:P 5 "const_int_operand" "n")))
1585           (match_operand:TX 3 "register_operand" "w"))])]
1586   "TARGET_SIMD
1587    && INTVAL (operands[5])
1588       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1589   "stp\\t%q2, %q3, [%0, %4]!"
1590   [(set_attr "type" "neon_stp_q")]
1591 )
1592
1593 ;; -------------------------------------------------------------------
1594 ;; Sign/Zero extension
1595 ;; -------------------------------------------------------------------
1596
1597 (define_expand "<optab>sidi2"
1598   [(set (match_operand:DI 0 "register_operand")
1599         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1600   ""
1601 )
1602
1603 (define_insn "*extendsidi2_aarch64"
1604   [(set (match_operand:DI 0 "register_operand" "=r,r")
1605         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1606   ""
1607   "@
1608    sxtw\t%0, %w1
1609    ldrsw\t%0, %1"
1610   [(set_attr "type" "extend,load_4")]
1611 )
1612
1613 (define_insn "*load_pair_extendsidi2_aarch64"
1614   [(set (match_operand:DI 0 "register_operand" "=r")
1615         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1616    (set (match_operand:DI 2 "register_operand" "=r")
1617         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1618   "rtx_equal_p (XEXP (operands[3], 0),
1619                 plus_constant (Pmode,
1620                                XEXP (operands[1], 0),
1621                                GET_MODE_SIZE (SImode)))"
1622   "ldpsw\\t%0, %2, %1"
1623   [(set_attr "type" "load_8")]
1624 )
1625
1626 (define_insn "*zero_extendsidi2_aarch64"
1627   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1628         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1629   ""
1630   "@
1631    uxtw\t%0, %w1
1632    ldr\t%w0, %1
1633    fmov\t%s0, %w1
1634    ldr\t%s0, %1
1635    fmov\t%w0, %s1
1636    fmov\t%s0, %s1"
1637   [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1638    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1639 )
1640
1641 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1642   [(set (match_operand:DI 0 "register_operand" "=r,w")
1643         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1644    (set (match_operand:DI 2 "register_operand" "=r,w")
1645         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1646   "rtx_equal_p (XEXP (operands[3], 0),
1647                 plus_constant (Pmode,
1648                                XEXP (operands[1], 0),
1649                                GET_MODE_SIZE (SImode)))"
1650   "@
1651    ldp\t%w0, %w2, %1
1652    ldp\t%s0, %s2, %1"
1653   [(set_attr "type" "load_8,neon_load1_2reg")
1654    (set_attr "arch" "*,fp")]
1655 )
1656
1657 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1658   [(set (match_operand:GPI 0 "register_operand")
1659         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1660   ""
1661 )
1662
1663 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1664   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1665         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1666   ""
1667   "@
1668    sxt<SHORT:size>\t%<GPI:w>0, %w1
1669    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1670   [(set_attr "type" "extend,load_4")]
1671 )
1672
1673 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1674   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1675         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1676   ""
1677   "@
1678    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1679    ldr<SHORT:size>\t%w0, %1
1680    ldr\t%<SHORT:size>0, %1"
1681   [(set_attr "type" "logic_imm,load_4,f_loads")
1682    (set_attr "arch" "*,*,fp")]
1683 )
1684
1685 (define_expand "<optab>qihi2"
1686   [(set (match_operand:HI 0 "register_operand")
1687         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1688   ""
1689 )
1690
1691 (define_insn "*extendqihi2_aarch64"
1692   [(set (match_operand:HI 0 "register_operand" "=r,r")
1693         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1694   ""
1695   "@
1696    sxtb\t%w0, %w1
1697    ldrsb\t%w0, %1"
1698   [(set_attr "type" "extend,load_4")]
1699 )
1700
1701 (define_insn "*zero_extendqihi2_aarch64"
1702   [(set (match_operand:HI 0 "register_operand" "=r,r")
1703         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1704   ""
1705   "@
1706    and\t%w0, %w1, 255
1707    ldrb\t%w0, %1"
1708   [(set_attr "type" "logic_imm,load_4")]
1709 )
1710
1711 ;; -------------------------------------------------------------------
1712 ;; Simple arithmetic
1713 ;; -------------------------------------------------------------------
1714
1715 (define_expand "add<mode>3"
1716   [(set
1717     (match_operand:GPI 0 "register_operand" "")
1718     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1719               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1720   ""
1721 {
1722   /* If operands[1] is a subreg extract the inner RTX.  */
1723   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1724
1725   /* If the constant is too large for a single instruction and isn't frame
1726      based, split off the immediate so it is available for CSE.  */
1727   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1728       && can_create_pseudo_p ()
1729       && (!REG_P (op1)
1730          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1731     operands[2] = force_reg (<MODE>mode, operands[2]);
1732   /* Expand polynomial additions now if the destination is the stack
1733      pointer, since we don't want to use that as a temporary.  */
1734   else if (operands[0] == stack_pointer_rtx
1735            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1736     {
1737       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1738                                 operands[2], NULL_RTX, NULL_RTX);
1739       DONE;
1740     }
1741 })
1742
1743 (define_insn "*add<mode>3_aarch64"
1744   [(set
1745     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1746     (plus:GPI
1747      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1748      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1749   ""
1750   "@
1751   add\\t%<w>0, %<w>1, %2
1752   add\\t%<w>0, %<w>1, %<w>2
1753   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1754   sub\\t%<w>0, %<w>1, #%n2
1755   #
1756   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1757   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1758   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1759    (set_attr "arch" "*,*,simd,*,*,*")]
1760 )
1761
1762 ;; zero_extend version of above
1763 (define_insn "*addsi3_aarch64_uxtw"
1764   [(set
1765     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1766     (zero_extend:DI
1767      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1768               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1769   ""
1770   "@
1771   add\\t%w0, %w1, %2
1772   add\\t%w0, %w1, %w2
1773   sub\\t%w0, %w1, #%n2
1774   #"
1775   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1776 )
1777
1778 ;; If there's a free register, and we can load the constant with a
1779 ;; single instruction, do so.  This has a chance to improve scheduling.
1780 (define_peephole2
1781   [(match_scratch:GPI 3 "r")
1782    (set (match_operand:GPI 0 "register_operand")
1783         (plus:GPI
1784           (match_operand:GPI 1 "register_operand")
1785           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1786   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1787   [(set (match_dup 3) (match_dup 2))
1788    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1789 )
1790
1791 (define_peephole2
1792   [(match_scratch:SI 3 "r")
1793    (set (match_operand:DI 0 "register_operand")
1794         (zero_extend:DI
1795           (plus:SI
1796             (match_operand:SI 1 "register_operand")
1797             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1798   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1799   [(set (match_dup 3) (match_dup 2))
1800    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1801 )
1802
1803 ;; After peephole2 has had a chance to run, split any remaining long
1804 ;; additions into two add immediates.
1805 (define_split
1806   [(set (match_operand:GPI 0 "register_operand")
1807         (plus:GPI
1808           (match_operand:GPI 1 "register_operand")
1809           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1810   "epilogue_completed"
1811   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1812    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1813   {
1814     HOST_WIDE_INT i = INTVAL (operands[2]);
1815     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1816     operands[3] = GEN_INT (i - s);
1817     operands[4] = GEN_INT (s);
1818   }
1819 )
1820
1821 ;; Match addition of polynomial offsets that require one temporary, for which
1822 ;; we can use the early-clobbered destination register.  This is a separate
1823 ;; pattern so that the early clobber doesn't affect register allocation
1824 ;; for other forms of addition.  However, we still need to provide an
1825 ;; all-register alternative, in case the offset goes out of range after
1826 ;; elimination.  For completeness we might as well provide all GPR-based
1827 ;; alternatives from the main pattern.
1828 ;;
1829 ;; We don't have a pattern for additions requiring two temporaries since at
1830 ;; present LRA doesn't allow new scratches to be added during elimination.
1831 ;; Such offsets should be rare anyway.
1832 ;;
1833 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1834 ;; here would go away.  We could just handle all polynomial constants in
1835 ;; this pattern.
1836 (define_insn_and_split "*add<mode>3_poly_1"
1837   [(set
1838     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1839     (plus:GPI
1840      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1841      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1842   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1843   "@
1844   add\\t%<w>0, %<w>1, %2
1845   add\\t%<w>0, %<w>1, %<w>2
1846   sub\\t%<w>0, %<w>1, #%n2
1847   #
1848   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1849   #"
1850   "&& epilogue_completed
1851    && !reg_overlap_mentioned_p (operands[0], operands[1])
1852    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1853   [(const_int 0)]
1854   {
1855     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1856                               operands[2], operands[0], NULL_RTX);
1857     DONE;
1858   }
1859   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1860   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1861 )
1862
1863 (define_split
1864   [(set (match_operand:DI 0 "register_operand")
1865         (zero_extend:DI
1866           (plus:SI
1867             (match_operand:SI 1 "register_operand")
1868             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1869   "epilogue_completed"
1870   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1871    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1872   {
1873     HOST_WIDE_INT i = INTVAL (operands[2]);
1874     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1875     operands[3] = GEN_INT (i - s);
1876     operands[4] = GEN_INT (s);
1877     operands[5] = gen_lowpart (SImode, operands[0]);
1878   }
1879 )
1880
1881 (define_expand "addv<mode>4"
1882   [(match_operand:GPI 0 "register_operand")
1883    (match_operand:GPI 1 "register_operand")
1884    (match_operand:GPI 2 "aarch64_plus_operand")
1885    (label_ref (match_operand 3 "" ""))]
1886   ""
1887 {
1888   if (CONST_INT_P (operands[2]))
1889     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1890                                             operands[2]));
1891   else
1892     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1893   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1894
1895   DONE;
1896 })
1897
1898 (define_expand "uaddv<mode>4"
1899   [(match_operand:GPI 0 "register_operand")
1900    (match_operand:GPI 1 "register_operand")
1901    (match_operand:GPI 2 "register_operand")
1902    (label_ref (match_operand 3 "" ""))]
1903   ""
1904 {
1905   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1906   aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1907
1908   DONE;
1909 })
1910
1911 (define_expand "addti3"
1912   [(set (match_operand:TI 0 "register_operand" "")
1913         (plus:TI (match_operand:TI 1 "register_operand" "")
1914                  (match_operand:TI 2 "aarch64_reg_or_imm" "")))]
1915   ""
1916 {
1917   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1918
1919   aarch64_addti_scratch_regs (operands[1], operands[2],
1920                               &low_dest, &op1_low, &op2_low,
1921                               &high_dest, &op1_high, &op2_high);
1922
1923   if (op2_low == const0_rtx)
1924     {
1925       low_dest = op1_low;
1926       if (!aarch64_pluslong_operand (op2_high, DImode))
1927         op2_high = force_reg (DImode, op2_high);
1928       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1929     }
1930   else
1931     {
1932       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1933                                       force_reg (DImode, op2_low)));
1934       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1935                                      force_reg (DImode, op2_high)));
1936     }
1937
1938   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1939   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1940
1941   DONE;
1942 })
1943
1944 (define_expand "addvti4"
1945   [(match_operand:TI 0 "register_operand" "")
1946    (match_operand:TI 1 "register_operand" "")
1947    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1948    (label_ref (match_operand 3 "" ""))]
1949   ""
1950 {
1951   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1952
1953   aarch64_addti_scratch_regs (operands[1], operands[2],
1954                               &low_dest, &op1_low, &op2_low,
1955                               &high_dest, &op1_high, &op2_high);
1956
1957   if (op2_low == const0_rtx)
1958     {
1959       low_dest = op1_low;
1960       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1961                                       force_reg (DImode, op2_high)));
1962     }
1963   else
1964     {
1965       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1966                                       force_reg (DImode, op2_low)));
1967       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1968                                       force_reg (DImode, op2_high)));
1969     }
1970
1971   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1972   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1973
1974   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1975   DONE;
1976 })
1977
1978 (define_expand "uaddvti4"
1979   [(match_operand:TI 0 "register_operand" "")
1980    (match_operand:TI 1 "register_operand" "")
1981    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1982    (label_ref (match_operand 3 "" ""))]
1983   ""
1984 {
1985   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1986
1987   aarch64_addti_scratch_regs (operands[1], operands[2],
1988                               &low_dest, &op1_low, &op2_low,
1989                               &high_dest, &op1_high, &op2_high);
1990
1991   if (op2_low == const0_rtx)
1992     {
1993       low_dest = op1_low;
1994       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
1995                                       force_reg (DImode, op2_high)));
1996     }
1997   else
1998     {
1999       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2000                                       force_reg (DImode, op2_low)));
2001       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2002                                       force_reg (DImode, op2_high)));
2003     }
2004
2005   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2006   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2007
2008   aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2009   DONE;
2010  })
2011
2012 (define_insn "add<mode>3_compare0"
2013   [(set (reg:CC_NZ CC_REGNUM)
2014         (compare:CC_NZ
2015          (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2016                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2017          (const_int 0)))
2018    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2019         (plus:GPI (match_dup 1) (match_dup 2)))]
2020   ""
2021   "@
2022   adds\\t%<w>0, %<w>1, %<w>2
2023   adds\\t%<w>0, %<w>1, %2
2024   subs\\t%<w>0, %<w>1, #%n2"
2025   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2026 )
2027
2028 ;; zero_extend version of above
2029 (define_insn "*addsi3_compare0_uxtw"
2030   [(set (reg:CC_NZ CC_REGNUM)
2031         (compare:CC_NZ
2032          (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2033                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2034          (const_int 0)))
2035    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2036         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2037   ""
2038   "@
2039   adds\\t%w0, %w1, %w2
2040   adds\\t%w0, %w1, %2
2041   subs\\t%w0, %w1, #%n2"
2042   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2043 )
2044
2045 (define_insn "*add<mode>3_compareC_cconly"
2046   [(set (reg:CC_C CC_REGNUM)
2047         (compare:CC_C
2048           (plus:GPI
2049             (match_operand:GPI 0 "register_operand" "r,r,r")
2050             (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2051           (match_dup 0)))]
2052   ""
2053   "@
2054   cmn\\t%<w>0, %<w>1
2055   cmn\\t%<w>0, %1
2056   cmp\\t%<w>0, #%n1"
2057   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2058 )
2059
2060 (define_insn "add<mode>3_compareC"
2061   [(set (reg:CC_C CC_REGNUM)
2062         (compare:CC_C
2063           (plus:GPI
2064             (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2065             (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2066           (match_dup 1)))
2067    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2068         (plus:GPI (match_dup 1) (match_dup 2)))]
2069   ""
2070   "@
2071   adds\\t%<w>0, %<w>1, %<w>2
2072   adds\\t%<w>0, %<w>1, %2
2073   subs\\t%<w>0, %<w>1, #%n2"
2074   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2075 )
2076
2077 (define_insn "*add<mode>3_compareV_cconly_imm"
2078   [(set (reg:CC_V CC_REGNUM)
2079         (compare:CC_V
2080           (plus:<DWI>
2081             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2082             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2083           (sign_extend:<DWI>
2084            (plus:GPI
2085             (match_dup 0)
2086             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2087   "INTVAL (operands[1]) == INTVAL (operands[2])"
2088   "@
2089   cmn\\t%<w>0, %<w>1
2090   cmp\\t%<w>0, #%n1"
2091   [(set_attr "type" "alus_imm")]
2092 )
2093
2094 (define_insn "*add<mode>3_compareV_cconly"
2095   [(set (reg:CC_V CC_REGNUM)
2096         (compare:CC_V
2097           (plus:<DWI>
2098             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2099             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2100           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2101   ""
2102   "cmn\\t%<w>0, %<w>1"
2103   [(set_attr "type" "alus_sreg")]
2104 )
2105
2106 (define_insn "add<mode>3_compareV_imm"
2107   [(set (reg:CC_V CC_REGNUM)
2108         (compare:CC_V
2109           (plus:<DWI>
2110             (sign_extend:<DWI>
2111               (match_operand:GPI 1 "register_operand" "rk,rk"))
2112             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2113           (sign_extend:<DWI>
2114             (plus:GPI (match_dup 1) (match_dup 2)))))
2115    (set (match_operand:GPI 0 "register_operand" "=r,r")
2116         (plus:GPI (match_dup 1) (match_dup 2)))]
2117    ""
2118    "@
2119    adds\\t%<w>0, %<w>1, %<w>2
2120    subs\\t%<w>0, %<w>1, #%n2"
2121   [(set_attr "type" "alus_imm,alus_imm")]
2122 )
2123
2124 (define_insn "add<mode>3_compareV"
2125   [(set (reg:CC_V CC_REGNUM)
2126         (compare:CC_V
2127           (plus:<DWI>
2128             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2129             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2130           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2131    (set (match_operand:GPI 0 "register_operand" "=r")
2132         (plus:GPI (match_dup 1) (match_dup 2)))]
2133   ""
2134   "adds\\t%<w>0, %<w>1, %<w>2"
2135   [(set_attr "type" "alus_sreg")]
2136 )
2137
2138 (define_insn "*adds_shift_imm_<mode>"
2139   [(set (reg:CC_NZ CC_REGNUM)
2140         (compare:CC_NZ
2141          (plus:GPI (ASHIFT:GPI
2142                     (match_operand:GPI 1 "register_operand" "r")
2143                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2144                    (match_operand:GPI 3 "register_operand" "r"))
2145          (const_int 0)))
2146    (set (match_operand:GPI 0 "register_operand" "=r")
2147         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2148                   (match_dup 3)))]
2149   ""
2150   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2151   [(set_attr "type" "alus_shift_imm")]
2152 )
2153
2154 (define_insn "*subs_shift_imm_<mode>"
2155   [(set (reg:CC_NZ CC_REGNUM)
2156         (compare:CC_NZ
2157          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2158                     (ASHIFT:GPI
2159                      (match_operand:GPI 2 "register_operand" "r")
2160                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2161          (const_int 0)))
2162    (set (match_operand:GPI 0 "register_operand" "=r")
2163         (minus:GPI (match_dup 1)
2164                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2165   ""
2166   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2167   [(set_attr "type" "alus_shift_imm")]
2168 )
2169
2170 (define_insn "*adds_mul_imm_<mode>"
2171   [(set (reg:CC_NZ CC_REGNUM)
2172         (compare:CC_NZ
2173          (plus:GPI (mult:GPI
2174                     (match_operand:GPI 1 "register_operand" "r")
2175                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2176                    (match_operand:GPI 3 "register_operand" "r"))
2177          (const_int 0)))
2178    (set (match_operand:GPI 0 "register_operand" "=r")
2179         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2180                   (match_dup 3)))]
2181   ""
2182   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2183   [(set_attr "type" "alus_shift_imm")]
2184 )
2185
2186 (define_insn "*subs_mul_imm_<mode>"
2187   [(set (reg:CC_NZ CC_REGNUM)
2188         (compare:CC_NZ
2189          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2190                     (mult:GPI
2191                      (match_operand:GPI 2 "register_operand" "r")
2192                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2193          (const_int 0)))
2194    (set (match_operand:GPI 0 "register_operand" "=r")
2195         (minus:GPI (match_dup 1)
2196                    (mult:GPI (match_dup 2) (match_dup 3))))]
2197   ""
2198   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2199   [(set_attr "type" "alus_shift_imm")]
2200 )
2201
2202 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2203   [(set (reg:CC_NZ CC_REGNUM)
2204         (compare:CC_NZ
2205          (plus:GPI
2206           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2207           (match_operand:GPI 2 "register_operand" "rk"))
2208         (const_int 0)))
2209    (set (match_operand:GPI 0 "register_operand" "=r")
2210         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2211   ""
2212   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2213   [(set_attr "type" "alus_ext")]
2214 )
2215
2216 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2217   [(set (reg:CC_NZ CC_REGNUM)
2218         (compare:CC_NZ
2219          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2220                     (ANY_EXTEND:GPI
2221                      (match_operand:ALLX 2 "register_operand" "r")))
2222         (const_int 0)))
2223    (set (match_operand:GPI 0 "register_operand" "=r")
2224         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2225   ""
2226   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2227   [(set_attr "type" "alus_ext")]
2228 )
2229
2230 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2231   [(set (reg:CC_NZ CC_REGNUM)
2232         (compare:CC_NZ
2233          (plus:GPI (ashift:GPI
2234                     (ANY_EXTEND:GPI
2235                      (match_operand:ALLX 1 "register_operand" "r"))
2236                     (match_operand 2 "aarch64_imm3" "Ui3"))
2237                    (match_operand:GPI 3 "register_operand" "rk"))
2238          (const_int 0)))
2239    (set (match_operand:GPI 0 "register_operand" "=rk")
2240         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2241                               (match_dup 2))
2242                   (match_dup 3)))]
2243   ""
2244   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2245   [(set_attr "type" "alus_ext")]
2246 )
2247
2248 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2249   [(set (reg:CC_NZ CC_REGNUM)
2250         (compare:CC_NZ
2251          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2252                     (ashift:GPI
2253                      (ANY_EXTEND:GPI
2254                       (match_operand:ALLX 2 "register_operand" "r"))
2255                      (match_operand 3 "aarch64_imm3" "Ui3")))
2256          (const_int 0)))
2257    (set (match_operand:GPI 0 "register_operand" "=rk")
2258         (minus:GPI (match_dup 1)
2259                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2260                                (match_dup 3))))]
2261   ""
2262   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2263   [(set_attr "type" "alus_ext")]
2264 )
2265
2266 (define_insn "*adds_<optab><mode>_multp2"
2267   [(set (reg:CC_NZ CC_REGNUM)
2268         (compare:CC_NZ
2269          (plus:GPI (ANY_EXTRACT:GPI
2270                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2271                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2272                     (match_operand 3 "const_int_operand" "n")
2273                     (const_int 0))
2274                    (match_operand:GPI 4 "register_operand" "rk"))
2275         (const_int 0)))
2276    (set (match_operand:GPI 0 "register_operand" "=r")
2277         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2278                                    (match_dup 3)
2279                                    (const_int 0))
2280                   (match_dup 4)))]
2281   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2282   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2283   [(set_attr "type" "alus_ext")]
2284 )
2285
2286 (define_insn "*subs_<optab><mode>_multp2"
2287   [(set (reg:CC_NZ CC_REGNUM)
2288         (compare:CC_NZ
2289          (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2290                     (ANY_EXTRACT:GPI
2291                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2292                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2293                      (match_operand 3 "const_int_operand" "n")
2294                      (const_int 0)))
2295         (const_int 0)))
2296    (set (match_operand:GPI 0 "register_operand" "=r")
2297         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2298                                   (mult:GPI (match_dup 1) (match_dup 2))
2299                                   (match_dup 3)
2300                                   (const_int 0))))]
2301   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2302   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2303   [(set_attr "type" "alus_ext")]
2304 )
2305
2306 (define_insn "*add<mode>3nr_compare0"
2307   [(set (reg:CC_NZ CC_REGNUM)
2308         (compare:CC_NZ
2309          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2310                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2311          (const_int 0)))]
2312   ""
2313   "@
2314   cmn\\t%<w>0, %<w>1
2315   cmn\\t%<w>0, %1
2316   cmp\\t%<w>0, #%n1"
2317   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2318 )
2319
2320 (define_insn "aarch64_sub<mode>_compare0"
2321   [(set (reg:CC_NZ CC_REGNUM)
2322         (compare:CC_NZ
2323          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2324                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2325          (const_int 0)))]
2326   ""
2327   "cmp\\t%<w>0, %<w>1"
2328   [(set_attr "type" "alus_sreg")]
2329 )
2330
2331 (define_insn "*compare_neg<mode>"
2332   [(set (reg:CC_Z CC_REGNUM)
2333         (compare:CC_Z
2334          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2335          (match_operand:GPI 1 "register_operand" "r")))]
2336   ""
2337   "cmn\\t%<w>1, %<w>0"
2338   [(set_attr "type" "alus_sreg")]
2339 )
2340
2341 (define_insn "*add_<shift>_<mode>"
2342   [(set (match_operand:GPI 0 "register_operand" "=r")
2343         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2344                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2345                   (match_operand:GPI 3 "register_operand" "r")))]
2346   ""
2347   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2348   [(set_attr "type" "alu_shift_imm")]
2349 )
2350
2351 ;; zero_extend version of above
2352 (define_insn "*add_<shift>_si_uxtw"
2353   [(set (match_operand:DI 0 "register_operand" "=r")
2354         (zero_extend:DI
2355          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2356                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2357                   (match_operand:SI 3 "register_operand" "r"))))]
2358   ""
2359   "add\\t%w0, %w3, %w1, <shift> %2"
2360   [(set_attr "type" "alu_shift_imm")]
2361 )
2362
2363 (define_insn "*add_mul_imm_<mode>"
2364   [(set (match_operand:GPI 0 "register_operand" "=r")
2365         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2366                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2367                   (match_operand:GPI 3 "register_operand" "r")))]
2368   ""
2369   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2370   [(set_attr "type" "alu_shift_imm")]
2371 )
2372
2373 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2374   [(set (match_operand:GPI 0 "register_operand" "=rk")
2375         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2376                   (match_operand:GPI 2 "register_operand" "r")))]
2377   ""
2378   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2379   [(set_attr "type" "alu_ext")]
2380 )
2381
2382 ;; zero_extend version of above
2383 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2384   [(set (match_operand:DI 0 "register_operand" "=rk")
2385         (zero_extend:DI
2386          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2387                   (match_operand:GPI 2 "register_operand" "r"))))]
2388   ""
2389   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2390   [(set_attr "type" "alu_ext")]
2391 )
2392
2393 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2394   [(set (match_operand:GPI 0 "register_operand" "=rk")
2395         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2396                                (match_operand:ALLX 1 "register_operand" "r"))
2397                               (match_operand 2 "aarch64_imm3" "Ui3"))
2398                   (match_operand:GPI 3 "register_operand" "r")))]
2399   ""
2400   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2401   [(set_attr "type" "alu_ext")]
2402 )
2403
2404 ;; zero_extend version of above
2405 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2406   [(set (match_operand:DI 0 "register_operand" "=rk")
2407         (zero_extend:DI
2408          (plus:SI (ashift:SI (ANY_EXTEND:SI
2409                               (match_operand:SHORT 1 "register_operand" "r"))
2410                              (match_operand 2 "aarch64_imm3" "Ui3"))
2411                   (match_operand:SI 3 "register_operand" "r"))))]
2412   ""
2413   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2414   [(set_attr "type" "alu_ext")]
2415 )
2416
2417 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2418   [(set (match_operand:GPI 0 "register_operand" "=rk")
2419         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2420                              (match_operand:ALLX 1 "register_operand" "r"))
2421                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2422                   (match_operand:GPI 3 "register_operand" "r")))]
2423   ""
2424   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2425   [(set_attr "type" "alu_ext")]
2426 )
2427
2428 ;; zero_extend version of above
2429 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2430   [(set (match_operand:DI 0 "register_operand" "=rk")
2431         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2432                              (match_operand:SHORT 1 "register_operand" "r"))
2433                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2434                   (match_operand:SI 3 "register_operand" "r"))))]
2435   ""
2436   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2437   [(set_attr "type" "alu_ext")]
2438 )
2439
2440 (define_insn "*add_<optab><mode>_multp2"
2441   [(set (match_operand:GPI 0 "register_operand" "=rk")
2442         (plus:GPI (ANY_EXTRACT:GPI
2443                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2444                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2445                    (match_operand 3 "const_int_operand" "n")
2446                    (const_int 0))
2447                   (match_operand:GPI 4 "register_operand" "r")))]
2448   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2449   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2450   [(set_attr "type" "alu_ext")]
2451 )
2452
2453 ;; zero_extend version of above
2454 (define_insn "*add_<optab>si_multp2_uxtw"
2455   [(set (match_operand:DI 0 "register_operand" "=rk")
2456         (zero_extend:DI
2457          (plus:SI (ANY_EXTRACT:SI
2458                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2459                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2460                    (match_operand 3 "const_int_operand" "n")
2461                    (const_int 0))
2462                   (match_operand:SI 4 "register_operand" "r"))))]
2463   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2464   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2465   [(set_attr "type" "alu_ext")]
2466 )
2467
2468 (define_expand "add<mode>3_carryin"
2469   [(set (match_operand:GPI 0 "register_operand")
2470         (plus:GPI
2471           (plus:GPI
2472             (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2473             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2474           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2475    ""
2476    ""
2477 )
2478
2479 ;; Note that add with carry with two zero inputs is matched by cset,
2480 ;; and that add with carry with one zero input is matched by cinc.
2481
2482 (define_insn "*add<mode>3_carryin"
2483   [(set (match_operand:GPI 0 "register_operand" "=r")
2484         (plus:GPI
2485           (plus:GPI
2486             (match_operand:GPI 3 "aarch64_carry_operation" "")
2487             (match_operand:GPI 1 "register_operand" "r"))
2488           (match_operand:GPI 2 "register_operand" "r")))]
2489    ""
2490    "adc\\t%<w>0, %<w>1, %<w>2"
2491   [(set_attr "type" "adc_reg")]
2492 )
2493
2494 ;; zero_extend version of above
2495 (define_insn "*addsi3_carryin_uxtw"
2496   [(set (match_operand:DI 0 "register_operand" "=r")
2497         (zero_extend:DI
2498           (plus:SI
2499             (plus:SI
2500               (match_operand:SI 3 "aarch64_carry_operation" "")
2501               (match_operand:SI 1 "register_operand" "r"))
2502             (match_operand:SI 2 "register_operand" "r"))))]
2503    ""
2504    "adc\\t%w0, %w1, %w2"
2505   [(set_attr "type" "adc_reg")]
2506 )
2507
2508 (define_expand "add<mode>3_carryinC"
2509   [(parallel
2510      [(set (match_dup 3)
2511            (compare:CC_ADC
2512              (plus:<DWI>
2513                (plus:<DWI>
2514                  (match_dup 4)
2515                  (zero_extend:<DWI>
2516                    (match_operand:GPI 1 "register_operand" "")))
2517                (zero_extend:<DWI>
2518                  (match_operand:GPI 2 "register_operand" "")))
2519              (match_dup 6)))
2520       (set (match_operand:GPI 0 "register_operand")
2521            (plus:GPI
2522              (plus:GPI (match_dup 5) (match_dup 1))
2523              (match_dup 2)))])]
2524    ""
2525 {
2526   operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2527   rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2528   operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2529   operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2530   operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2531                                       << GET_MODE_BITSIZE (<MODE>mode),
2532                                       TImode);
2533 })
2534
2535 (define_insn "*add<mode>3_carryinC_zero"
2536   [(set (reg:CC_ADC CC_REGNUM)
2537         (compare:CC_ADC
2538           (plus:<DWI>
2539             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2540             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2541           (match_operand 4 "const_scalar_int_operand" "")))
2542    (set (match_operand:GPI 0 "register_operand" "=r")
2543         (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2544                   (match_dup 1)))]
2545   "rtx_mode_t (operands[4], <DWI>mode)
2546    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2547    "adcs\\t%<w>0, %<w>1, <w>zr"
2548   [(set_attr "type" "adc_reg")]
2549 )
2550
2551 (define_insn "*add<mode>3_carryinC"
2552   [(set (reg:CC_ADC CC_REGNUM)
2553         (compare:CC_ADC
2554           (plus:<DWI>
2555             (plus:<DWI>
2556               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2557               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2558             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2559           (match_operand 5 "const_scalar_int_operand" "")))
2560    (set (match_operand:GPI 0 "register_operand" "=r")
2561         (plus:GPI
2562           (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2563                     (match_dup 1))
2564           (match_dup 2)))]
2565   "rtx_mode_t (operands[5], <DWI>mode)
2566    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2567    "adcs\\t%<w>0, %<w>1, %<w>2"
2568   [(set_attr "type" "adc_reg")]
2569 )
2570
2571 (define_expand "add<mode>3_carryinV"
2572   [(parallel
2573      [(set (reg:CC_V CC_REGNUM)
2574            (compare:CC_V
2575              (plus:<DWI>
2576                (plus:<DWI>
2577                  (match_dup 3)
2578                  (sign_extend:<DWI>
2579                    (match_operand:GPI 1 "register_operand" "")))
2580                (sign_extend:<DWI>
2581                  (match_operand:GPI 2 "register_operand" "")))
2582            (sign_extend:<DWI>
2583              (plus:GPI
2584                (plus:GPI (match_dup 4) (match_dup 1))
2585                (match_dup 2)))))
2586       (set (match_operand:GPI 0 "register_operand")
2587            (plus:GPI
2588              (plus:GPI (match_dup 4) (match_dup 1))
2589              (match_dup 2)))])]
2590    ""
2591 {
2592   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2593   operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2594   operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2595 })
2596
2597 (define_insn "*add<mode>3_carryinV_zero"
2598   [(set (reg:CC_V CC_REGNUM)
2599         (compare:CC_V
2600           (plus:<DWI>
2601             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2602             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2603           (sign_extend:<DWI>
2604             (plus:GPI
2605               (match_operand:GPI 3 "aarch64_carry_operation" "")
2606               (match_dup 1)))))
2607    (set (match_operand:GPI 0 "register_operand" "=r")
2608         (plus:GPI (match_dup 3) (match_dup 1)))]
2609    ""
2610    "adcs\\t%<w>0, %<w>1, <w>zr"
2611   [(set_attr "type" "adc_reg")]
2612 )
2613
2614 (define_insn "*add<mode>3_carryinV"
2615   [(set (reg:CC_V CC_REGNUM)
2616         (compare:CC_V
2617           (plus:<DWI>
2618             (plus:<DWI>
2619               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2620               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2621             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2622           (sign_extend:<DWI>
2623             (plus:GPI
2624               (plus:GPI
2625                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2626                 (match_dup 1))
2627               (match_dup 2)))))
2628    (set (match_operand:GPI 0 "register_operand" "=r")
2629         (plus:GPI
2630           (plus:GPI (match_dup 4) (match_dup 1))
2631           (match_dup 2)))]
2632    ""
2633    "adcs\\t%<w>0, %<w>1, %<w>2"
2634   [(set_attr "type" "adc_reg")]
2635 )
2636
2637 (define_insn "*add_uxt<mode>_shift2"
2638   [(set (match_operand:GPI 0 "register_operand" "=rk")
2639         (plus:GPI (and:GPI
2640                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2641                                (match_operand 2 "aarch64_imm3" "Ui3"))
2642                    (match_operand 3 "const_int_operand" "n"))
2643                   (match_operand:GPI 4 "register_operand" "r")))]
2644   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2645   "*
2646   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2647                                            INTVAL (operands[3])));
2648   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2649   [(set_attr "type" "alu_ext")]
2650 )
2651
2652 ;; zero_extend version of above
2653 (define_insn "*add_uxtsi_shift2_uxtw"
2654   [(set (match_operand:DI 0 "register_operand" "=rk")
2655         (zero_extend:DI
2656          (plus:SI (and:SI
2657                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2658                               (match_operand 2 "aarch64_imm3" "Ui3"))
2659                    (match_operand 3 "const_int_operand" "n"))
2660                   (match_operand:SI 4 "register_operand" "r"))))]
2661   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2662   "*
2663   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2664                                            INTVAL (operands[3])));
2665   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2666   [(set_attr "type" "alu_ext")]
2667 )
2668
2669 (define_insn "*add_uxt<mode>_multp2"
2670   [(set (match_operand:GPI 0 "register_operand" "=rk")
2671         (plus:GPI (and:GPI
2672                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2673                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2674                    (match_operand 3 "const_int_operand" "n"))
2675                   (match_operand:GPI 4 "register_operand" "r")))]
2676   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2677   "*
2678   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2679                                            INTVAL (operands[3])));
2680   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2681   [(set_attr "type" "alu_ext")]
2682 )
2683
2684 ;; zero_extend version of above
2685 (define_insn "*add_uxtsi_multp2_uxtw"
2686   [(set (match_operand:DI 0 "register_operand" "=rk")
2687         (zero_extend:DI
2688          (plus:SI (and:SI
2689                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2690                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2691                    (match_operand 3 "const_int_operand" "n"))
2692                   (match_operand:SI 4 "register_operand" "r"))))]
2693   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2694   "*
2695   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2696                                            INTVAL (operands[3])));
2697   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2698   [(set_attr "type" "alu_ext")]
2699 )
2700
2701 (define_insn "subsi3"
2702   [(set (match_operand:SI 0 "register_operand" "=rk")
2703         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2704                   (match_operand:SI 2 "register_operand" "r")))]
2705   ""
2706   "sub\\t%w0, %w1, %w2"
2707   [(set_attr "type" "alu_sreg")]
2708 )
2709
2710 ;; zero_extend version of above
2711 (define_insn "*subsi3_uxtw"
2712   [(set (match_operand:DI 0 "register_operand" "=rk")
2713         (zero_extend:DI
2714          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2715                    (match_operand:SI 2 "register_operand" "r"))))]
2716   ""
2717   "sub\\t%w0, %w1, %w2"
2718   [(set_attr "type" "alu_sreg")]
2719 )
2720
2721 (define_insn "subdi3"
2722   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2723         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2724                   (match_operand:DI 2 "register_operand" "r,w")))]
2725   ""
2726   "@
2727    sub\\t%x0, %x1, %x2
2728    sub\\t%d0, %d1, %d2"
2729   [(set_attr "type" "alu_sreg, neon_sub")
2730    (set_attr "arch" "*,simd")]
2731 )
2732
2733 (define_expand "subv<GPI:mode>4"
2734   [(match_operand:GPI 0 "register_operand")
2735    (match_operand:GPI 1 "register_operand")
2736    (match_operand:GPI 2 "aarch64_plus_operand")
2737    (label_ref (match_operand 3 "" ""))]
2738   ""
2739 {
2740   if (CONST_INT_P (operands[2]))
2741     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2742   else
2743     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2744   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2745
2746   DONE;
2747 })
2748
2749 (define_insn "subv<GPI:mode>_insn"
2750   [(set (reg:CC_V CC_REGNUM)
2751         (compare:CC_V
2752          (sign_extend:<DWI>
2753           (minus:GPI
2754            (match_operand:GPI 1 "register_operand" "rk")
2755            (match_operand:GPI 2 "register_operand" "r")))
2756          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2757                       (sign_extend:<DWI> (match_dup 2)))))
2758    (set (match_operand:GPI 0 "register_operand" "=r")
2759         (minus:GPI (match_dup 1) (match_dup 2)))]
2760   ""
2761   "subs\\t%<w>0, %<w>1, %<w>2"
2762   [(set_attr "type" "alus_sreg")]
2763 )
2764
2765 (define_insn "subv<GPI:mode>_imm"
2766   [(set (reg:CC_V CC_REGNUM)
2767         (compare:CC_V
2768          (sign_extend:<DWI>
2769           (minus:GPI
2770            (match_operand:GPI 1 "register_operand" "rk,rk")
2771            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2772          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2773                       (match_dup 2))))
2774    (set (match_operand:GPI 0 "register_operand" "=r,r")
2775         (minus:GPI (match_dup 1) (match_dup 2)))]
2776   ""
2777   "@
2778    subs\\t%<w>0, %<w>1, %2
2779    adds\\t%<w>0, %<w>1, #%n2"
2780   [(set_attr "type" "alus_sreg")]
2781 )
2782
2783 (define_expand "negv<GPI:mode>3"
2784   [(match_operand:GPI 0 "register_operand")
2785    (match_operand:GPI 1 "register_operand")
2786    (label_ref (match_operand 2 "" ""))]
2787   ""
2788   {
2789     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2790     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2791
2792     DONE;
2793   }
2794 )
2795
2796 (define_insn "negv<GPI:mode>_insn"
2797   [(set (reg:CC_V CC_REGNUM)
2798         (compare:CC_V
2799          (sign_extend:<DWI>
2800           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2801          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2802    (set (match_operand:GPI 0 "register_operand" "=r")
2803         (neg:GPI (match_dup 1)))]
2804   ""
2805   "negs\\t%<w>0, %<w>1"
2806   [(set_attr "type" "alus_sreg")]
2807 )
2808
2809 (define_insn "negv<GPI:mode>_cmp_only"
2810   [(set (reg:CC_V CC_REGNUM)
2811         (compare:CC_V
2812          (sign_extend:<DWI>
2813           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2814          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2815   ""
2816   "negs\\t%<w>zr, %<w>0"
2817   [(set_attr "type" "alus_sreg")]
2818 )
2819
2820 (define_insn "*cmpv<GPI:mode>_insn"
2821   [(set (reg:CC_V CC_REGNUM)
2822         (compare:CC_V
2823          (sign_extend:<DWI>
2824           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2825                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2826          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2827                     (sign_extend:<DWI> (match_dup 1)))))]
2828   ""
2829   "@
2830    cmp\\t%<w>0, %<w>1
2831    cmp\\t%<w>0, %1
2832    cmp\\t%<w>0, #%n1"
2833   [(set_attr "type" "alus_sreg")]
2834 )
2835
2836 (define_expand "usubv<mode>4"
2837   [(match_operand:GPI 0 "register_operand")
2838    (match_operand:GPI 1 "aarch64_reg_or_zero")
2839    (match_operand:GPI 2 "aarch64_reg_or_zero")
2840    (label_ref (match_operand 3 "" ""))]
2841   ""
2842 {
2843   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2844   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2845
2846   DONE;
2847 })
2848
2849 (define_expand "subti3"
2850   [(set (match_operand:TI 0 "register_operand" "")
2851         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero" "")
2852                   (match_operand:TI 2 "register_operand" "")))]
2853   ""
2854 {
2855   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2856
2857   aarch64_subvti_scratch_regs (operands[1], operands[2],
2858                                &low_dest, &op1_low, &op2_low,
2859                                &high_dest, &op1_high, &op2_high);
2860
2861   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2862   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2863
2864   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2865   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2866   DONE;
2867 })
2868
2869 (define_expand "subvti4"
2870   [(match_operand:TI 0 "register_operand")
2871    (match_operand:TI 1 "register_operand")
2872    (match_operand:TI 2 "aarch64_reg_or_imm")
2873    (label_ref (match_operand 3 "" ""))]
2874   ""
2875 {
2876   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2877
2878   aarch64_subvti_scratch_regs (operands[1], operands[2],
2879                                &low_dest, &op1_low, &op2_low,
2880                                &high_dest, &op1_high, &op2_high);
2881   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2882                          high_dest, op1_high, op2_high, false);
2883
2884   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2885   DONE;
2886 })
2887
2888 (define_expand "usubvti4"
2889   [(match_operand:TI 0 "register_operand")
2890    (match_operand:TI 1 "register_operand")
2891    (match_operand:TI 2 "aarch64_reg_or_imm")
2892    (label_ref (match_operand 3 "" ""))]
2893   ""
2894 {
2895   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2896
2897   aarch64_subvti_scratch_regs (operands[1], operands[2],
2898                                     &low_dest, &op1_low, &op2_low,
2899                                &high_dest, &op1_high, &op2_high);
2900   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2901                          high_dest, op1_high, op2_high, true);
2902
2903   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2904   DONE;
2905 })
2906
2907 (define_expand "negvti3"
2908   [(match_operand:TI 0 "register_operand")
2909    (match_operand:TI 1 "register_operand")
2910    (label_ref (match_operand 2 "" ""))]
2911   ""
2912   {
2913     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2914                                    gen_lowpart (DImode, operands[1])));
2915     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2916                                     gen_highpart (DImode, operands[1])));
2917     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2918
2919     DONE;
2920   }
2921 )
2922
2923 (define_insn "negdi_carryout"
2924   [(set (reg:CC CC_REGNUM)
2925         (compare:CC
2926          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2927    (set (match_operand:DI 0 "register_operand" "=r")
2928         (neg:DI (match_dup 1)))]
2929   ""
2930   "negs\\t%0, %1"
2931   [(set_attr "type" "alus_sreg")]
2932 )
2933
2934 (define_insn "negvdi_carryinV"
2935   [(set (reg:CC_V CC_REGNUM)
2936         (compare:CC_V
2937          (neg:TI (plus:TI
2938                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2939                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2940          (sign_extend:TI
2941           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2942                            (match_dup 1))))))
2943    (set (match_operand:DI 0 "register_operand" "=r")
2944         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2945                          (match_dup 1))))]
2946   ""
2947   "ngcs\\t%0, %1"
2948   [(set_attr "type" "alus_sreg")]
2949 )
2950
2951 (define_insn "*sub<mode>3_compare0"
2952   [(set (reg:CC_NZ CC_REGNUM)
2953         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2954                                   (match_operand:GPI 2 "register_operand" "r"))
2955                        (const_int 0)))
2956    (set (match_operand:GPI 0 "register_operand" "=r")
2957         (minus:GPI (match_dup 1) (match_dup 2)))]
2958   ""
2959   "subs\\t%<w>0, %<w>1, %<w>2"
2960   [(set_attr "type" "alus_sreg")]
2961 )
2962
2963 ;; zero_extend version of above
2964 (define_insn "*subsi3_compare0_uxtw"
2965   [(set (reg:CC_NZ CC_REGNUM)
2966         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
2967                                  (match_operand:SI 2 "register_operand" "r"))
2968                        (const_int 0)))
2969    (set (match_operand:DI 0 "register_operand" "=r")
2970         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2971   ""
2972   "subs\\t%w0, %w1, %w2"
2973   [(set_attr "type" "alus_sreg")]
2974 )
2975
2976 (define_insn "sub<mode>3_compare1_imm"
2977   [(set (reg:CC CC_REGNUM)
2978         (compare:CC
2979           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
2980           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2981    (set (match_operand:GPI 0 "register_operand" "=r,r")
2982         (plus:GPI
2983           (match_dup 1)
2984           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2985   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2986   "@
2987   subs\\t%<w>0, %<w>1, %2
2988   adds\\t%<w>0, %<w>1, #%n2"
2989   [(set_attr "type" "alus_imm")]
2990 )
2991
2992 (define_insn "sub<mode>3_compare1"
2993   [(set (reg:CC CC_REGNUM)
2994         (compare:CC
2995           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
2996           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2997    (set (match_operand:GPI 0 "register_operand" "=r")
2998         (minus:GPI (match_dup 1) (match_dup 2)))]
2999   ""
3000   "subs\\t%<w>0, %<w>1, %<w>2"
3001   [(set_attr "type" "alus_sreg")]
3002 )
3003
3004 (define_peephole2
3005   [(set (match_operand:GPI 0 "aarch64_general_reg")
3006         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3007                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3008    (set (reg:CC CC_REGNUM)
3009         (compare:CC
3010           (match_dup 1)
3011           (match_dup 2)))]
3012   "!reg_overlap_mentioned_p (operands[0], operands[1])
3013    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3014   [(const_int 0)]
3015   {
3016     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3017                                          operands[2]));
3018     DONE;
3019   }
3020 )
3021
3022 ;; Same as the above peephole but with the compare and minus in
3023 ;; swapped order.  The restriction on overlap between operand 0
3024 ;; and operands 1 and 2 doesn't apply here.
3025 (define_peephole2
3026   [(set (reg:CC CC_REGNUM)
3027         (compare:CC
3028           (match_operand:GPI 1 "aarch64_reg_or_zero")
3029           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3030    (set (match_operand:GPI 0 "aarch64_general_reg")
3031         (minus:GPI (match_dup 1)
3032                    (match_dup 2)))]
3033   ""
3034   [(const_int 0)]
3035   {
3036     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3037                                          operands[2]));
3038     DONE;
3039   }
3040 )
3041
3042 (define_peephole2
3043   [(set (match_operand:GPI 0 "aarch64_general_reg")
3044         (plus:GPI (match_operand:GPI 1 "register_operand")
3045                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3046    (set (reg:CC CC_REGNUM)
3047         (compare:CC
3048           (match_dup 1)
3049           (match_operand:GPI 3 "const_int_operand")))]
3050   "!reg_overlap_mentioned_p (operands[0], operands[1])
3051    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3052   [(const_int 0)]
3053   {
3054     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3055                                          operands[3], operands[2]));
3056     DONE;
3057   }
3058 )
3059
3060 ;; Same as the above peephole but with the compare and minus in
3061 ;; swapped order.  The restriction on overlap between operand 0
3062 ;; and operands 1 doesn't apply here.
3063 (define_peephole2
3064   [(set (reg:CC CC_REGNUM)
3065         (compare:CC
3066           (match_operand:GPI 1 "register_operand")
3067           (match_operand:GPI 3 "const_int_operand")))
3068    (set (match_operand:GPI 0 "aarch64_general_reg")
3069         (plus:GPI (match_dup 1)
3070                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3071   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3072   [(const_int 0)]
3073   {
3074     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3075                                          operands[3], operands[2]));
3076     DONE;
3077   }
3078 )
3079
3080 (define_insn "*sub_<shift>_<mode>"
3081   [(set (match_operand:GPI 0 "register_operand" "=r")
3082         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3083                    (ASHIFT:GPI
3084                     (match_operand:GPI 1 "register_operand" "r")
3085                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3086   ""
3087   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3088   [(set_attr "type" "alu_shift_imm")]
3089 )
3090
3091 ;; zero_extend version of above
3092 (define_insn "*sub_<shift>_si_uxtw"
3093   [(set (match_operand:DI 0 "register_operand" "=r")
3094         (zero_extend:DI
3095          (minus:SI (match_operand:SI 3 "register_operand" "r")
3096                    (ASHIFT:SI
3097                     (match_operand:SI 1 "register_operand" "r")
3098                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3099   ""
3100   "sub\\t%w0, %w3, %w1, <shift> %2"
3101   [(set_attr "type" "alu_shift_imm")]
3102 )
3103
3104 (define_insn "*sub_mul_imm_<mode>"
3105   [(set (match_operand:GPI 0 "register_operand" "=r")
3106         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3107                    (mult:GPI
3108                     (match_operand:GPI 1 "register_operand" "r")
3109                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3110   ""
3111   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3112   [(set_attr "type" "alu_shift_imm")]
3113 )
3114
3115 ;; zero_extend version of above
3116 (define_insn "*sub_mul_imm_si_uxtw"
3117   [(set (match_operand:DI 0 "register_operand" "=r")
3118         (zero_extend:DI
3119          (minus:SI (match_operand:SI 3 "register_operand" "r")
3120                    (mult:SI
3121                     (match_operand:SI 1 "register_operand" "r")
3122                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3123   ""
3124   "sub\\t%w0, %w3, %w1, lsl %p2"
3125   [(set_attr "type" "alu_shift_imm")]
3126 )
3127
3128 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3129   [(set (match_operand:GPI 0 "register_operand" "=rk")
3130         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3131                    (ANY_EXTEND:GPI
3132                     (match_operand:ALLX 2 "register_operand" "r"))))]
3133   ""
3134   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3135   [(set_attr "type" "alu_ext")]
3136 )
3137
3138 ;; zero_extend version of above
3139 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3140   [(set (match_operand:DI 0 "register_operand" "=rk")
3141         (zero_extend:DI
3142          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3143                    (ANY_EXTEND:SI
3144                     (match_operand:SHORT 2 "register_operand" "r")))))]
3145   ""
3146   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3147   [(set_attr "type" "alu_ext")]
3148 )
3149
3150 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3151   [(set (match_operand:GPI 0 "register_operand" "=rk")
3152         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3153                    (ashift:GPI (ANY_EXTEND:GPI
3154                                 (match_operand:ALLX 2 "register_operand" "r"))
3155                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3156   ""
3157   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3158   [(set_attr "type" "alu_ext")]
3159 )
3160
3161 ;; zero_extend version of above
3162 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3163   [(set (match_operand:DI 0 "register_operand" "=rk")
3164         (zero_extend:DI
3165          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3166                    (ashift:SI (ANY_EXTEND:SI
3167                                (match_operand:SHORT 2 "register_operand" "r"))
3168                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3169   ""
3170   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3171   [(set_attr "type" "alu_ext")]
3172 )
3173
3174 (define_insn "*sub_<optab><mode>_multp2"
3175   [(set (match_operand:GPI 0 "register_operand" "=rk")
3176         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3177                    (ANY_EXTRACT:GPI
3178                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3179                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3180                     (match_operand 3 "const_int_operand" "n")
3181                     (const_int 0))))]
3182   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3183   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3184   [(set_attr "type" "alu_ext")]
3185 )
3186
3187 ;; zero_extend version of above
3188 (define_insn "*sub_<optab>si_multp2_uxtw"
3189   [(set (match_operand:DI 0 "register_operand" "=rk")
3190         (zero_extend:DI
3191          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3192                    (ANY_EXTRACT:SI
3193                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3194                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3195                     (match_operand 3 "const_int_operand" "n")
3196                     (const_int 0)))))]
3197   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3198   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3199   [(set_attr "type" "alu_ext")]
3200 )
3201
3202 ;; The hardware description is op1 + ~op2 + C.
3203 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3204 ;;                           = op1 - op2 - 1 + 1 - !C
3205 ;;                           = op1 - op2 - !C.
3206 ;; We describe the latter.
3207
3208 (define_insn "*sub<mode>3_carryin0"
3209   [(set (match_operand:GPI 0 "register_operand" "=r")
3210         (minus:GPI
3211           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3212           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3213    ""
3214    "sbc\\t%<w>0, %<w>1, <w>zr"
3215   [(set_attr "type" "adc_reg")]
3216 )
3217
3218 ;; zero_extend version of the above
3219 (define_insn "*subsi3_carryin_uxtw"
3220   [(set (match_operand:DI 0 "register_operand" "=r")
3221         (zero_extend:DI
3222           (minus:SI
3223             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3224             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3225    ""
3226    "sbc\\t%w0, %w1, wzr"
3227   [(set_attr "type" "adc_reg")]
3228 )
3229
3230 (define_expand "sub<mode>3_carryin"
3231   [(set (match_operand:GPI 0 "register_operand")
3232         (minus:GPI
3233           (minus:GPI
3234             (match_operand:GPI 1 "aarch64_reg_or_zero")
3235             (match_operand:GPI 2 "register_operand"))
3236           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3237    ""
3238    ""
3239 )
3240
3241 (define_insn "*sub<mode>3_carryin"
3242   [(set (match_operand:GPI 0 "register_operand" "=r")
3243         (minus:GPI
3244           (minus:GPI
3245             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3246             (match_operand:GPI 2 "register_operand" "r"))
3247           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3248
3249    ""
3250    "sbc\\t%<w>0, %<w>1, %<w>2"
3251   [(set_attr "type" "adc_reg")]
3252 )
3253
3254 ;; zero_extend version of the above
3255 (define_insn "*subsi3_carryin_uxtw"
3256   [(set (match_operand:DI 0 "register_operand" "=r")
3257         (zero_extend:DI
3258           (minus:SI
3259             (minus:SI
3260               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3261               (match_operand:SI 2 "register_operand" "r"))
3262             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3263
3264    ""
3265    "sbc\\t%w0, %w1, %w2"
3266   [(set_attr "type" "adc_reg")]
3267 )
3268
3269 (define_insn "*sub<mode>3_carryin_alt"
3270   [(set (match_operand:GPI 0 "register_operand" "=r")
3271         (minus:GPI
3272           (minus:GPI
3273             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3274             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3275           (match_operand:GPI 2 "register_operand" "r")))]
3276    ""
3277    "sbc\\t%<w>0, %<w>1, %<w>2"
3278   [(set_attr "type" "adc_reg")]
3279 )
3280
3281 ;; zero_extend version of the above
3282 (define_insn "*subsi3_carryin_alt_uxtw"
3283   [(set (match_operand:DI 0 "register_operand" "=r")
3284         (zero_extend:DI
3285           (minus:SI
3286             (minus:SI
3287               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3288               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3289             (match_operand:SI 2 "register_operand" "r"))))]
3290    ""
3291    "sbc\\t%w0, %w1, %w2"
3292   [(set_attr "type" "adc_reg")]
3293 )
3294
3295 (define_expand "usub<GPI:mode>3_carryinC"
3296   [(parallel
3297      [(set (reg:CC CC_REGNUM)
3298            (compare:CC
3299              (zero_extend:<DWI>
3300                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3301              (plus:<DWI>
3302                (zero_extend:<DWI>
3303                  (match_operand:GPI 2 "register_operand" ""))
3304                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3305       (set (match_operand:GPI 0 "register_operand" "")
3306            (minus:GPI
3307              (minus:GPI (match_dup 1) (match_dup 2))
3308              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3309    ""
3310 )
3311
3312 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3313   [(set (reg:CC CC_REGNUM)
3314         (compare:CC
3315           (const_int 0)
3316           (plus:<DWI>
3317             (zero_extend:<DWI>
3318               (match_operand:GPI 1 "register_operand" "r"))
3319             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3320    (set (match_operand:GPI 0 "register_operand" "=r")
3321         (minus:GPI
3322           (neg:GPI (match_dup 1))
3323           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3324    ""
3325    "sbcs\\t%<w>0, <w>zr, %<w>1"
3326   [(set_attr "type" "adc_reg")]
3327 )
3328
3329 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3330   [(set (reg:CC CC_REGNUM)
3331         (compare:CC
3332           (zero_extend:<DWI>
3333             (match_operand:GPI 1 "register_operand" "r"))
3334           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3335    (set (match_operand:GPI 0 "register_operand" "=r")
3336         (minus:GPI
3337           (match_dup 1)
3338           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3339    ""
3340    "sbcs\\t%<w>0, %<w>1, <w>zr"
3341   [(set_attr "type" "adc_reg")]
3342 )
3343
3344 (define_insn "*usub<GPI:mode>3_carryinC"
3345   [(set (reg:CC CC_REGNUM)
3346         (compare:CC
3347           (zero_extend:<DWI>
3348             (match_operand:GPI 1 "register_operand" "r"))
3349           (plus:<DWI>
3350             (zero_extend:<DWI>
3351               (match_operand:GPI 2 "register_operand" "r"))
3352             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3353    (set (match_operand:GPI 0 "register_operand" "=r")
3354         (minus:GPI
3355           (minus:GPI (match_dup 1) (match_dup 2))
3356           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3357    ""
3358    "sbcs\\t%<w>0, %<w>1, %<w>2"
3359   [(set_attr "type" "adc_reg")]
3360 )
3361
3362 (define_expand "sub<GPI:mode>3_carryinV"
3363   [(parallel
3364      [(set (reg:CC_V CC_REGNUM)
3365            (compare:CC_V
3366             (minus:<DWI>
3367              (sign_extend:<DWI>
3368                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3369              (plus:<DWI>
3370                (sign_extend:<DWI>
3371                  (match_operand:GPI 2 "register_operand" ""))
3372                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3373             (sign_extend:<DWI>
3374              (minus:GPI (match_dup 1)
3375                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3376                                   (match_dup 2))))))
3377       (set (match_operand:GPI 0 "register_operand" "")
3378            (minus:GPI
3379              (minus:GPI (match_dup 1) (match_dup 2))
3380              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3381    ""
3382 )
3383
3384 (define_insn "*sub<mode>3_carryinV_z2"
3385   [(set (reg:CC_V CC_REGNUM)
3386         (compare:CC_V
3387          (minus:<DWI>
3388           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3389           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3390          (sign_extend:<DWI>
3391           (minus:GPI (match_dup 1)
3392                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3393    (set (match_operand:GPI 0 "register_operand" "=r")
3394         (minus:GPI
3395          (match_dup 1) (match_dup 3)))]
3396    ""
3397    "sbcs\\t%<w>0, %<w>1, <w>zr"
3398   [(set_attr "type" "adc_reg")]
3399 )
3400
3401 (define_insn "*sub<mode>3_carryinV"
3402   [(set (reg:CC_V CC_REGNUM)
3403         (compare:CC_V
3404          (minus:<DWI>
3405           (sign_extend:<DWI>
3406             (match_operand:GPI 1 "register_operand" "r"))
3407           (plus:<DWI>
3408             (sign_extend:<DWI>
3409               (match_operand:GPI 2 "register_operand" "r"))
3410             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3411          (sign_extend:<DWI>
3412           (minus:GPI
3413            (match_dup 1)
3414            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3415                      (match_dup 2))))))
3416    (set (match_operand:GPI 0 "register_operand" "=r")
3417         (minus:GPI
3418           (minus:GPI (match_dup 1) (match_dup 2))
3419           (match_dup 4)))]
3420    ""
3421    "sbcs\\t%<w>0, %<w>1, %<w>2"
3422   [(set_attr "type" "adc_reg")]
3423 )
3424
3425 (define_insn "*sub_uxt<mode>_shift2"
3426   [(set (match_operand:GPI 0 "register_operand" "=rk")
3427         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3428                    (and:GPI
3429                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3430                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3431                     (match_operand 3 "const_int_operand" "n"))))]
3432   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3433   "*
3434   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3435                                            INTVAL (operands[3])));
3436   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3437   [(set_attr "type" "alu_ext")]
3438 )
3439
3440 ;; zero_extend version of above
3441 (define_insn "*sub_uxtsi_shift2_uxtw"
3442   [(set (match_operand:DI 0 "register_operand" "=rk")
3443         (zero_extend:DI
3444          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3445                    (and:SI
3446                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3447                                (match_operand 2 "aarch64_imm3" "Ui3"))
3448                     (match_operand 3 "const_int_operand" "n")))))]
3449   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3450   "*
3451   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3452                                            INTVAL (operands[3])));
3453   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3454   [(set_attr "type" "alu_ext")]
3455 )
3456
3457 (define_insn "*sub_uxt<mode>_multp2"
3458   [(set (match_operand:GPI 0 "register_operand" "=rk")
3459         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3460                    (and:GPI
3461                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3462                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3463                     (match_operand 3 "const_int_operand" "n"))))]
3464   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3465   "*
3466   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3467                                            INTVAL (operands[3])));
3468   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3469   [(set_attr "type" "alu_ext")]
3470 )
3471
3472 ;; zero_extend version of above
3473 (define_insn "*sub_uxtsi_multp2_uxtw"
3474   [(set (match_operand:DI 0 "register_operand" "=rk")
3475         (zero_extend:DI
3476          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3477                    (and:SI
3478                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3479                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3480                     (match_operand 3 "const_int_operand" "n")))))]
3481   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3482   "*
3483   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3484                                            INTVAL (operands[3])));
3485   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3486   [(set_attr "type" "alu_ext")]
3487 )
3488
3489 (define_expand "abs<mode>2"
3490   [(match_operand:GPI 0 "register_operand" "")
3491    (match_operand:GPI 1 "register_operand" "")]
3492   ""
3493   {
3494     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3495     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3496     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3497     DONE;
3498   }
3499 )
3500
3501 (define_insn "neg<mode>2"
3502   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3503         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3504   ""
3505   "@
3506    neg\\t%<w>0, %<w>1
3507    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3508   [(set_attr "type" "alu_sreg, neon_neg<q>")
3509    (set_attr "arch" "*,simd")]
3510 )
3511
3512 ;; zero_extend version of above
3513 (define_insn "*negsi2_uxtw"
3514   [(set (match_operand:DI 0 "register_operand" "=r")
3515         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3516   ""
3517   "neg\\t%w0, %w1"
3518   [(set_attr "type" "alu_sreg")]
3519 )
3520
3521 (define_insn "*ngc<mode>"
3522   [(set (match_operand:GPI 0 "register_operand" "=r")
3523         (minus:GPI
3524           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3525           (match_operand:GPI 1 "register_operand" "r")))]
3526   ""
3527   "ngc\\t%<w>0, %<w>1"
3528   [(set_attr "type" "adc_reg")]
3529 )
3530
3531 (define_insn "*ngcsi_uxtw"
3532   [(set (match_operand:DI 0 "register_operand" "=r")
3533         (zero_extend:DI
3534           (minus:SI
3535             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3536             (match_operand:SI 1 "register_operand" "r"))))]
3537   ""
3538   "ngc\\t%w0, %w1"
3539   [(set_attr "type" "adc_reg")]
3540 )
3541
3542 (define_insn "neg<mode>2_compare0"
3543   [(set (reg:CC_NZ CC_REGNUM)
3544         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3545                        (const_int 0)))
3546    (set (match_operand:GPI 0 "register_operand" "=r")
3547         (neg:GPI (match_dup 1)))]
3548   ""
3549   "negs\\t%<w>0, %<w>1"
3550   [(set_attr "type" "alus_sreg")]
3551 )
3552
3553 ;; zero_extend version of above
3554 (define_insn "*negsi2_compare0_uxtw"
3555   [(set (reg:CC_NZ CC_REGNUM)
3556         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3557                        (const_int 0)))
3558    (set (match_operand:DI 0 "register_operand" "=r")
3559         (zero_extend:DI (neg:SI (match_dup 1))))]
3560   ""
3561   "negs\\t%w0, %w1"
3562   [(set_attr "type" "alus_sreg")]
3563 )
3564
3565 (define_insn "*neg_<shift><mode>3_compare0"
3566   [(set (reg:CC_NZ CC_REGNUM)
3567         (compare:CC_NZ
3568          (neg:GPI (ASHIFT:GPI
3569                    (match_operand:GPI 1 "register_operand" "r")
3570                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3571          (const_int 0)))
3572    (set (match_operand:GPI 0 "register_operand" "=r")
3573         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3574   ""
3575   "negs\\t%<w>0, %<w>1, <shift> %2"
3576   [(set_attr "type" "alus_shift_imm")]
3577 )
3578
3579 (define_insn "*neg_<shift>_<mode>2"
3580   [(set (match_operand:GPI 0 "register_operand" "=r")
3581         (neg:GPI (ASHIFT:GPI
3582                   (match_operand:GPI 1 "register_operand" "r")
3583                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3584   ""
3585   "neg\\t%<w>0, %<w>1, <shift> %2"
3586   [(set_attr "type" "alu_shift_imm")]
3587 )
3588
3589 ;; zero_extend version of above
3590 (define_insn "*neg_<shift>_si2_uxtw"
3591   [(set (match_operand:DI 0 "register_operand" "=r")
3592         (zero_extend:DI
3593          (neg:SI (ASHIFT:SI
3594                   (match_operand:SI 1 "register_operand" "r")
3595                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3596   ""
3597   "neg\\t%w0, %w1, <shift> %2"
3598   [(set_attr "type" "alu_shift_imm")]
3599 )
3600
3601 (define_insn "*neg_mul_imm_<mode>2"
3602   [(set (match_operand:GPI 0 "register_operand" "=r")
3603         (neg:GPI (mult:GPI
3604                   (match_operand:GPI 1 "register_operand" "r")
3605                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3606   ""
3607   "neg\\t%<w>0, %<w>1, lsl %p2"
3608   [(set_attr "type" "alu_shift_imm")]
3609 )
3610
3611 ;; zero_extend version of above
3612 (define_insn "*neg_mul_imm_si2_uxtw"
3613   [(set (match_operand:DI 0 "register_operand" "=r")
3614         (zero_extend:DI
3615          (neg:SI (mult:SI
3616                   (match_operand:SI 1 "register_operand" "r")
3617                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3618   ""
3619   "neg\\t%w0, %w1, lsl %p2"
3620   [(set_attr "type" "alu_shift_imm")]
3621 )
3622
3623 (define_insn "mul<mode>3"
3624   [(set (match_operand:GPI 0 "register_operand" "=r")
3625         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3626                   (match_operand:GPI 2 "register_operand" "r")))]
3627   ""
3628   "mul\\t%<w>0, %<w>1, %<w>2"
3629   [(set_attr "type" "mul")]
3630 )
3631
3632 ;; zero_extend version of above
3633 (define_insn "*mulsi3_uxtw"
3634   [(set (match_operand:DI 0 "register_operand" "=r")
3635         (zero_extend:DI
3636          (mult:SI (match_operand:SI 1 "register_operand" "r")
3637                   (match_operand:SI 2 "register_operand" "r"))))]
3638   ""
3639   "mul\\t%w0, %w1, %w2"
3640   [(set_attr "type" "mul")]
3641 )
3642
3643 (define_insn "madd<mode>"
3644   [(set (match_operand:GPI 0 "register_operand" "=r")
3645         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3646                             (match_operand:GPI 2 "register_operand" "r"))
3647                   (match_operand:GPI 3 "register_operand" "r")))]
3648   ""
3649   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3650   [(set_attr "type" "mla")]
3651 )
3652
3653 ;; zero_extend version of above
3654 (define_insn "*maddsi_uxtw"
3655   [(set (match_operand:DI 0 "register_operand" "=r")
3656         (zero_extend:DI
3657          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3658                            (match_operand:SI 2 "register_operand" "r"))
3659                   (match_operand:SI 3 "register_operand" "r"))))]
3660   ""
3661   "madd\\t%w0, %w1, %w2, %w3"
3662   [(set_attr "type" "mla")]
3663 )
3664
3665 (define_insn "*msub<mode>"
3666   [(set (match_operand:GPI 0 "register_operand" "=r")
3667         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3668                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3669                              (match_operand:GPI 2 "register_operand" "r"))))]
3670
3671   ""
3672   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3673   [(set_attr "type" "mla")]
3674 )
3675
3676 ;; zero_extend version of above
3677 (define_insn "*msubsi_uxtw"
3678   [(set (match_operand:DI 0 "register_operand" "=r")
3679         (zero_extend:DI
3680          (minus:SI (match_operand:SI 3 "register_operand" "r")
3681                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3682                             (match_operand:SI 2 "register_operand" "r")))))]
3683
3684   ""
3685   "msub\\t%w0, %w1, %w2, %w3"
3686   [(set_attr "type" "mla")]
3687 )
3688
3689 (define_insn "*mul<mode>_neg"
3690   [(set (match_operand:GPI 0 "register_operand" "=r")
3691         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3692                   (match_operand:GPI 2 "register_operand" "r")))]
3693
3694   ""
3695   "mneg\\t%<w>0, %<w>1, %<w>2"
3696   [(set_attr "type" "mul")]
3697 )
3698
3699 ;; zero_extend version of above
3700 (define_insn "*mulsi_neg_uxtw"
3701   [(set (match_operand:DI 0 "register_operand" "=r")
3702         (zero_extend:DI
3703          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3704                   (match_operand:SI 2 "register_operand" "r"))))]
3705
3706   ""
3707   "mneg\\t%w0, %w1, %w2"
3708   [(set_attr "type" "mul")]
3709 )
3710
3711 (define_insn "<su_optab>mulsidi3"
3712   [(set (match_operand:DI 0 "register_operand" "=r")
3713         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3714                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3715   ""
3716   "<su>mull\\t%0, %w1, %w2"
3717   [(set_attr "type" "<su>mull")]
3718 )
3719
3720 (define_insn "<su_optab>maddsidi4"
3721   [(set (match_operand:DI 0 "register_operand" "=r")
3722         (plus:DI (mult:DI
3723                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3724                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3725                  (match_operand:DI 3 "register_operand" "r")))]
3726   ""
3727   "<su>maddl\\t%0, %w1, %w2, %3"
3728   [(set_attr "type" "<su>mlal")]
3729 )
3730
3731 (define_insn "<su_optab>msubsidi4"
3732   [(set (match_operand:DI 0 "register_operand" "=r")
3733         (minus:DI
3734          (match_operand:DI 3 "register_operand" "r")
3735          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3736                   (ANY_EXTEND:DI
3737                    (match_operand:SI 2 "register_operand" "r")))))]
3738   ""
3739   "<su>msubl\\t%0, %w1, %w2, %3"
3740   [(set_attr "type" "<su>mlal")]
3741 )
3742
3743 (define_insn "*<su_optab>mulsidi_neg"
3744   [(set (match_operand:DI 0 "register_operand" "=r")
3745         (mult:DI (neg:DI
3746                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3747                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3748   ""
3749   "<su>mnegl\\t%0, %w1, %w2"
3750   [(set_attr "type" "<su>mull")]
3751 )
3752
3753 (define_expand "<su_optab>mulditi3"
3754   [(set (match_operand:TI 0 "register_operand")
3755         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3756                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3757   ""
3758 {
3759   rtx low = gen_reg_rtx (DImode);
3760   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3761
3762   rtx high = gen_reg_rtx (DImode);
3763   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3764
3765   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3766   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3767   DONE;
3768 })
3769
3770 ;; The default expansion of multi3 using umuldi3_highpart will perform
3771 ;; the additions in an order that fails to combine into two madd insns.
3772 (define_expand "multi3"
3773   [(set (match_operand:TI 0 "register_operand")
3774         (mult:TI (match_operand:TI 1 "register_operand")
3775                  (match_operand:TI 2 "register_operand")))]
3776   ""
3777 {
3778   rtx l0 = gen_reg_rtx (DImode);
3779   rtx l1 = gen_lowpart (DImode, operands[1]);
3780   rtx l2 = gen_lowpart (DImode, operands[2]);
3781   rtx h0 = gen_reg_rtx (DImode);
3782   rtx h1 = gen_highpart (DImode, operands[1]);
3783   rtx h2 = gen_highpart (DImode, operands[2]);
3784
3785   emit_insn (gen_muldi3 (l0, l1, l2));
3786   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3787   emit_insn (gen_madddi (h0, h1, l2, h0));
3788   emit_insn (gen_madddi (h0, l1, h2, h0));
3789
3790   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3791   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3792   DONE;
3793 })
3794
3795 (define_insn "<su>muldi3_highpart"
3796   [(set (match_operand:DI 0 "register_operand" "=r")
3797         (truncate:DI
3798          (lshiftrt:TI
3799           (mult:TI
3800            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3801            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3802           (const_int 64))))]
3803   ""
3804   "<su>mulh\\t%0, %1, %2"
3805   [(set_attr "type" "<su>mull")]
3806 )
3807
3808 (define_insn "<su_optab>div<mode>3"
3809   [(set (match_operand:GPI 0 "register_operand" "=r")
3810         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3811                      (match_operand:GPI 2 "register_operand" "r")))]
3812   ""
3813   "<su>div\\t%<w>0, %<w>1, %<w>2"
3814   [(set_attr "type" "<su>div")]
3815 )
3816
3817 ;; zero_extend version of above
3818 (define_insn "*<su_optab>divsi3_uxtw"
3819   [(set (match_operand:DI 0 "register_operand" "=r")
3820         (zero_extend:DI
3821          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3822                      (match_operand:SI 2 "register_operand" "r"))))]
3823   ""
3824   "<su>div\\t%w0, %w1, %w2"
3825   [(set_attr "type" "<su>div")]
3826 )
3827
3828 ;; -------------------------------------------------------------------
3829 ;; Comparison insns
3830 ;; -------------------------------------------------------------------
3831
3832 (define_insn "cmp<mode>"
3833   [(set (reg:CC CC_REGNUM)
3834         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3835                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3836   ""
3837   "@
3838    cmp\\t%<w>0, %<w>1
3839    cmp\\t%<w>0, %1
3840    cmn\\t%<w>0, #%n1"
3841   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3842 )
3843
3844 (define_insn "fcmp<mode>"
3845   [(set (reg:CCFP CC_REGNUM)
3846         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3847                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3848    "TARGET_FLOAT"
3849    "@
3850     fcmp\\t%<s>0, #0.0
3851     fcmp\\t%<s>0, %<s>1"
3852   [(set_attr "type" "fcmp<s>")]
3853 )
3854
3855 (define_insn "fcmpe<mode>"
3856   [(set (reg:CCFPE CC_REGNUM)
3857         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3858                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3859    "TARGET_FLOAT"
3860    "@
3861     fcmpe\\t%<s>0, #0.0
3862     fcmpe\\t%<s>0, %<s>1"
3863   [(set_attr "type" "fcmp<s>")]
3864 )
3865
3866 (define_insn "*cmp_swp_<shift>_reg<mode>"
3867   [(set (reg:CC_SWP CC_REGNUM)
3868         (compare:CC_SWP (ASHIFT:GPI
3869                          (match_operand:GPI 0 "register_operand" "r")
3870                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3871                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3872   ""
3873   "cmp\\t%<w>2, %<w>0, <shift> %1"
3874   [(set_attr "type" "alus_shift_imm")]
3875 )
3876
3877 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3878   [(set (reg:CC_SWP CC_REGNUM)
3879         (compare:CC_SWP (ANY_EXTEND:GPI
3880                          (match_operand:ALLX 0 "register_operand" "r"))
3881                         (match_operand:GPI 1 "register_operand" "r")))]
3882   ""
3883   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3884   [(set_attr "type" "alus_ext")]
3885 )
3886
3887 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3888   [(set (reg:CC_SWP CC_REGNUM)
3889         (compare:CC_SWP (ashift:GPI
3890                          (ANY_EXTEND:GPI
3891                           (match_operand:ALLX 0 "register_operand" "r"))
3892                          (match_operand 1 "aarch64_imm3" "Ui3"))
3893         (match_operand:GPI 2 "register_operand" "r")))]
3894   ""
3895   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3896   [(set_attr "type" "alus_ext")]
3897 )
3898
3899 ;; -------------------------------------------------------------------
3900 ;; Store-flag and conditional select insns
3901 ;; -------------------------------------------------------------------
3902
3903 (define_expand "cstore<mode>4"
3904   [(set (match_operand:SI 0 "register_operand" "")
3905         (match_operator:SI 1 "aarch64_comparison_operator"
3906          [(match_operand:GPI 2 "register_operand" "")
3907           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3908   ""
3909   "
3910   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3911                                       operands[3]);
3912   operands[3] = const0_rtx;
3913   "
3914 )
3915
3916 (define_expand "cstorecc4"
3917   [(set (match_operand:SI 0 "register_operand")
3918        (match_operator 1 "aarch64_comparison_operator_mode"
3919         [(match_operand 2 "cc_register")
3920          (match_operand 3 "const0_operand")]))]
3921   ""
3922 "{
3923   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3924   DONE;
3925 }")
3926
3927
3928 (define_expand "cstore<mode>4"
3929   [(set (match_operand:SI 0 "register_operand" "")
3930         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3931          [(match_operand:GPF 2 "register_operand" "")
3932           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3933   ""
3934   "
3935   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3936                                       operands[3]);
3937   operands[3] = const0_rtx;
3938   "
3939 )
3940
3941 (define_insn "aarch64_cstore<mode>"
3942   [(set (match_operand:ALLI 0 "register_operand" "=r")
3943         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3944          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3945   ""
3946   "cset\\t%<w>0, %m1"
3947   [(set_attr "type" "csel")]
3948 )
3949
3950 ;; For a 24-bit immediate CST we can optimize the compare for equality
3951 ;; and branch sequence from:
3952 ;;      mov     x0, #imm1
3953 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3954 ;;      cmp     x1, x0
3955 ;;      cset    x2, <ne,eq>
3956 ;; into the shorter:
3957 ;;      sub     x0, x1, #(CST & 0xfff000)
3958 ;;      subs    x0, x0, #(CST & 0x000fff)
3959 ;;      cset x2, <ne, eq>.
3960 (define_insn_and_split "*compare_cstore<mode>_insn"
3961   [(set (match_operand:GPI 0 "register_operand" "=r")
3962          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3963                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3964    (clobber (reg:CC CC_REGNUM))]
3965   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3966    && !aarch64_plus_operand (operands[2], <MODE>mode)
3967    && !reload_completed"
3968   "#"
3969   "&& true"
3970   [(const_int 0)]
3971   {
3972     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3973     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3974     rtx tmp = gen_reg_rtx (<MODE>mode);
3975     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3976     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3977     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3978     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3979     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3980     DONE;
3981   }
3982   [(set_attr "type" "csel")]
3983 )
3984
3985 ;; zero_extend version of the above
3986 (define_insn "*cstoresi_insn_uxtw"
3987   [(set (match_operand:DI 0 "register_operand" "=r")
3988         (zero_extend:DI
3989          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3990           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3991   ""
3992   "cset\\t%w0, %m1"
3993   [(set_attr "type" "csel")]
3994 )
3995
3996 (define_insn "cstore<mode>_neg"
3997   [(set (match_operand:ALLI 0 "register_operand" "=r")
3998         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3999                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4000   ""
4001   "csetm\\t%<w>0, %m1"
4002   [(set_attr "type" "csel")]
4003 )
4004
4005 ;; zero_extend version of the above
4006 (define_insn "*cstoresi_neg_uxtw"
4007   [(set (match_operand:DI 0 "register_operand" "=r")
4008         (zero_extend:DI
4009          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4010                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4011   ""
4012   "csetm\\t%w0, %m1"
4013   [(set_attr "type" "csel")]
4014 )
4015
4016 (define_expand "cmov<mode>6"
4017   [(set (match_operand:GPI 0 "register_operand" "")
4018         (if_then_else:GPI
4019          (match_operator 1 "aarch64_comparison_operator"
4020           [(match_operand:GPI 2 "register_operand" "")
4021            (match_operand:GPI 3 "aarch64_plus_operand" "")])
4022          (match_operand:GPI 4 "register_operand" "")
4023          (match_operand:GPI 5 "register_operand" "")))]
4024   ""
4025   "
4026   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4027                                       operands[3]);
4028   operands[3] = const0_rtx;
4029   "
4030 )
4031
4032 (define_expand "cmov<mode>6"
4033   [(set (match_operand:GPF 0 "register_operand" "")
4034         (if_then_else:GPF
4035          (match_operator 1 "aarch64_comparison_operator"
4036           [(match_operand:GPF 2 "register_operand" "")
4037            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
4038          (match_operand:GPF 4 "register_operand" "")
4039          (match_operand:GPF 5 "register_operand" "")))]
4040   ""
4041   "
4042   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4043                                       operands[3]);
4044   operands[3] = const0_rtx;
4045   "
4046 )
4047
4048 (define_insn "*cmov<mode>_insn"
4049   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4050         (if_then_else:ALLI
4051          (match_operator 1 "aarch64_comparison_operator"
4052           [(match_operand 2 "cc_register" "") (const_int 0)])
4053          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4054          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4055   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4056      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4057   ;; Final two alternatives should be unreachable, but included for completeness
4058   "@
4059    csel\\t%<w>0, %<w>3, %<w>4, %m1
4060    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4061    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4062    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4063    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4064    mov\\t%<w>0, -1
4065    mov\\t%<w>0, 1"
4066   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4067 )
4068
4069 ;; zero_extend version of above
4070 (define_insn "*cmovsi_insn_uxtw"
4071   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4072         (zero_extend:DI
4073          (if_then_else:SI
4074           (match_operator 1 "aarch64_comparison_operator"
4075            [(match_operand 2 "cc_register" "") (const_int 0)])
4076           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4077           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4078   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4079      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4080   ;; Final two alternatives should be unreachable, but included for completeness
4081   "@
4082    csel\\t%w0, %w3, %w4, %m1
4083    csinv\\t%w0, %w3, wzr, %m1
4084    csinv\\t%w0, %w4, wzr, %M1
4085    csinc\\t%w0, %w3, wzr, %m1
4086    csinc\\t%w0, %w4, wzr, %M1
4087    mov\\t%w0, -1
4088    mov\\t%w0, 1"
4089   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4090 )
4091
4092 (define_insn "*cmovdi_insn_uxtw"
4093   [(set (match_operand:DI 0 "register_operand" "=r")
4094         (if_then_else:DI
4095          (match_operator 1 "aarch64_comparison_operator"
4096           [(match_operand 2 "cc_register" "") (const_int 0)])
4097          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4098          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4099   ""
4100   "csel\\t%w0, %w3, %w4, %m1"
4101   [(set_attr "type" "csel")]
4102 )
4103
4104 (define_insn "*cmov<mode>_insn"
4105   [(set (match_operand:GPF 0 "register_operand" "=w")
4106         (if_then_else:GPF
4107          (match_operator 1 "aarch64_comparison_operator"
4108           [(match_operand 2 "cc_register" "") (const_int 0)])
4109          (match_operand:GPF 3 "register_operand" "w")
4110          (match_operand:GPF 4 "register_operand" "w")))]
4111   "TARGET_FLOAT"
4112   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4113   [(set_attr "type" "fcsel")]
4114 )
4115
4116 (define_expand "mov<mode>cc"
4117   [(set (match_operand:ALLI 0 "register_operand" "")
4118         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
4119                            (match_operand:ALLI 2 "register_operand" "")
4120                            (match_operand:ALLI 3 "register_operand" "")))]
4121   ""
4122   {
4123     rtx ccreg;
4124     enum rtx_code code = GET_CODE (operands[1]);
4125
4126     if (code == UNEQ || code == LTGT)
4127       FAIL;
4128
4129     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4130                                      XEXP (operands[1], 1));
4131     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4132   }
4133 )
4134
4135 (define_expand "mov<GPF:mode><GPI:mode>cc"
4136   [(set (match_operand:GPI 0 "register_operand" "")
4137         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
4138                           (match_operand:GPF 2 "register_operand" "")
4139                           (match_operand:GPF 3 "register_operand" "")))]
4140   ""
4141   {
4142     rtx ccreg;
4143     enum rtx_code code = GET_CODE (operands[1]);
4144
4145     if (code == UNEQ || code == LTGT)
4146       FAIL;
4147
4148     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4149                                   XEXP (operands[1], 1));
4150     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4151   }
4152 )
4153
4154 (define_expand "mov<mode>cc"
4155   [(set (match_operand:GPF 0 "register_operand" "")
4156         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
4157                           (match_operand:GPF 2 "register_operand" "")
4158                           (match_operand:GPF 3 "register_operand" "")))]
4159   ""
4160   {
4161     rtx ccreg;
4162     enum rtx_code code = GET_CODE (operands[1]);
4163
4164     if (code == UNEQ || code == LTGT)
4165       FAIL;
4166
4167     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4168                                   XEXP (operands[1], 1));
4169     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4170   }
4171 )
4172
4173 (define_expand "<neg_not_op><mode>cc"
4174   [(set (match_operand:GPI 0 "register_operand" "")
4175         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
4176                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
4177                           (match_operand:GPI 3 "register_operand" "")))]
4178   ""
4179   {
4180     rtx ccreg;
4181     enum rtx_code code = GET_CODE (operands[1]);
4182
4183     if (code == UNEQ || code == LTGT)
4184       FAIL;
4185
4186     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4187                                       XEXP (operands[1], 1));
4188     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4189   }
4190 )
4191
4192 ;; CRC32 instructions.
4193 (define_insn "aarch64_<crc_variant>"
4194   [(set (match_operand:SI 0 "register_operand" "=r")
4195         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4196                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4197          CRC))]
4198   "TARGET_CRC32"
4199   {
4200     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4201       return "<crc_variant>\\t%w0, %w1, %x2";
4202     else
4203       return "<crc_variant>\\t%w0, %w1, %w2";
4204   }
4205   [(set_attr "type" "crc")]
4206 )
4207
4208 (define_insn "*csinc2<mode>_insn"
4209   [(set (match_operand:GPI 0 "register_operand" "=r")
4210         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4211                   (match_operand:GPI 1 "register_operand" "r")))]
4212   ""
4213   "cinc\\t%<w>0, %<w>1, %m2"
4214   [(set_attr "type" "csel")]
4215 )
4216
4217 (define_insn "csinc3<mode>_insn"
4218   [(set (match_operand:GPI 0 "register_operand" "=r")
4219         (if_then_else:GPI
4220           (match_operand 1 "aarch64_comparison_operation" "")
4221           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4222                     (const_int 1))
4223           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4224   ""
4225   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4226   [(set_attr "type" "csel")]
4227 )
4228
4229 (define_insn "*csinv3<mode>_insn"
4230   [(set (match_operand:GPI 0 "register_operand" "=r")
4231         (if_then_else:GPI
4232           (match_operand 1 "aarch64_comparison_operation" "")
4233           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4234           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4235   ""
4236   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4237   [(set_attr "type" "csel")]
4238 )
4239
4240 (define_insn "csneg3_uxtw_insn"
4241   [(set (match_operand:DI 0 "register_operand" "=r")
4242         (zero_extend:DI
4243           (if_then_else:SI
4244             (match_operand 1 "aarch64_comparison_operation" "")
4245             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4246             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4247   ""
4248   "csneg\\t%w0, %w3, %w2, %M1"
4249   [(set_attr "type" "csel")]
4250 )
4251
4252 (define_insn "csneg3<mode>_insn"
4253   [(set (match_operand:GPI 0 "register_operand" "=r")
4254         (if_then_else:GPI
4255           (match_operand 1 "aarch64_comparison_operation" "")
4256           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4257           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4258   ""
4259   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4260   [(set_attr "type" "csel")]
4261 )
4262
4263 ;; If X can be loaded by a single CNT[BHWD] instruction,
4264 ;;
4265 ;;    A = UMAX (B, X)
4266 ;;
4267 ;; is equivalent to:
4268 ;;
4269 ;;    TMP = UQDEC[BHWD] (B, X)
4270 ;;    A = TMP + X
4271 ;;
4272 ;; Defining the pattern this way means that:
4273 ;;
4274 ;;    A = UMAX (B, X) - X
4275 ;;
4276 ;; becomes:
4277 ;;
4278 ;;    TMP1 = UQDEC[BHWD] (B, X)
4279 ;;    TMP2 = TMP1 + X
4280 ;;    A = TMP2 - X
4281 ;;
4282 ;; which combine can optimize to:
4283 ;;
4284 ;;    A = UQDEC[BHWD] (B, X)
4285 ;;
4286 ;; We don't use match_operand predicates because the order of the operands
4287 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4288 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4289 (define_expand "umax<mode>3"
4290   [(set (match_operand:GPI 0 "register_operand")
4291         (umax:GPI (match_operand:GPI 1 "")
4292                   (match_operand:GPI 2 "")))]
4293   "TARGET_SVE"
4294   {
4295     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4296       std::swap (operands[1], operands[2]);
4297     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4298       FAIL;
4299     rtx temp = gen_reg_rtx (<MODE>mode);
4300     operands[1] = force_reg (<MODE>mode, operands[1]);
4301     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4302     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4303     DONE;
4304   }
4305 )
4306
4307 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4308 (define_insn "aarch64_uqdec<mode>"
4309   [(set (match_operand:GPI 0 "register_operand" "=r")
4310         (minus:GPI
4311          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4312                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4313          (match_dup 2)))]
4314   "TARGET_SVE"
4315   {
4316     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4317   }
4318 )
4319
4320 ;; -------------------------------------------------------------------
4321 ;; Logical operations
4322 ;; -------------------------------------------------------------------
4323
4324
4325 (define_insn_and_split "*aarch64_and<mode>_imm2"
4326   [(set (match_operand:GPI 0 "register_operand" "=rk")
4327         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4328                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4329   ""
4330   "#"
4331   "true"
4332   [(const_int 0)]
4333   {
4334      HOST_WIDE_INT val = INTVAL (operands[2]);
4335      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4336      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4337
4338      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4339      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4340      DONE;
4341   }
4342 )
4343
4344 (define_insn "<optab><mode>3"
4345   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4346         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4347                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4348   ""
4349   "@
4350   <logical>\\t%<w>0, %<w>1, %<w>2
4351   <logical>\\t%<w>0, %<w>1, %2
4352   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4353   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4354    (set_attr "arch" "*,*,simd")]
4355 )
4356
4357 ;; zero_extend version of above
4358 (define_insn "*<optab>si3_uxtw"
4359   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4360         (zero_extend:DI
4361          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4362                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4363   ""
4364   "@
4365    <logical>\\t%w0, %w1, %w2
4366    <logical>\\t%w0, %w1, %2"
4367   [(set_attr "type" "logic_reg,logic_imm")]
4368 )
4369
4370 (define_insn "*and<mode>3_compare0"
4371   [(set (reg:CC_NZ CC_REGNUM)
4372         (compare:CC_NZ
4373          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4374                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4375          (const_int 0)))
4376    (set (match_operand:GPI 0 "register_operand" "=r,r")
4377         (and:GPI (match_dup 1) (match_dup 2)))]
4378   ""
4379   "@
4380    ands\\t%<w>0, %<w>1, %<w>2
4381    ands\\t%<w>0, %<w>1, %2"
4382   [(set_attr "type" "logics_reg,logics_imm")]
4383 )
4384
4385 ;; zero_extend version of above
4386 (define_insn "*andsi3_compare0_uxtw"
4387   [(set (reg:CC_NZ CC_REGNUM)
4388         (compare:CC_NZ
4389          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4390                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4391          (const_int 0)))
4392    (set (match_operand:DI 0 "register_operand" "=r,r")
4393         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4394   ""
4395   "@
4396    ands\\t%w0, %w1, %w2
4397    ands\\t%w0, %w1, %2"
4398   [(set_attr "type" "logics_reg,logics_imm")]
4399 )
4400
4401 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4402   [(set (reg:CC_NZ CC_REGNUM)
4403         (compare:CC_NZ
4404          (and:GPI (SHIFT:GPI
4405                    (match_operand:GPI 1 "register_operand" "r")
4406                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4407                   (match_operand:GPI 3 "register_operand" "r"))
4408          (const_int 0)))
4409    (set (match_operand:GPI 0 "register_operand" "=r")
4410         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4411   ""
4412   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4413   [(set_attr "type" "logics_shift_imm")]
4414 )
4415
4416 ;; zero_extend version of above
4417 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4418   [(set (reg:CC_NZ CC_REGNUM)
4419         (compare:CC_NZ
4420          (and:SI (SHIFT:SI
4421                   (match_operand:SI 1 "register_operand" "r")
4422                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4423                  (match_operand:SI 3 "register_operand" "r"))
4424          (const_int 0)))
4425    (set (match_operand:DI 0 "register_operand" "=r")
4426         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4427                                 (match_dup 3))))]
4428   ""
4429   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4430   [(set_attr "type" "logics_shift_imm")]
4431 )
4432
4433 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4434   [(set (match_operand:GPI 0 "register_operand" "=r")
4435         (LOGICAL:GPI (SHIFT:GPI
4436                       (match_operand:GPI 1 "register_operand" "r")
4437                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4438                      (match_operand:GPI 3 "register_operand" "r")))]
4439   ""
4440   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4441   [(set_attr "type" "logic_shift_imm")]
4442 )
4443
4444 (define_insn "*<optab>_rol<mode>3"
4445   [(set (match_operand:GPI 0 "register_operand" "=r")
4446         (LOGICAL:GPI (rotate:GPI
4447                       (match_operand:GPI 1 "register_operand" "r")
4448                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4449                      (match_operand:GPI 3 "register_operand" "r")))]
4450   ""
4451   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4452   [(set_attr "type" "logic_shift_imm")]
4453 )
4454
4455 ;; zero_extend versions of above
4456 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4457   [(set (match_operand:DI 0 "register_operand" "=r")
4458         (zero_extend:DI
4459          (LOGICAL:SI (SHIFT:SI
4460                       (match_operand:SI 1 "register_operand" "r")
4461                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4462                      (match_operand:SI 3 "register_operand" "r"))))]
4463   ""
4464   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4465   [(set_attr "type" "logic_shift_imm")]
4466 )
4467
4468 (define_insn "*<optab>_rolsi3_uxtw"
4469   [(set (match_operand:DI 0 "register_operand" "=r")
4470         (zero_extend:DI
4471          (LOGICAL:SI (rotate:SI
4472                       (match_operand:SI 1 "register_operand" "r")
4473                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4474                      (match_operand:SI 3 "register_operand" "r"))))]
4475   ""
4476   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4477   [(set_attr "type" "logic_shift_imm")]
4478 )
4479
4480 (define_insn "one_cmpl<mode>2"
4481   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4482         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4483   ""
4484   "@
4485   mvn\\t%<w>0, %<w>1
4486   mvn\\t%0.8b, %1.8b"
4487   [(set_attr "type" "logic_reg,neon_logic")
4488    (set_attr "arch" "*,simd")]
4489 )
4490
4491 (define_insn "*one_cmpl_<optab><mode>2"
4492   [(set (match_operand:GPI 0 "register_operand" "=r")
4493         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4494                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4495   ""
4496   "mvn\\t%<w>0, %<w>1, <shift> %2"
4497   [(set_attr "type" "logic_shift_imm")]
4498 )
4499
4500 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4501
4502 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4503   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4504         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4505                      (match_operand:GPI 2 "register_operand" "r,w")))]
4506   ""
4507   "@
4508   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4509   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4510   [(set_attr "type" "logic_reg,neon_logic")
4511    (set_attr "arch" "*,simd")]
4512 )
4513
4514 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4515   [(set (match_operand:DI 0 "register_operand" "=r")
4516         (zero_extend:DI
4517           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4518                        (match_operand:SI 2 "register_operand" "r"))))]
4519   ""
4520   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4521   [(set_attr "type" "logic_reg")]
4522 )
4523
4524 (define_insn "*xor_one_cmplsidi3_ze"
4525   [(set (match_operand:DI 0 "register_operand" "=r")
4526         (zero_extend:DI
4527           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4528                           (match_operand:SI 2 "register_operand" "r")))))]
4529   ""
4530   "eon\\t%w0, %w1, %w2"
4531   [(set_attr "type" "logic_reg")]
4532 )
4533
4534 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4535 ;; eon does not operate on SIMD registers so the vector variant must be split.
4536 (define_insn_and_split "*xor_one_cmpl<mode>3"
4537   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4538         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4539                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4540   ""
4541   "@
4542   eon\\t%<w>0, %<w>1, %<w>2
4543   #"
4544   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4545   [(set (match_operand:GPI 0 "register_operand" "=w")
4546         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4547                  (match_operand:GPI 2 "register_operand" "w")))
4548    (set (match_dup 0) (not:GPI (match_dup 0)))]
4549   ""
4550   [(set_attr "type" "logic_reg,multiple")
4551    (set_attr "arch" "*,simd")]
4552 )
4553
4554 (define_insn "*and_one_cmpl<mode>3_compare0"
4555   [(set (reg:CC_NZ CC_REGNUM)
4556         (compare:CC_NZ
4557          (and:GPI (not:GPI
4558                    (match_operand:GPI 1 "register_operand" "r"))
4559                   (match_operand:GPI 2 "register_operand" "r"))
4560          (const_int 0)))
4561    (set (match_operand:GPI 0 "register_operand" "=r")
4562         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4563   ""
4564   "bics\\t%<w>0, %<w>2, %<w>1"
4565   [(set_attr "type" "logics_reg")]
4566 )
4567
4568 ;; zero_extend version of above
4569 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4570   [(set (reg:CC_NZ CC_REGNUM)
4571         (compare:CC_NZ
4572          (and:SI (not:SI
4573                   (match_operand:SI 1 "register_operand" "r"))
4574                  (match_operand:SI 2 "register_operand" "r"))
4575          (const_int 0)))
4576    (set (match_operand:DI 0 "register_operand" "=r")
4577         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4578   ""
4579   "bics\\t%w0, %w2, %w1"
4580   [(set_attr "type" "logics_reg")]
4581 )
4582
4583 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4584   [(set (reg:CC_NZ CC_REGNUM)
4585     (compare:CC_NZ
4586      (and:GPI (not:GPI
4587            (match_operand:GPI 0 "register_operand" "r"))
4588           (match_operand:GPI 1 "register_operand" "r"))
4589      (const_int 0)))]
4590   ""
4591   "bics\\t<w>zr, %<w>1, %<w>0"
4592   [(set_attr "type" "logics_reg")]
4593 )
4594
4595 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4596   [(set (match_operand:GPI 0 "register_operand" "=r")
4597         (LOGICAL:GPI (not:GPI
4598                       (SHIFT:GPI
4599                        (match_operand:GPI 1 "register_operand" "r")
4600                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4601                      (match_operand:GPI 3 "register_operand" "r")))]
4602   ""
4603   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4604   [(set_attr "type" "logic_shift_imm")]
4605 )
4606
4607 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4608   [(set (match_operand:GPI 0 "register_operand" "=r")
4609         (not:GPI (xor:GPI
4610                       (SHIFT:GPI
4611                        (match_operand:GPI 1 "register_operand" "r")
4612                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4613                      (match_operand:GPI 3 "register_operand" "r"))))]
4614   ""
4615   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4616   [(set_attr "type" "logic_shift_imm")]
4617 )
4618
4619 ;; Zero-extend version of the above.
4620 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4621   [(set (match_operand:DI 0 "register_operand" "=r")
4622         (zero_extend:DI
4623           (not:SI (xor:SI
4624                     (SHIFT:SI
4625                       (match_operand:SI 1 "register_operand" "r")
4626                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4627                     (match_operand:SI 3 "register_operand" "r")))))]
4628   ""
4629   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4630   [(set_attr "type" "logic_shift_imm")]
4631 )
4632
4633 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4634   [(set (reg:CC_NZ CC_REGNUM)
4635         (compare:CC_NZ
4636          (and:GPI (not:GPI
4637                    (SHIFT:GPI
4638                     (match_operand:GPI 1 "register_operand" "r")
4639                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4640                   (match_operand:GPI 3 "register_operand" "r"))
4641          (const_int 0)))
4642    (set (match_operand:GPI 0 "register_operand" "=r")
4643         (and:GPI (not:GPI
4644                   (SHIFT:GPI
4645                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4646   ""
4647   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4648   [(set_attr "type" "logics_shift_imm")]
4649 )
4650
4651 ;; zero_extend version of above
4652 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4653   [(set (reg:CC_NZ CC_REGNUM)
4654         (compare:CC_NZ
4655          (and:SI (not:SI
4656                   (SHIFT:SI
4657                    (match_operand:SI 1 "register_operand" "r")
4658                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4659                  (match_operand:SI 3 "register_operand" "r"))
4660          (const_int 0)))
4661    (set (match_operand:DI 0 "register_operand" "=r")
4662         (zero_extend:DI (and:SI
4663                          (not:SI
4664                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4665   ""
4666   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4667   [(set_attr "type" "logics_shift_imm")]
4668 )
4669
4670 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4671   [(set (reg:CC_NZ CC_REGNUM)
4672     (compare:CC_NZ
4673      (and:GPI (not:GPI
4674            (SHIFT:GPI
4675             (match_operand:GPI 0 "register_operand" "r")
4676             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4677           (match_operand:GPI 2 "register_operand" "r"))
4678      (const_int 0)))]
4679   ""
4680   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4681   [(set_attr "type" "logics_shift_imm")]
4682 )
4683
4684 (define_insn "clz<mode>2"
4685   [(set (match_operand:GPI 0 "register_operand" "=r")
4686         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4687   ""
4688   "clz\\t%<w>0, %<w>1"
4689   [(set_attr "type" "clz")]
4690 )
4691
4692 (define_expand "ffs<mode>2"
4693   [(match_operand:GPI 0 "register_operand")
4694    (match_operand:GPI 1 "register_operand")]
4695   ""
4696   {
4697     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4698     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4699
4700     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4701     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4702     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4703     DONE;
4704   }
4705 )
4706
4707 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4708 ;;
4709 ;; MOV  v.1d, x0
4710 ;; CNT  v1.8b, v.8b
4711 ;; ADDV b2, v1.8b
4712 ;; MOV  w0, v2.b[0]
4713
4714 (define_expand "popcount<mode>2"
4715   [(match_operand:GPI 0 "register_operand")
4716    (match_operand:GPI 1 "register_operand")]
4717   "TARGET_SIMD"
4718 {
4719   rtx v = gen_reg_rtx (V8QImode);
4720   rtx v1 = gen_reg_rtx (V8QImode);
4721   rtx r = gen_reg_rtx (QImode);
4722   rtx in = operands[1];
4723   rtx out = operands[0];
4724   if(<MODE>mode == SImode)
4725     {
4726       rtx tmp;
4727       tmp = gen_reg_rtx (DImode);
4728       /* If we have SImode, zero extend to DImode, pop count does
4729          not change if we have extra zeros. */
4730       emit_insn (gen_zero_extendsidi2 (tmp, in));
4731       in = tmp;
4732     }
4733   emit_move_insn (v, gen_lowpart (V8QImode, in));
4734   emit_insn (gen_popcountv8qi2 (v1, v));
4735   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4736   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4737   DONE;
4738 })
4739
4740 (define_insn "clrsb<mode>2"
4741   [(set (match_operand:GPI 0 "register_operand" "=r")
4742         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4743   ""
4744   "cls\\t%<w>0, %<w>1"
4745   [(set_attr "type" "clz")]
4746 )
4747
4748 (define_insn "rbit<mode>2"
4749   [(set (match_operand:GPI 0 "register_operand" "=r")
4750         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4751   ""
4752   "rbit\\t%<w>0, %<w>1"
4753   [(set_attr "type" "rbit")]
4754 )
4755
4756 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4757 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4758 ;; expression and split after reload to enable scheduling them apart if
4759 ;; needed.
4760
4761 (define_insn_and_split "ctz<mode>2"
4762  [(set (match_operand:GPI           0 "register_operand" "=r")
4763        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4764   ""
4765   "#"
4766   "reload_completed"
4767   [(const_int 0)]
4768   "
4769   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4770   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4771   DONE;
4772 ")
4773
4774 (define_insn "*and<mode>_compare0"
4775   [(set (reg:CC_NZ CC_REGNUM)
4776         (compare:CC_NZ
4777          (match_operand:SHORT 0 "register_operand" "r")
4778          (const_int 0)))]
4779   ""
4780   "tst\\t%<w>0, <short_mask>"
4781   [(set_attr "type" "alus_imm")]
4782 )
4783
4784 (define_insn "*ands<mode>_compare0"
4785   [(set (reg:CC_NZ CC_REGNUM)
4786         (compare:CC_NZ
4787          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4788          (const_int 0)))
4789    (set (match_operand:GPI 0 "register_operand" "=r")
4790         (zero_extend:GPI (match_dup 1)))]
4791   ""
4792   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4793   [(set_attr "type" "alus_imm")]
4794 )
4795
4796 (define_insn "*and<mode>3nr_compare0"
4797   [(set (reg:CC_NZ CC_REGNUM)
4798         (compare:CC_NZ
4799          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4800                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4801          (const_int 0)))]
4802   ""
4803   "@
4804    tst\\t%<w>0, %<w>1
4805    tst\\t%<w>0, %1"
4806   [(set_attr "type" "logics_reg,logics_imm")]
4807 )
4808
4809 (define_split
4810   [(set (reg:CC_NZ CC_REGNUM)
4811         (compare:CC_NZ
4812          (and:GPI (match_operand:GPI 0 "register_operand")
4813                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4814          (const_int 0)))
4815    (clobber (match_operand:SI 2 "register_operand"))]
4816   ""
4817   [(set (match_dup 2) (match_dup 1))
4818    (set (reg:CC_NZ CC_REGNUM)
4819         (compare:CC_NZ
4820          (and:GPI (match_dup 0)
4821                   (match_dup 2))
4822          (const_int 0)))]
4823 )
4824
4825 (define_insn "*and<mode>3nr_compare0_zextract"
4826   [(set (reg:CC_NZ CC_REGNUM)
4827         (compare:CC_NZ
4828          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4829                   (match_operand:GPI 1 "const_int_operand" "n")
4830                   (match_operand:GPI 2 "const_int_operand" "n"))
4831          (const_int 0)))]
4832   "INTVAL (operands[1]) > 0
4833    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4834         <= GET_MODE_BITSIZE (<MODE>mode))
4835    && aarch64_bitmask_imm (
4836         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4837                                                  operands[2])),
4838         <MODE>mode)"
4839   {
4840     operands[1]
4841       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4842     return "tst\\t%<w>0, %1";
4843   }
4844   [(set_attr "type" "logics_shift_imm")]
4845 )
4846
4847 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4848   [(set (reg:CC_NZ CC_REGNUM)
4849         (compare:CC_NZ
4850          (and:GPI (SHIFT:GPI
4851                    (match_operand:GPI 0 "register_operand" "r")
4852                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4853                   (match_operand:GPI 2 "register_operand" "r"))
4854         (const_int 0)))]
4855   ""
4856   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4857   [(set_attr "type" "logics_shift_imm")]
4858 )
4859
4860 (define_split
4861   [(set (reg:CC_NZ CC_REGNUM)
4862         (compare:CC_NZ
4863          (and:GPI (SHIFT:GPI
4864                    (match_operand:GPI 0 "register_operand")
4865                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4866                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4867         (const_int 0)))
4868     (clobber (match_operand:SI 3 "register_operand"))]
4869   ""
4870   [(set (match_dup 3) (match_dup 2))
4871    (set (reg:CC_NZ CC_REGNUM)
4872         (compare:CC_NZ
4873          (and:GPI (SHIFT:GPI
4874                    (match_dup 0)
4875                    (match_dup 1))
4876                   (match_dup 3))
4877          (const_int 0)))]
4878 )
4879
4880 ;; -------------------------------------------------------------------
4881 ;; Shifts
4882 ;; -------------------------------------------------------------------
4883
4884 (define_expand "<optab><mode>3"
4885   [(set (match_operand:GPI 0 "register_operand")
4886         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4887                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4888   ""
4889   {
4890     if (CONST_INT_P (operands[2]))
4891       {
4892         operands[2] = GEN_INT (INTVAL (operands[2])
4893                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4894
4895         if (operands[2] == const0_rtx)
4896           {
4897             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4898             DONE;
4899           }
4900       }
4901   }
4902 )
4903
4904 (define_expand "ashl<mode>3"
4905   [(set (match_operand:SHORT 0 "register_operand")
4906         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4907                       (match_operand:QI 2 "const_int_operand")))]
4908   ""
4909   {
4910     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4911
4912     if (operands[2] == const0_rtx)
4913       {
4914         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4915         DONE;
4916       }
4917   }
4918 )
4919
4920 (define_expand "rotr<mode>3"
4921   [(set (match_operand:GPI 0 "register_operand")
4922         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4923                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4924   ""
4925   {
4926     if (CONST_INT_P (operands[2]))
4927       {
4928         operands[2] = GEN_INT (INTVAL (operands[2])
4929                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4930
4931         if (operands[2] == const0_rtx)
4932           {
4933             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4934             DONE;
4935           }
4936       }
4937   }
4938 )
4939
4940 (define_expand "rotl<mode>3"
4941   [(set (match_operand:GPI 0 "register_operand")
4942         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4943                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4944   ""
4945   {
4946     /* (SZ - cnt) % SZ == -cnt % SZ */
4947     if (CONST_INT_P (operands[2]))
4948       {
4949         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4950                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4951         if (operands[2] == const0_rtx)
4952           {
4953             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4954             DONE;
4955           }
4956       }
4957     else
4958       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4959                                         NULL_RTX, 1);
4960   }
4961 )
4962
4963 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4964 ;; they truncate the shift/rotate amount by the size of the registers they
4965 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4966 ;; such redundant masking instructions.  GCC can do that automatically when
4967 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4968 ;; because some of the SISD shift alternatives don't perform this truncations.
4969 ;; So this pattern exists to catch such cases.
4970
4971 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4972   [(set (match_operand:GPI 0 "register_operand" "=r")
4973         (SHIFT:GPI
4974           (match_operand:GPI 1 "register_operand" "r")
4975           (match_operator 4 "subreg_lowpart_operator"
4976            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4977                      (match_operand 3 "const_int_operand" "n"))])))]
4978   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4979   "<shift>\t%<w>0, %<w>1, %<w>2"
4980   [(set_attr "type" "shift_reg")]
4981 )
4982
4983 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4984   [(set (match_operand:GPI 0 "register_operand" "=&r")
4985         (SHIFT:GPI
4986           (match_operand:GPI 1 "register_operand" "r")
4987           (match_operator 4 "subreg_lowpart_operator"
4988           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4989                            (match_operand 3 "const_int_operand" "n")))])))]
4990   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4991   "#"
4992   "&& true"
4993   [(const_int 0)]
4994   {
4995     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4996                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4997     emit_insn (gen_negsi2 (tmp, operands[2]));
4998
4999     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5000     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5001                                      SUBREG_BYTE (operands[4]));
5002     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5003     DONE;
5004   }
5005 )
5006
5007 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5008   [(set (match_operand:GPI 0 "register_operand" "=&r")
5009         (ashift:GPI
5010           (match_operand:GPI 1 "register_operand" "r")
5011           (minus:QI (match_operand 2 "const_int_operand" "n")
5012                     (match_operator 5 "subreg_lowpart_operator"
5013                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5014                              (match_operand 4 "const_int_operand" "n"))]))))]
5015   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5016    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5017   "#"
5018   "&& true"
5019   [(const_int 0)]
5020   {
5021     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5022                : operands[0]);
5023
5024     emit_insn (gen_negsi2 (tmp, operands[3]));
5025
5026     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5027     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5028                                      SUBREG_BYTE (operands[5]));
5029
5030     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5031     DONE;
5032   }
5033 )
5034
5035 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5036   [(set (match_operand:DI 0 "register_operand" "=r")
5037         (SHIFT:DI
5038           (match_operand:DI 1 "register_operand" "r")
5039           (match_operator 4 "subreg_lowpart_operator"
5040            [(and:SI (match_operand:SI 2 "register_operand" "r")
5041                     (match_operand 3 "const_int_operand" "n"))])))]
5042   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5043 {
5044   rtx xop[3];
5045   xop[0] = operands[0];
5046   xop[1] = operands[1];
5047   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5048   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5049   return "";
5050 }
5051   [(set_attr "type" "shift_reg")]
5052 )
5053
5054 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5055   [(set (match_operand:GPI 0 "register_operand" "=&r")
5056         (ASHIFT:GPI
5057           (match_operand:GPI 1 "register_operand" "r")
5058           (minus:QI (match_operand 2 "const_int_operand" "n")
5059                     (match_operand:QI 3 "register_operand" "r"))))]
5060   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5061   "#"
5062   "&& true"
5063   [(const_int 0)]
5064   {
5065     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5066
5067     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5068                : gen_lowpart (SImode, operands[0]));
5069
5070     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5071
5072     rtx and_op = gen_rtx_AND (SImode, tmp,
5073                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5074
5075     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5076
5077     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5078     DONE;
5079   }
5080   [(set_attr "length" "8")]
5081 )
5082
5083 ;; Logical left shift using SISD or Integer instruction
5084 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5085   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5086         (ashift:GPI
5087           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5088           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5089   ""
5090   "@
5091    lsl\t%<w>0, %<w>1, %2
5092    lsl\t%<w>0, %<w>1, %<w>2
5093    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5094    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5095   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5096    (set_attr "arch" "*,*,simd,simd")]
5097 )
5098
5099 ;; Logical right shift using SISD or Integer instruction
5100 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5101   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5102         (lshiftrt:GPI
5103          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5104          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5105                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5106   ""
5107   "@
5108    lsr\t%<w>0, %<w>1, %2
5109    lsr\t%<w>0, %<w>1, %<w>2
5110    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5111    #
5112    #"
5113   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5114    (set_attr "arch" "*,*,simd,simd,simd")]
5115 )
5116
5117 (define_split
5118   [(set (match_operand:DI 0 "aarch64_simd_register")
5119         (lshiftrt:DI
5120            (match_operand:DI 1 "aarch64_simd_register")
5121            (match_operand:QI 2 "aarch64_simd_register")))]
5122   "TARGET_SIMD && reload_completed"
5123   [(set (match_dup 3)
5124         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5125    (set (match_dup 0)
5126         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5127   {
5128     operands[3] = gen_lowpart (QImode, operands[0]);
5129   }
5130 )
5131
5132 (define_split
5133   [(set (match_operand:SI 0 "aarch64_simd_register")
5134         (lshiftrt:SI
5135            (match_operand:SI 1 "aarch64_simd_register")
5136            (match_operand:QI 2 "aarch64_simd_register")))]
5137   "TARGET_SIMD && reload_completed"
5138   [(set (match_dup 3)
5139         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5140    (set (match_dup 0)
5141         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5142   {
5143     operands[3] = gen_lowpart (QImode, operands[0]);
5144   }
5145 )
5146
5147 ;; Arithmetic right shift using SISD or Integer instruction
5148 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5149   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5150         (ashiftrt:GPI
5151           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5152           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5153                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5154   ""
5155   "@
5156    asr\t%<w>0, %<w>1, %2
5157    asr\t%<w>0, %<w>1, %<w>2
5158    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5159    #
5160    #"
5161   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5162    (set_attr "arch" "*,*,simd,simd,simd")]
5163 )
5164
5165 (define_split
5166   [(set (match_operand:DI 0 "aarch64_simd_register")
5167         (ashiftrt:DI
5168            (match_operand:DI 1 "aarch64_simd_register")
5169            (match_operand:QI 2 "aarch64_simd_register")))]
5170   "TARGET_SIMD && reload_completed"
5171   [(set (match_dup 3)
5172         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5173    (set (match_dup 0)
5174         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5175 {
5176   operands[3] = gen_lowpart (QImode, operands[0]);
5177 }
5178 )
5179
5180 (define_split
5181   [(set (match_operand:SI 0 "aarch64_simd_register")
5182         (ashiftrt:SI
5183            (match_operand:SI 1 "aarch64_simd_register")
5184            (match_operand:QI 2 "aarch64_simd_register")))]
5185   "TARGET_SIMD && reload_completed"
5186   [(set (match_dup 3)
5187         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5188    (set (match_dup 0)
5189         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5190 {
5191   operands[3] = gen_lowpart (QImode, operands[0]);
5192 }
5193 )
5194
5195 (define_insn "*aarch64_sisd_ushl"
5196   [(set (match_operand:DI 0 "register_operand" "=w")
5197         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5198                     (match_operand:QI 2 "register_operand" "w")]
5199                    UNSPEC_SISD_USHL))]
5200   "TARGET_SIMD"
5201   "ushl\t%d0, %d1, %d2"
5202   [(set_attr "type" "neon_shift_reg")]
5203 )
5204
5205 (define_insn "*aarch64_ushl_2s"
5206   [(set (match_operand:SI 0 "register_operand" "=w")
5207         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5208                     (match_operand:QI 2 "register_operand" "w")]
5209                    UNSPEC_USHL_2S))]
5210   "TARGET_SIMD"
5211   "ushl\t%0.2s, %1.2s, %2.2s"
5212   [(set_attr "type" "neon_shift_reg")]
5213 )
5214
5215 (define_insn "*aarch64_sisd_sshl"
5216   [(set (match_operand:DI 0 "register_operand" "=w")
5217         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5218                     (match_operand:QI 2 "register_operand" "w")]
5219                    UNSPEC_SISD_SSHL))]
5220   "TARGET_SIMD"
5221   "sshl\t%d0, %d1, %d2"
5222   [(set_attr "type" "neon_shift_reg")]
5223 )
5224
5225 (define_insn "*aarch64_sshl_2s"
5226   [(set (match_operand:SI 0 "register_operand" "=w")
5227         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5228                     (match_operand:QI 2 "register_operand" "w")]
5229                    UNSPEC_SSHL_2S))]
5230   "TARGET_SIMD"
5231   "sshl\t%0.2s, %1.2s, %2.2s"
5232   [(set_attr "type" "neon_shift_reg")]
5233 )
5234
5235 (define_insn "*aarch64_sisd_neg_qi"
5236   [(set (match_operand:QI 0 "register_operand" "=w")
5237         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5238                    UNSPEC_SISD_NEG))]
5239   "TARGET_SIMD"
5240   "neg\t%d0, %d1"
5241   [(set_attr "type" "neon_neg")]
5242 )
5243
5244 ;; Rotate right
5245 (define_insn "*ror<mode>3_insn"
5246   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5247      (rotatert:GPI
5248        (match_operand:GPI 1 "register_operand" "r,r")
5249        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5250   ""
5251   "@
5252    ror\\t%<w>0, %<w>1, %2
5253    ror\\t%<w>0, %<w>1, %<w>2"
5254   [(set_attr "type" "rotate_imm,shift_reg")]
5255 )
5256
5257 ;; zero_extend version of above
5258 (define_insn "*<optab>si3_insn_uxtw"
5259   [(set (match_operand:DI 0 "register_operand" "=r,r")
5260         (zero_extend:DI (SHIFT:SI
5261          (match_operand:SI 1 "register_operand" "r,r")
5262          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5263   ""
5264   "@
5265    <shift>\\t%w0, %w1, %2
5266    <shift>\\t%w0, %w1, %w2"
5267   [(set_attr "type" "bfx,shift_reg")]
5268 )
5269
5270 (define_insn "*<optab><mode>3_insn"
5271   [(set (match_operand:SHORT 0 "register_operand" "=r")
5272         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5273                       (match_operand 2 "const_int_operand" "n")))]
5274   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5275 {
5276   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5277   return "<bfshift>\t%w0, %w1, %2, %3";
5278 }
5279   [(set_attr "type" "bfx")]
5280 )
5281
5282 (define_insn "*extr<mode>5_insn"
5283   [(set (match_operand:GPI 0 "register_operand" "=r")
5284         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5285                              (match_operand 3 "const_int_operand" "n"))
5286                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5287                                (match_operand 4 "const_int_operand" "n"))))]
5288   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5289    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5290   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5291   [(set_attr "type" "rotate_imm")]
5292 )
5293
5294 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5295 ;; so we have to match both orderings.
5296 (define_insn "*extr<mode>5_insn_alt"
5297   [(set (match_operand:GPI 0 "register_operand" "=r")
5298         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5299                                 (match_operand 4 "const_int_operand" "n"))
5300                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5301                               (match_operand 3 "const_int_operand" "n"))))]
5302   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5303    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5304        == GET_MODE_BITSIZE (<MODE>mode))"
5305   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5306   [(set_attr "type" "rotate_imm")]
5307 )
5308
5309 ;; zero_extend version of the above
5310 (define_insn "*extrsi5_insn_uxtw"
5311   [(set (match_operand:DI 0 "register_operand" "=r")
5312         (zero_extend:DI
5313          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5314                             (match_operand 3 "const_int_operand" "n"))
5315                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5316                               (match_operand 4 "const_int_operand" "n")))))]
5317   "UINTVAL (operands[3]) < 32 &&
5318    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5319   "extr\\t%w0, %w1, %w2, %4"
5320   [(set_attr "type" "rotate_imm")]
5321 )
5322
5323 (define_insn "*extrsi5_insn_uxtw_alt"
5324   [(set (match_operand:DI 0 "register_operand" "=r")
5325         (zero_extend:DI
5326          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5327                                (match_operand 4 "const_int_operand" "n"))
5328                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5329                             (match_operand 3 "const_int_operand" "n")))))]
5330   "UINTVAL (operands[3]) < 32 &&
5331    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5332   "extr\\t%w0, %w1, %w2, %4"
5333   [(set_attr "type" "rotate_imm")]
5334 )
5335
5336 (define_insn "*ror<mode>3_insn"
5337   [(set (match_operand:GPI 0 "register_operand" "=r")
5338         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5339                     (match_operand 2 "const_int_operand" "n")))]
5340   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5341 {
5342   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5343   return "ror\\t%<w>0, %<w>1, %3";
5344 }
5345   [(set_attr "type" "rotate_imm")]
5346 )
5347
5348 ;; zero_extend version of the above
5349 (define_insn "*rorsi3_insn_uxtw"
5350   [(set (match_operand:DI 0 "register_operand" "=r")
5351         (zero_extend:DI
5352          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5353                     (match_operand 2 "const_int_operand" "n"))))]
5354   "UINTVAL (operands[2]) < 32"
5355 {
5356   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5357   return "ror\\t%w0, %w1, %3";
5358 }
5359   [(set_attr "type" "rotate_imm")]
5360 )
5361
5362 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5363   [(set (match_operand:GPI 0 "register_operand" "=r")
5364         (ANY_EXTEND:GPI
5365          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5366                        (match_operand 2 "const_int_operand" "n"))))]
5367   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5368 {
5369   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5370   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5371 }
5372   [(set_attr "type" "bfx")]
5373 )
5374
5375 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5376   [(set (match_operand:GPI 0 "register_operand" "=r")
5377         (zero_extend:GPI
5378          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5379                          (match_operand 2 "const_int_operand" "n"))))]
5380   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5381 {
5382   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5383   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5384 }
5385   [(set_attr "type" "bfx")]
5386 )
5387
5388 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5389   [(set (match_operand:GPI 0 "register_operand" "=r")
5390         (sign_extend:GPI
5391          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5392                          (match_operand 2 "const_int_operand" "n"))))]
5393   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5394 {
5395   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5396   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5397 }
5398   [(set_attr "type" "bfx")]
5399 )
5400
5401 ;; -------------------------------------------------------------------
5402 ;; Bitfields
5403 ;; -------------------------------------------------------------------
5404
5405 (define_expand "<optab>"
5406   [(set (match_operand:DI 0 "register_operand" "=r")
5407         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5408                         (match_operand 2
5409                           "aarch64_simd_shift_imm_offset_di")
5410                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5411   ""
5412   {
5413     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5414                    1, GET_MODE_BITSIZE (DImode) - 1))
5415      FAIL;
5416   }
5417 )
5418
5419
5420 (define_insn "*<optab><mode>"
5421   [(set (match_operand:GPI 0 "register_operand" "=r")
5422         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5423                          (match_operand 2
5424                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5425                          (match_operand 3
5426                            "aarch64_simd_shift_imm_<mode>" "n")))]
5427   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5428              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5429   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5430   [(set_attr "type" "bfx")]
5431 )
5432
5433 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5434 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5435 (define_split
5436   [(set (match_operand:DI 0 "register_operand")
5437         (zero_extract:DI (match_operand:DI 1 "register_operand")
5438                          (match_operand 2
5439                            "aarch64_simd_shift_imm_offset_di")
5440                          (match_operand 3
5441                            "aarch64_simd_shift_imm_di")))]
5442   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5443              GET_MODE_BITSIZE (DImode) - 1)
5444    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5445        == GET_MODE_BITSIZE (SImode)"
5446   [(set (match_dup 0)
5447         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5448   {
5449     operands[4] = gen_lowpart (SImode, operands[1]);
5450   }
5451 )
5452
5453 ;; Bitfield Insert (insv)
5454 (define_expand "insv<mode>"
5455   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5456                           (match_operand 1 "const_int_operand")
5457                           (match_operand 2 "const_int_operand"))
5458         (match_operand:GPI 3 "general_operand"))]
5459   ""
5460 {
5461   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5462   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5463   rtx value = operands[3];
5464
5465   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5466     FAIL;
5467
5468   if (CONST_INT_P (value))
5469     {
5470       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5471
5472       /* Prefer AND/OR for inserting all zeros or all ones.  */
5473       if ((UINTVAL (value) & mask) == 0
5474            || (UINTVAL (value) & mask) == mask)
5475         FAIL;
5476
5477       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5478       if (width == 16 && (pos % 16) == 0)
5479         DONE;
5480     }
5481   operands[3] = force_reg (<MODE>mode, value);
5482 })
5483
5484 (define_insn "*insv_reg<mode>"
5485   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5486                           (match_operand 1 "const_int_operand" "n")
5487                           (match_operand 2 "const_int_operand" "n"))
5488         (match_operand:GPI 3 "register_operand" "r"))]
5489   "!(UINTVAL (operands[1]) == 0
5490      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5491          > GET_MODE_BITSIZE (<MODE>mode)))"
5492   "bfi\\t%<w>0, %<w>3, %2, %1"
5493   [(set_attr "type" "bfm")]
5494 )
5495
5496 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5497   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5498                           (match_operand 1 "const_int_operand" "n")
5499                           (match_operand 2 "const_int_operand" "n"))
5500         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5501   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5502   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5503   [(set_attr "type" "bfm")]
5504 )
5505
5506 ;;  Match a bfi instruction where the shift of OP3 means that we are
5507 ;;  actually copying the least significant bits of OP3 into OP0 by way
5508 ;;  of the AND masks and the IOR instruction.  A similar instruction
5509 ;;  with the two parts of the IOR swapped around was never triggered
5510 ;;  in a bootstrap build and test of GCC so it was not included.
5511
5512 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5513   [(set (match_operand:GPI 0 "register_operand" "=r")
5514         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5515                           (match_operand:GPI 2 "const_int_operand" "n"))
5516                  (and:GPI (ashift:GPI
5517                            (match_operand:GPI 3 "register_operand" "r")
5518                            (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5519                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5520   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5521                                       UINTVAL (operands[4]),
5522                                       UINTVAL(operands[5]))"
5523   "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5524   [(set_attr "type" "bfm")]
5525 )
5526
5527 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5528   [(set (match_operand:GPI 0 "register_operand" "=r")
5529         (ior:GPI (and:GPI (ashift:GPI
5530                            (match_operand:GPI 1 "register_operand" "r")
5531                            (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5532                           (match_operand:GPI 3 "const_int_operand" "n"))
5533                  (and:GPI (match_operand:GPI 4 "register_operand" "0")
5534                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5535   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5536                                       UINTVAL (operands[2]),
5537                                       UINTVAL(operands[3]))"
5538   "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5539   [(set_attr "type" "bfm")]
5540 )
5541
5542 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5543 ;; the shift is large enough to remove the need for an AND instruction.
5544
5545 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5546   [(set (match_operand:GPI 0 "register_operand" "=r")
5547         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5548                           (match_operand:GPI 2 "const_int_operand" "n"))
5549                  (ashift:GPI
5550                           (match_operand:GPI 3 "register_operand" "r")
5551                           (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5552   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5553                                       UINTVAL (operands[4]),
5554                                       HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5555 {
5556   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5557   return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5558 }
5559   [(set_attr "type" "bfm")]
5560 )
5561
5562 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5563   [(set (match_operand:GPI 0 "register_operand" "=r")
5564         (ior:GPI (ashift:GPI
5565                           (match_operand:GPI 1 "register_operand" "r")
5566                           (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5567                  (and:GPI (match_operand:GPI 3 "register_operand" "0")
5568                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5569   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5570                                       UINTVAL (operands[2]),
5571                                       HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5572 {
5573   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5574   return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5575 }
5576   [(set_attr "type" "bfm")]
5577 )
5578
5579 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5580 ;; copying the least significant bits of OP3 to OP0.  We need two versions
5581 ;; of the instruction to handle different checks on the constant values.
5582
5583 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5584   [(set (match_operand:GPI 0 "register_operand" "=r")
5585         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5586                           (match_operand:GPI 2 "const_int_operand" "n"))
5587                  (and:GPI (match_operand:GPI 3 "register_operand" "r")
5588                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5589   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5590                                       UINTVAL (operands[4]))"
5591   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5592   [(set_attr "type" "bfm")]
5593 )
5594
5595 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5596   [(set (match_operand:GPI 0 "register_operand" "=r")
5597         (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5598                           (match_operand:GPI 4 "const_int_operand" "n"))
5599                  (and:GPI (match_operand:GPI 1 "register_operand" "0")
5600                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5601   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5602                                       UINTVAL (operands[4]))"
5603   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5604   [(set_attr "type" "bfm")]
5605 )
5606
5607 (define_insn "*extr_insv_lower_reg<mode>"
5608   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5609                           (match_operand 1 "const_int_operand" "n")
5610                           (const_int 0))
5611         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5612                           (match_dup 1)
5613                           (match_operand 3 "const_int_operand" "n")))]
5614   "!(UINTVAL (operands[1]) == 0
5615      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5616          > GET_MODE_BITSIZE (<MODE>mode)))"
5617   "bfxil\\t%<w>0, %<w>2, %3, %1"
5618   [(set_attr "type" "bfm")]
5619 )
5620
5621 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5622   [(set (match_operand:GPI 0 "register_operand" "=r")
5623         (ashift:GPI (ANY_EXTEND:GPI
5624                      (match_operand:ALLX 1 "register_operand" "r"))
5625                     (match_operand 2 "const_int_operand" "n")))]
5626   "UINTVAL (operands[2]) < <GPI:sizen>"
5627 {
5628   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5629               ? GEN_INT (<ALLX:sizen>)
5630               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5631   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5632 }
5633   [(set_attr "type" "bfx")]
5634 )
5635
5636 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5637
5638 (define_insn "*andim_ashift<mode>_bfiz"
5639   [(set (match_operand:GPI 0 "register_operand" "=r")
5640         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5641                              (match_operand 2 "const_int_operand" "n"))
5642                  (match_operand 3 "const_int_operand" "n")))]
5643   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5644   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5645   [(set_attr "type" "bfx")]
5646 )
5647
5648 ;; Match sbfiz pattern in a shift left + shift right operation.
5649
5650 (define_insn "*ashift<mode>_extv_bfiz"
5651   [(set (match_operand:GPI 0 "register_operand" "=r")
5652         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5653                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5654                                       (const_int 0))
5655                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5656   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5657              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5658   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5659   [(set_attr "type" "bfx")]
5660 )
5661
5662 ;; When the bit position and width of the equivalent extraction add up to 32
5663 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5664 ;; zero-extension of the X-reg.
5665 (define_split
5666   [(set (match_operand:DI 0 "register_operand")
5667         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5668                              (match_operand 2 "const_int_operand"))
5669                  (match_operand 3 "const_int_operand")))]
5670  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5671   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5672       == GET_MODE_BITSIZE (SImode)"
5673   [(set (match_dup 0)
5674         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5675   {
5676     operands[4] = gen_lowpart (SImode, operands[1]);
5677   }
5678 )
5679
5680 (define_insn "bswap<mode>2"
5681   [(set (match_operand:GPI 0 "register_operand" "=r")
5682         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5683   ""
5684   "rev\\t%<w>0, %<w>1"
5685   [(set_attr "type" "rev")]
5686 )
5687
5688 (define_insn "bswaphi2"
5689   [(set (match_operand:HI 0 "register_operand" "=r")
5690         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5691   ""
5692   "rev16\\t%w0, %w1"
5693   [(set_attr "type" "rev")]
5694 )
5695
5696 (define_insn "*aarch64_bfxil<mode>"
5697   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5698     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5699                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5700             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5701                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5702   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5703   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5704     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5705   {
5706     switch (which_alternative)
5707     {
5708       case 0:
5709         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5710         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5711       case 1:
5712         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5713         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5714       default:
5715         gcc_unreachable ();
5716     }
5717   }
5718   [(set_attr "type" "bfm")]
5719 )
5720
5721 ; Zero-extended version of above (aarch64_bfxil)
5722 (define_insn "*aarch64_bfxilsi_uxtw"
5723   [(set (match_operand:DI 0 "register_operand" "=r,r")
5724         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5725                                         "r,0")
5726                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5727             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5728                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5729   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5730   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5731     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5732   {
5733     switch (which_alternative)
5734     {
5735       case 0:
5736         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5737         return "bfxil\\t%0, %1, 0, %3";
5738       case 1:
5739         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5740         return "bfxil\\t%0, %2, 0, %3";
5741       default:
5742         gcc_unreachable ();
5743     }
5744   }
5745   [(set_attr "type" "bfm")]
5746 )
5747
5748 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5749 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5750 ;; each valid permutation.
5751
5752 (define_insn "rev16<mode>2"
5753   [(set (match_operand:GPI 0 "register_operand" "=r")
5754         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5755                                       (const_int 8))
5756                           (match_operand:GPI 3 "const_int_operand" "n"))
5757                  (and:GPI (lshiftrt:GPI (match_dup 1)
5758                                         (const_int 8))
5759                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5760   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5761    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5762   "rev16\\t%<w>0, %<w>1"
5763   [(set_attr "type" "rev")]
5764 )
5765
5766 (define_insn "rev16<mode>2_alt"
5767   [(set (match_operand:GPI 0 "register_operand" "=r")
5768         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5769                                         (const_int 8))
5770                           (match_operand:GPI 2 "const_int_operand" "n"))
5771                  (and:GPI (ashift:GPI (match_dup 1)
5772                                       (const_int 8))
5773                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5774   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5775    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5776   "rev16\\t%<w>0, %<w>1"
5777   [(set_attr "type" "rev")]
5778 )
5779
5780 ;; zero_extend version of above
5781 (define_insn "*bswapsi2_uxtw"
5782   [(set (match_operand:DI 0 "register_operand" "=r")
5783         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5784   ""
5785   "rev\\t%w0, %w1"
5786   [(set_attr "type" "rev")]
5787 )
5788
5789 ;; -------------------------------------------------------------------
5790 ;; Floating-point intrinsics
5791 ;; -------------------------------------------------------------------
5792
5793 ;; frint floating-point round to integral standard patterns.
5794 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5795
5796 (define_insn "<frint_pattern><mode>2"
5797   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5798         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5799          FRINT))]
5800   "TARGET_FLOAT"
5801   "frint<frint_suffix>\\t%<s>0, %<s>1"
5802   [(set_attr "type" "f_rint<stype>")]
5803 )
5804
5805 ;; frcvt floating-point round to integer and convert standard patterns.
5806 ;; Expands to lbtrunc, lceil, lfloor, lround.
5807 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5808   [(set (match_operand:GPI 0 "register_operand" "=r")
5809         (FIXUORS:GPI
5810           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5811            FCVT)))]
5812   "TARGET_FLOAT"
5813   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5814   [(set_attr "type" "f_cvtf2i")]
5815 )
5816
5817 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5818   [(set (match_operand:GPI 0 "register_operand" "=r")
5819         (FIXUORS:GPI
5820           (mult:GPF
5821             (match_operand:GPF 1 "register_operand" "w")
5822             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5823   "TARGET_FLOAT
5824    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5825                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5826   {
5827     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5828     char buf[64];
5829     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5830     output_asm_insn (buf, operands);
5831     return "";
5832   }
5833   [(set_attr "type" "f_cvtf2i")]
5834 )
5835
5836 ;; fma - expand fma into patterns with the accumulator operand first since
5837 ;; reusing the accumulator results in better register allocation.
5838 ;; The register allocator considers copy preferences in operand order,
5839 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5840
5841 (define_expand "fma<mode>4"
5842   [(set (match_operand:GPF_F16 0 "register_operand")
5843         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5844                      (match_operand:GPF_F16 2 "register_operand")
5845                      (match_operand:GPF_F16 3 "register_operand")))]
5846   "TARGET_FLOAT"
5847 )
5848
5849 (define_insn "*aarch64_fma<mode>4"
5850   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5851         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5852                      (match_operand:GPF_F16 3 "register_operand" "w")
5853                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5854   "TARGET_FLOAT"
5855   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5856   [(set_attr "type" "fmac<stype>")]
5857 )
5858
5859 (define_expand "fnma<mode>4"
5860   [(set (match_operand:GPF_F16 0 "register_operand")
5861         (fma:GPF_F16
5862           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5863           (match_operand:GPF_F16 2 "register_operand")
5864           (match_operand:GPF_F16 3 "register_operand")))]
5865   "TARGET_FLOAT"
5866 )
5867
5868 (define_insn "*aarch64_fnma<mode>4"
5869   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5870         (fma:GPF_F16
5871           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5872           (match_operand:GPF_F16 3 "register_operand" "w")
5873           (match_operand:GPF_F16 1 "register_operand" "w")))]
5874   "TARGET_FLOAT"
5875   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5876   [(set_attr "type" "fmac<stype>")]
5877 )
5878
5879
5880 (define_expand "fms<mode>4"
5881   [(set (match_operand:GPF 0 "register_operand")
5882         (fma:GPF (match_operand:GPF 1 "register_operand")
5883                  (match_operand:GPF 2 "register_operand")
5884                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5885   "TARGET_FLOAT"
5886 )
5887
5888 (define_insn "*aarch64_fms<mode>4"
5889   [(set (match_operand:GPF 0 "register_operand" "=w")
5890         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5891                  (match_operand:GPF 3 "register_operand" "w")
5892                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5893   "TARGET_FLOAT"
5894   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5895   [(set_attr "type" "fmac<s>")]
5896 )
5897
5898 (define_expand "fnms<mode>4"
5899   [(set (match_operand:GPF 0 "register_operand")
5900         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5901                  (match_operand:GPF 2 "register_operand")
5902                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5903   "TARGET_FLOAT"
5904 )
5905
5906 (define_insn "*aarch64_fnms<mode>4"
5907   [(set (match_operand:GPF 0 "register_operand" "=w")
5908         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5909                  (match_operand:GPF 3 "register_operand" "w")
5910                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5911   "TARGET_FLOAT"
5912   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5913   [(set_attr "type" "fmac<s>")]
5914 )
5915
5916 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5917 (define_insn "*aarch64_fnmadd<mode>4"
5918   [(set (match_operand:GPF 0 "register_operand" "=w")
5919         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5920                           (match_operand:GPF 3 "register_operand" "w")
5921                           (match_operand:GPF 1 "register_operand" "w"))))]
5922   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5923   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5924   [(set_attr "type" "fmac<s>")]
5925 )
5926
5927 ;; -------------------------------------------------------------------
5928 ;; Floating-point conversions
5929 ;; -------------------------------------------------------------------
5930
5931 (define_insn "extendsfdf2"
5932   [(set (match_operand:DF 0 "register_operand" "=w")
5933         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5934   "TARGET_FLOAT"
5935   "fcvt\\t%d0, %s1"
5936   [(set_attr "type" "f_cvt")]
5937 )
5938
5939 (define_insn "extendhfsf2"
5940   [(set (match_operand:SF 0 "register_operand" "=w")
5941         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5942   "TARGET_FLOAT"
5943   "fcvt\\t%s0, %h1"
5944   [(set_attr "type" "f_cvt")]
5945 )
5946
5947 (define_insn "extendhfdf2"
5948   [(set (match_operand:DF 0 "register_operand" "=w")
5949         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5950   "TARGET_FLOAT"
5951   "fcvt\\t%d0, %h1"
5952   [(set_attr "type" "f_cvt")]
5953 )
5954
5955 (define_insn "truncdfsf2"
5956   [(set (match_operand:SF 0 "register_operand" "=w")
5957         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5958   "TARGET_FLOAT"
5959   "fcvt\\t%s0, %d1"
5960   [(set_attr "type" "f_cvt")]
5961 )
5962
5963 (define_insn "truncsfhf2"
5964   [(set (match_operand:HF 0 "register_operand" "=w")
5965         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5966   "TARGET_FLOAT"
5967   "fcvt\\t%h0, %s1"
5968   [(set_attr "type" "f_cvt")]
5969 )
5970
5971 (define_insn "truncdfhf2"
5972   [(set (match_operand:HF 0 "register_operand" "=w")
5973         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5974   "TARGET_FLOAT"
5975   "fcvt\\t%h0, %d1"
5976   [(set_attr "type" "f_cvt")]
5977 )
5978
5979 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5980 ;; and making r = w more expensive
5981
5982 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5983   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5984         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5985   "TARGET_FLOAT"
5986   "@
5987    fcvtz<su>\t%<s>0, %<s>1
5988    fcvtz<su>\t%<w>0, %<s>1"
5989   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5990 )
5991
5992 ;; Convert HF -> SI or DI
5993
5994 (define_insn "<optab>_trunchf<GPI:mode>2"
5995   [(set (match_operand:GPI 0 "register_operand" "=r")
5996         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5997   "TARGET_FP_F16INST"
5998   "fcvtz<su>\t%<w>0, %h1"
5999   [(set_attr "type" "f_cvtf2i")]
6000 )
6001
6002 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6003 ;; input in a fp register and output in a integer register
6004
6005 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6006   [(set (match_operand:GPI 0 "register_operand" "=r")
6007         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6008   "TARGET_FLOAT"
6009   "fcvtz<su>\t%<w>0, %<fpw>1"
6010   [(set_attr "type" "f_cvtf2i")]
6011 )
6012
6013 (define_insn "*fix_to_zero_extend<mode>di2"
6014   [(set (match_operand:DI 0 "register_operand" "=r")
6015         (zero_extend:DI
6016          (unsigned_fix:SI
6017           (match_operand:GPF 1 "register_operand" "w"))))]
6018   "TARGET_FLOAT"
6019   "fcvtzu\t%w0, %<s>1"
6020   [(set_attr "type" "f_cvtf2i")]
6021 )
6022
6023 (define_insn "<optab><fcvt_target><GPF:mode>2"
6024   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6025         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6026   "TARGET_FLOAT"
6027   "@
6028    <su_optab>cvtf\t%<GPF:s>0, %<s>1
6029    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6030   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6031    (set_attr "arch" "simd,fp")]
6032 )
6033
6034 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6035   [(set (match_operand:GPF 0 "register_operand" "=w")
6036         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6037   "TARGET_FLOAT"
6038   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6039   [(set_attr "type" "f_cvti2f")]
6040 )
6041
6042 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6043 ;; midend will arrange for an SImode conversion to HFmode to first go
6044 ;; through DFmode, then to HFmode.  But first it will try converting
6045 ;; to DImode then down, which would match our DImode pattern below and
6046 ;; give very poor code-generation.  So, we must provide our own emulation
6047 ;; of the mid-end logic.
6048
6049 (define_insn "aarch64_fp16_<optab><mode>hf2"
6050   [(set (match_operand:HF 0 "register_operand" "=w")
6051         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6052   "TARGET_FP_F16INST"
6053   "<su_optab>cvtf\t%h0, %<w>1"
6054   [(set_attr "type" "f_cvti2f")]
6055 )
6056
6057 (define_expand "<optab>sihf2"
6058   [(set (match_operand:HF 0 "register_operand")
6059         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6060   "TARGET_FLOAT"
6061 {
6062   if (TARGET_FP_F16INST)
6063     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6064   else
6065     {
6066       rtx convert_target = gen_reg_rtx (DFmode);
6067       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6068       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6069     }
6070   DONE;
6071 }
6072 )
6073
6074 ;; For DImode there is no wide enough floating-point mode that we
6075 ;; can convert through natively (TFmode would work, but requires a library
6076 ;; call).  However, we know that any value >= 65504 will be rounded
6077 ;; to infinity on conversion.  This is well within the range of SImode, so
6078 ;; we can:
6079 ;;   Saturate to SImode.
6080 ;;   Convert from that to DFmode
6081 ;;   Convert from that to HFmode (phew!).
6082 ;; Note that the saturation to SImode requires the SIMD extensions.  If
6083 ;; we ever need to provide this pattern where the SIMD extensions are not
6084 ;; available, we would need a different approach.
6085
6086 (define_expand "<optab>dihf2"
6087   [(set (match_operand:HF 0 "register_operand")
6088         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6089   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6090 {
6091   if (TARGET_FP_F16INST)
6092     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6093   else
6094     {
6095       rtx sat_target = gen_reg_rtx (SImode);
6096       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6097       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6098     }
6099
6100   DONE;
6101 }
6102 )
6103
6104 ;; Convert between fixed-point and floating-point (scalar modes)
6105
6106 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6107   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6108         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6109                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6110          FCVT_F2FIXED))]
6111   ""
6112   "@
6113    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6114    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6115   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6116    (set_attr "arch" "fp,simd")]
6117 )
6118
6119 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6120   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6121         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6122                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6123          FCVT_FIXED2F))]
6124   ""
6125   "@
6126    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6127    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6128   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6129    (set_attr "arch" "fp,simd")]
6130 )
6131
6132 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6133   [(set (match_operand:GPI 0 "register_operand" "=r")
6134         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6135                      (match_operand:SI 2 "immediate_operand" "i")]
6136          FCVT_F2FIXED))]
6137   "TARGET_FP_F16INST"
6138    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6139   [(set_attr "type" "f_cvtf2i")]
6140 )
6141
6142 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6143   [(set (match_operand:HF 0 "register_operand" "=w")
6144         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6145                     (match_operand:SI 2 "immediate_operand" "i")]
6146          FCVT_FIXED2F))]
6147   "TARGET_FP_F16INST"
6148   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6149   [(set_attr "type" "f_cvti2f")]
6150 )
6151
6152 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6153   [(set (match_operand:HI 0 "register_operand" "=w")
6154         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6155                     (match_operand:SI 2 "immediate_operand" "i")]
6156          FCVT_F2FIXED))]
6157   "TARGET_SIMD"
6158   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6159   [(set_attr "type" "neon_fp_to_int_s")]
6160 )
6161
6162 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6163   [(set (match_operand:HF 0 "register_operand" "=w")
6164         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6165                     (match_operand:SI 2 "immediate_operand" "i")]
6166          FCVT_FIXED2F))]
6167   "TARGET_SIMD"
6168   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6169   [(set_attr "type" "neon_int_to_fp_s")]
6170 )
6171
6172 ;; -------------------------------------------------------------------
6173 ;; Floating-point arithmetic
6174 ;; -------------------------------------------------------------------
6175
6176 (define_insn "add<mode>3"
6177   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6178         (plus:GPF_F16
6179          (match_operand:GPF_F16 1 "register_operand" "w")
6180          (match_operand:GPF_F16 2 "register_operand" "w")))]
6181   "TARGET_FLOAT"
6182   "fadd\\t%<s>0, %<s>1, %<s>2"
6183   [(set_attr "type" "fadd<stype>")]
6184 )
6185
6186 (define_insn "sub<mode>3"
6187   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6188         (minus:GPF_F16
6189          (match_operand:GPF_F16 1 "register_operand" "w")
6190          (match_operand:GPF_F16 2 "register_operand" "w")))]
6191   "TARGET_FLOAT"
6192   "fsub\\t%<s>0, %<s>1, %<s>2"
6193   [(set_attr "type" "fadd<stype>")]
6194 )
6195
6196 (define_insn "mul<mode>3"
6197   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6198         (mult:GPF_F16
6199          (match_operand:GPF_F16 1 "register_operand" "w")
6200          (match_operand:GPF_F16 2 "register_operand" "w")))]
6201   "TARGET_FLOAT"
6202   "fmul\\t%<s>0, %<s>1, %<s>2"
6203   [(set_attr "type" "fmul<stype>")]
6204 )
6205
6206 (define_insn "*fnmul<mode>3"
6207   [(set (match_operand:GPF 0 "register_operand" "=w")
6208         (mult:GPF
6209                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6210                  (match_operand:GPF 2 "register_operand" "w")))]
6211   "TARGET_FLOAT && !flag_rounding_math"
6212   "fnmul\\t%<s>0, %<s>1, %<s>2"
6213   [(set_attr "type" "fmul<s>")]
6214 )
6215
6216 (define_insn "*fnmul<mode>3"
6217   [(set (match_operand:GPF 0 "register_operand" "=w")
6218         (neg:GPF (mult:GPF
6219                  (match_operand:GPF 1 "register_operand" "w")
6220                  (match_operand:GPF 2 "register_operand" "w"))))]
6221   "TARGET_FLOAT"
6222   "fnmul\\t%<s>0, %<s>1, %<s>2"
6223   [(set_attr "type" "fmul<s>")]
6224 )
6225
6226 (define_expand "div<mode>3"
6227  [(set (match_operand:GPF_F16 0 "register_operand")
6228        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6229                     (match_operand:GPF_F16 2 "register_operand")))]
6230  "TARGET_FLOAT"
6231 {
6232   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6233     DONE;
6234
6235   operands[1] = force_reg (<MODE>mode, operands[1]);
6236 })
6237
6238 (define_insn "*div<mode>3"
6239   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6240         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6241                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6242   "TARGET_FLOAT"
6243   "fdiv\\t%<s>0, %<s>1, %<s>2"
6244   [(set_attr "type" "fdiv<stype>")]
6245 )
6246
6247 (define_insn "neg<mode>2"
6248   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6249         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6250   "TARGET_FLOAT"
6251   "fneg\\t%<s>0, %<s>1"
6252   [(set_attr "type" "ffarith<stype>")]
6253 )
6254
6255 (define_expand "sqrt<mode>2"
6256   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6257         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6258   "TARGET_FLOAT"
6259 {
6260   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6261     DONE;
6262 })
6263
6264 (define_insn "*sqrt<mode>2"
6265   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6266         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6267   "TARGET_FLOAT"
6268   "fsqrt\\t%<s>0, %<s>1"
6269   [(set_attr "type" "fsqrt<stype>")]
6270 )
6271
6272 (define_insn "abs<mode>2"
6273   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6274         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6275   "TARGET_FLOAT"
6276   "fabs\\t%<s>0, %<s>1"
6277   [(set_attr "type" "ffarith<stype>")]
6278 )
6279
6280 ;; Given that smax/smin do not specify the result when either input is NaN,
6281 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6282 ;; for smin.
6283
6284 (define_insn "smax<mode>3"
6285   [(set (match_operand:GPF 0 "register_operand" "=w")
6286         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6287                   (match_operand:GPF 2 "register_operand" "w")))]
6288   "TARGET_FLOAT"
6289   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6290   [(set_attr "type" "f_minmax<s>")]
6291 )
6292
6293 (define_insn "smin<mode>3"
6294   [(set (match_operand:GPF 0 "register_operand" "=w")
6295         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6296                   (match_operand:GPF 2 "register_operand" "w")))]
6297   "TARGET_FLOAT"
6298   "fminnm\\t%<s>0, %<s>1, %<s>2"
6299   [(set_attr "type" "f_minmax<s>")]
6300 )
6301
6302 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6303 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6304 ;; which implement the IEEE fmax ()/fmin () functions.
6305 (define_insn "<maxmin_uns><mode>3"
6306   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6307         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6308                      (match_operand:GPF_F16 2 "register_operand" "w")]
6309                      FMAXMIN_UNS))]
6310   "TARGET_FLOAT"
6311   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6312   [(set_attr "type" "f_minmax<stype>")]
6313 )
6314
6315 (define_expand "lrint<GPF:mode><GPI:mode>2"
6316   [(match_operand:GPI 0 "register_operand")
6317    (match_operand:GPF 1 "register_operand")]
6318   "TARGET_FLOAT
6319    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
6320    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6321 {
6322   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6323   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6324   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6325   DONE;
6326 }
6327 )
6328
6329 ;; For copysign (x, y), we want to generate:
6330 ;;
6331 ;;   LDR d2, #(1 << 63)
6332 ;;   BSL v2.8b, [y], [x]
6333 ;;
6334 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6335 ;; we expect these operations to nearly always operate on
6336 ;; floating-point values, we do not want the operation to be
6337 ;; simplified into a bit-field insert operation that operates on the
6338 ;; integer side, since typically that would involve three inter-bank
6339 ;; register copies.  As we do not expect copysign to be followed by
6340 ;; other logical operations on the result, it seems preferable to keep
6341 ;; this as an unspec operation, rather than exposing the underlying
6342 ;; logic to the compiler.
6343
6344 (define_expand "copysign<GPF:mode>3"
6345   [(match_operand:GPF 0 "register_operand")
6346    (match_operand:GPF 1 "register_operand")
6347    (match_operand:GPF 2 "register_operand")]
6348   "TARGET_FLOAT && TARGET_SIMD"
6349 {
6350   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6351   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6352                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6353   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6354                                        bitmask));
6355   DONE;
6356 }
6357 )
6358
6359 (define_insn "copysign<GPF:mode>3_insn"
6360   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6361         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6362                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6363                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6364          UNSPEC_COPYSIGN))]
6365   "TARGET_FLOAT && TARGET_SIMD"
6366   "@
6367    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6368    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6369    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6370    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6371   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6372 )
6373
6374
6375 ;; For xorsign (x, y), we want to generate:
6376 ;;
6377 ;; LDR   d2, #1<<63
6378 ;; AND   v3.8B, v1.8B, v2.8B
6379 ;; EOR   v0.8B, v0.8B, v3.8B
6380 ;;
6381
6382 (define_expand "xorsign<mode>3"
6383   [(match_operand:GPF 0 "register_operand")
6384    (match_operand:GPF 1 "register_operand")
6385    (match_operand:GPF 2 "register_operand")]
6386   "TARGET_FLOAT && TARGET_SIMD"
6387 {
6388
6389   machine_mode imode = <V_INT_EQUIV>mode;
6390   rtx mask = gen_reg_rtx (imode);
6391   rtx op1x = gen_reg_rtx (imode);
6392   rtx op2x = gen_reg_rtx (imode);
6393
6394   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6395   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6396                                                      imode)));
6397
6398   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6399                                     lowpart_subreg (imode, operands[2],
6400                                                     <MODE>mode)));
6401   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6402                                     lowpart_subreg (imode, operands[1],
6403                                                     <MODE>mode),
6404                                     op2x));
6405   emit_move_insn (operands[0],
6406                   lowpart_subreg (<MODE>mode, op1x, imode));
6407   DONE;
6408 }
6409 )
6410
6411 ;; -------------------------------------------------------------------
6412 ;; Reload support
6413 ;; -------------------------------------------------------------------
6414 ;; Reload Scalar Floating point modes from constant pool.
6415 ;; The AArch64 port doesn't have __int128 constant move support.
6416 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6417  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6418        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6419   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6420  "TARGET_FLOAT"
6421  {
6422    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6423    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6424    DONE;
6425  }
6426 )
6427
6428 ;; Reload Vector modes from constant pool.
6429 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6430  [(set (match_operand:VALL 0 "register_operand" "=w")
6431        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6432   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6433  "TARGET_FLOAT"
6434  {
6435    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6436    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6437    DONE;
6438  }
6439 )
6440
6441 (define_expand "@aarch64_reload_mov<mode>"
6442   [(set (match_operand:TX 0 "register_operand" "=w")
6443         (match_operand:TX 1 "register_operand" "w"))
6444    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6445   ]
6446   "TARGET_FLOAT"
6447   {
6448     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6449     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6450     gen_aarch64_movtilow_tilow (op0, op1);
6451     gen_aarch64_movdi_tihigh (operands[2], op1);
6452     gen_aarch64_movtihigh_di (op0, operands[2]);
6453     DONE;
6454   }
6455 )
6456
6457 ;; The following secondary reload helpers patterns are invoked
6458 ;; after or during reload as we don't want these patterns to start
6459 ;; kicking in during the combiner.
6460
6461 (define_insn "@aarch64_movdi_<mode>low"
6462   [(set (match_operand:DI 0 "register_operand" "=r")
6463         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6464                          (const_int 64) (const_int 0)))]
6465   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6466   "fmov\\t%x0, %d1"
6467   [(set_attr "type" "f_mrc")
6468    (set_attr "length" "4")
6469   ])
6470
6471 (define_insn "@aarch64_movdi_<mode>high"
6472   [(set (match_operand:DI 0 "register_operand" "=r")
6473         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6474                          (const_int 64) (const_int 64)))]
6475   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6476   "fmov\\t%x0, %1.d[1]"
6477   [(set_attr "type" "f_mrc")
6478    (set_attr "length" "4")
6479   ])
6480
6481 (define_insn "@aarch64_mov<mode>high_di"
6482   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6483                          (const_int 64) (const_int 64))
6484         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6485   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6486   "fmov\\t%0.d[1], %x1"
6487   [(set_attr "type" "f_mcr")
6488    (set_attr "length" "4")
6489   ])
6490
6491 (define_insn "@aarch64_mov<mode>low_di"
6492   [(set (match_operand:TX 0 "register_operand" "=w")
6493         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6494   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6495   "fmov\\t%d0, %x1"
6496   [(set_attr "type" "f_mcr")
6497    (set_attr "length" "4")
6498   ])
6499
6500 (define_insn "aarch64_movtilow_tilow"
6501   [(set (match_operand:TI 0 "register_operand" "=w")
6502         (zero_extend:TI
6503           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6504   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6505   "fmov\\t%d0, %d1"
6506   [(set_attr "type" "fmov")
6507    (set_attr "length" "4")
6508   ])
6509
6510 ;; There is a deliberate reason why the parameters of high and lo_sum's
6511 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6512 ;; and lo_sum's to be used with the labels defining the jump tables in
6513 ;; rodata section.
6514
6515 (define_expand "add_losym"
6516   [(set (match_operand 0 "register_operand" "=r")
6517         (lo_sum (match_operand 1 "register_operand" "r")
6518                 (match_operand 2 "aarch64_valid_symref" "S")))]
6519   ""
6520 {
6521   machine_mode mode = GET_MODE (operands[0]);
6522
6523   emit_insn ((mode == DImode
6524               ? gen_add_losym_di
6525               : gen_add_losym_si) (operands[0],
6526                                    operands[1],
6527                                    operands[2]));
6528   DONE;
6529 })
6530
6531 (define_insn "add_losym_<mode>"
6532   [(set (match_operand:P 0 "register_operand" "=r")
6533         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6534                   (match_operand 2 "aarch64_valid_symref" "S")))]
6535   ""
6536   "add\\t%<w>0, %<w>1, :lo12:%c2"
6537   [(set_attr "type" "alu_imm")]
6538 )
6539
6540 (define_insn "ldr_got_small_<mode>"
6541   [(set (match_operand:PTR 0 "register_operand" "=r")
6542         (unspec:PTR [(mem:PTR (lo_sum:PTR
6543                               (match_operand:PTR 1 "register_operand" "r")
6544                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6545                     UNSPEC_GOTSMALLPIC))]
6546   ""
6547   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6548   [(set_attr "type" "load_<ldst_sz>")]
6549 )
6550
6551 (define_insn "ldr_got_small_sidi"
6552   [(set (match_operand:DI 0 "register_operand" "=r")
6553         (zero_extend:DI
6554          (unspec:SI [(mem:SI (lo_sum:DI
6555                              (match_operand:DI 1 "register_operand" "r")
6556                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6557                     UNSPEC_GOTSMALLPIC)))]
6558   "TARGET_ILP32"
6559   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6560   [(set_attr "type" "load_4")]
6561 )
6562
6563 (define_insn "ldr_got_small_28k_<mode>"
6564   [(set (match_operand:PTR 0 "register_operand" "=r")
6565         (unspec:PTR [(mem:PTR (lo_sum:PTR
6566                               (match_operand:PTR 1 "register_operand" "r")
6567                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6568                     UNSPEC_GOTSMALLPIC28K))]
6569   ""
6570   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6571   [(set_attr "type" "load_<ldst_sz>")]
6572 )
6573
6574 (define_insn "ldr_got_small_28k_sidi"
6575   [(set (match_operand:DI 0 "register_operand" "=r")
6576         (zero_extend:DI
6577          (unspec:SI [(mem:SI (lo_sum:DI
6578                              (match_operand:DI 1 "register_operand" "r")
6579                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6580                     UNSPEC_GOTSMALLPIC28K)))]
6581   "TARGET_ILP32"
6582   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6583   [(set_attr "type" "load_4")]
6584 )
6585
6586 (define_insn "ldr_got_tiny"
6587   [(set (match_operand:DI 0 "register_operand" "=r")
6588         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6589                    UNSPEC_GOTTINYPIC))]
6590   ""
6591   "ldr\\t%0, %L1"
6592   [(set_attr "type" "load_8")]
6593 )
6594
6595 (define_insn "aarch64_load_tp_hard"
6596   [(set (match_operand:DI 0 "register_operand" "=r")
6597         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6598   ""
6599   "mrs\\t%0, tpidr_el0"
6600   [(set_attr "type" "mrs")]
6601 )
6602
6603 ;; The TLS ABI specifically requires that the compiler does not schedule
6604 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6605 ;; Therefore we treat the stubs as an atomic sequence.
6606 (define_expand "tlsgd_small_<mode>"
6607  [(parallel [(set (match_operand 0 "register_operand" "")
6608                   (call (mem:DI (match_dup 2)) (const_int 1)))
6609              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
6610              (clobber (reg:DI LR_REGNUM))])]
6611  ""
6612 {
6613   operands[2] = aarch64_tls_get_addr ();
6614 })
6615
6616 (define_insn "*tlsgd_small_<mode>"
6617   [(set (match_operand 0 "register_operand" "")
6618         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6619    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6620    (clobber (reg:DI LR_REGNUM))
6621   ]
6622   ""
6623   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6624   [(set_attr "type" "call")
6625    (set_attr "length" "16")])
6626
6627 (define_insn "tlsie_small_<mode>"
6628   [(set (match_operand:PTR 0 "register_operand" "=r")
6629         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6630                    UNSPEC_GOTSMALLTLS))]
6631   ""
6632   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6633   [(set_attr "type" "load_4")
6634    (set_attr "length" "8")]
6635 )
6636
6637 (define_insn "tlsie_small_sidi"
6638   [(set (match_operand:DI 0 "register_operand" "=r")
6639         (zero_extend:DI
6640           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6641                       UNSPEC_GOTSMALLTLS)))]
6642   ""
6643   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6644   [(set_attr "type" "load_4")
6645    (set_attr "length" "8")]
6646 )
6647
6648 (define_insn "tlsie_tiny_<mode>"
6649   [(set (match_operand:PTR 0 "register_operand" "=&r")
6650         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6651                      (match_operand:PTR 2 "register_operand" "r")]
6652                    UNSPEC_GOTTINYTLS))]
6653   ""
6654   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6655   [(set_attr "type" "multiple")
6656    (set_attr "length" "8")]
6657 )
6658
6659 (define_insn "tlsie_tiny_sidi"
6660   [(set (match_operand:DI 0 "register_operand" "=&r")
6661         (zero_extend:DI
6662           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6663                       (match_operand:DI 2 "register_operand" "r")
6664                       ]
6665                       UNSPEC_GOTTINYTLS)))]
6666   ""
6667   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6668   [(set_attr "type" "multiple")
6669    (set_attr "length" "8")]
6670 )
6671
6672 (define_insn "tlsle12_<mode>"
6673   [(set (match_operand:P 0 "register_operand" "=r")
6674         (unspec:P [(match_operand:P 1 "register_operand" "r")
6675                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6676                    UNSPEC_TLSLE12))]
6677   ""
6678   "add\\t%<w>0, %<w>1, #%L2";
6679   [(set_attr "type" "alu_sreg")
6680    (set_attr "length" "4")]
6681 )
6682
6683 (define_insn "tlsle24_<mode>"
6684   [(set (match_operand:P 0 "register_operand" "=r")
6685         (unspec:P [(match_operand:P 1 "register_operand" "r")
6686                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6687                    UNSPEC_TLSLE24))]
6688   ""
6689   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6690   [(set_attr "type" "multiple")
6691    (set_attr "length" "8")]
6692 )
6693
6694 (define_insn "tlsle32_<mode>"
6695   [(set (match_operand:P 0 "register_operand" "=r")
6696         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6697                    UNSPEC_TLSLE32))]
6698   ""
6699   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6700   [(set_attr "type" "multiple")
6701    (set_attr "length" "8")]
6702 )
6703
6704 (define_insn "tlsle48_<mode>"
6705   [(set (match_operand:P 0 "register_operand" "=r")
6706         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6707                    UNSPEC_TLSLE48))]
6708   ""
6709   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6710   [(set_attr "type" "multiple")
6711    (set_attr "length" "12")]
6712 )
6713
6714 (define_expand "tlsdesc_small_<mode>"
6715   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6716   "TARGET_TLS_DESC"
6717   {
6718     if (TARGET_SVE)
6719       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6720     else
6721       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6722     DONE;
6723   }
6724 )
6725
6726 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6727 ;; R0 and LR.
6728 (define_insn "tlsdesc_small_advsimd_<mode>"
6729   [(set (reg:PTR R0_REGNUM)
6730         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6731                     UNSPEC_TLSDESC))
6732    (clobber (reg:DI LR_REGNUM))
6733    (clobber (reg:CC CC_REGNUM))
6734    (clobber (match_scratch:DI 1 "=r"))]
6735   "TARGET_TLS_DESC && !TARGET_SVE"
6736   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6737   [(set_attr "type" "call")
6738    (set_attr "length" "16")])
6739
6740 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6741 ;; all vector registers, and clobber all predicate registers, on
6742 ;; top of the usual R0 and LR.
6743 (define_insn "tlsdesc_small_sve_<mode>"
6744   [(set (reg:PTR R0_REGNUM)
6745         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6746                     UNSPEC_TLSDESC))
6747    (clobber (reg:DI LR_REGNUM))
6748    (clobber (reg:CC CC_REGNUM))
6749    (clobber_high (reg:TI V0_REGNUM))
6750    (clobber_high (reg:TI V1_REGNUM))
6751    (clobber_high (reg:TI V2_REGNUM))
6752    (clobber_high (reg:TI V3_REGNUM))
6753    (clobber_high (reg:TI V4_REGNUM))
6754    (clobber_high (reg:TI V5_REGNUM))
6755    (clobber_high (reg:TI V6_REGNUM))
6756    (clobber_high (reg:TI V7_REGNUM))
6757    (clobber_high (reg:TI V8_REGNUM))
6758    (clobber_high (reg:TI V9_REGNUM))
6759    (clobber_high (reg:TI V10_REGNUM))
6760    (clobber_high (reg:TI V11_REGNUM))
6761    (clobber_high (reg:TI V12_REGNUM))
6762    (clobber_high (reg:TI V13_REGNUM))
6763    (clobber_high (reg:TI V14_REGNUM))
6764    (clobber_high (reg:TI V15_REGNUM))
6765    (clobber_high (reg:TI V16_REGNUM))
6766    (clobber_high (reg:TI V17_REGNUM))
6767    (clobber_high (reg:TI V18_REGNUM))
6768    (clobber_high (reg:TI V19_REGNUM))
6769    (clobber_high (reg:TI V20_REGNUM))
6770    (clobber_high (reg:TI V21_REGNUM))
6771    (clobber_high (reg:TI V22_REGNUM))
6772    (clobber_high (reg:TI V23_REGNUM))
6773    (clobber_high (reg:TI V24_REGNUM))
6774    (clobber_high (reg:TI V25_REGNUM))
6775    (clobber_high (reg:TI V26_REGNUM))
6776    (clobber_high (reg:TI V27_REGNUM))
6777    (clobber_high (reg:TI V28_REGNUM))
6778    (clobber_high (reg:TI V29_REGNUM))
6779    (clobber_high (reg:TI V30_REGNUM))
6780    (clobber_high (reg:TI V31_REGNUM))
6781    (clobber (reg:VNx2BI P0_REGNUM))
6782    (clobber (reg:VNx2BI P1_REGNUM))
6783    (clobber (reg:VNx2BI P2_REGNUM))
6784    (clobber (reg:VNx2BI P3_REGNUM))
6785    (clobber (reg:VNx2BI P4_REGNUM))
6786    (clobber (reg:VNx2BI P5_REGNUM))
6787    (clobber (reg:VNx2BI P6_REGNUM))
6788    (clobber (reg:VNx2BI P7_REGNUM))
6789    (clobber (reg:VNx2BI P8_REGNUM))
6790    (clobber (reg:VNx2BI P9_REGNUM))
6791    (clobber (reg:VNx2BI P10_REGNUM))
6792    (clobber (reg:VNx2BI P11_REGNUM))
6793    (clobber (reg:VNx2BI P12_REGNUM))
6794    (clobber (reg:VNx2BI P13_REGNUM))
6795    (clobber (reg:VNx2BI P14_REGNUM))
6796    (clobber (reg:VNx2BI P15_REGNUM))
6797    (clobber (match_scratch:DI 1 "=r"))]
6798   "TARGET_TLS_DESC && TARGET_SVE"
6799   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6800   [(set_attr "type" "call")
6801    (set_attr "length" "16")])
6802
6803 (define_insn "stack_tie"
6804   [(set (mem:BLK (scratch))
6805         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6806                      (match_operand:DI 1 "register_operand" "rk")]
6807                     UNSPEC_PRLG_STK))]
6808   ""
6809   ""
6810   [(set_attr "length" "0")]
6811 )
6812
6813 ;; Pointer authentication patterns are always provided.  In architecture
6814 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6815 ;; This lets the user write portable software which authenticates pointers
6816 ;; when run on something which implements ARMv8.3-A, and which runs
6817 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6818 ;; implemented.
6819
6820 ;; Signing/Authenticating R30 using SP as the salt.
6821
6822 (define_insn "<pauth_mnem_prefix>sp"
6823   [(set (reg:DI R30_REGNUM)
6824         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6825   ""
6826   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6827 )
6828
6829 ;; Signing/Authenticating X17 using X16 as the salt.
6830
6831 (define_insn "<pauth_mnem_prefix>1716"
6832   [(set (reg:DI R17_REGNUM)
6833         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6834   ""
6835   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6836 )
6837
6838 ;; Stripping the signature in R30.
6839
6840 (define_insn "xpaclri"
6841   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6842   ""
6843   "hint\t7 // xpaclri"
6844 )
6845
6846 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6847 ;; all of memory.  This blocks insns from being moved across this point.
6848
6849 (define_insn "blockage"
6850   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6851   ""
6852   ""
6853   [(set_attr "length" "0")
6854    (set_attr "type" "block")]
6855 )
6856
6857 (define_insn "probe_stack_range"
6858   [(set (match_operand:DI 0 "register_operand" "=rk")
6859         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6860                              (match_operand:DI 2 "register_operand" "r")]
6861                               UNSPECV_PROBE_STACK_RANGE))]
6862   ""
6863 {
6864   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6865 }
6866   [(set_attr "length" "32")]
6867 )
6868
6869 ;; This instruction is used to generate the stack clash stack adjustment and
6870 ;; probing loop.  We can't change the control flow during prologue and epilogue
6871 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6872
6873 (define_insn "@probe_sve_stack_clash_<mode>"
6874   [(set (match_operand:P 0 "register_operand" "=rk")
6875         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6876                             (match_operand:P 2 "register_operand" "r")
6877                             (match_operand:P 3 "const_int_operand" "n")
6878                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6879                              UNSPECV_PROBE_STACK_RANGE))]
6880   "TARGET_SVE"
6881 {
6882   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6883                                                operands[3], operands[4]);
6884 }
6885   [(set_attr "length" "28")]
6886 )
6887
6888 ;; Named pattern for expanding thread pointer reference.
6889 (define_expand "get_thread_pointerdi"
6890   [(match_operand:DI 0 "register_operand" "=r")]
6891   ""
6892 {
6893   rtx tmp = aarch64_load_tp (operands[0]);
6894   if (tmp != operands[0])
6895     emit_move_insn (operands[0], tmp);
6896   DONE;
6897 })
6898
6899 ;; Named patterns for stack smashing protection.
6900 (define_expand "stack_protect_set"
6901   [(match_operand 0 "memory_operand")
6902    (match_operand 1 "memory_operand")]
6903   ""
6904 {
6905   machine_mode mode = GET_MODE (operands[0]);
6906   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6907   {
6908     /* Generate access through the system register.  */
6909     rtx tmp_reg = gen_reg_rtx (mode);
6910     if (mode == DImode)
6911     {
6912         emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6913         emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6914                                GEN_INT (aarch64_stack_protector_guard_offset)));
6915     }
6916     else
6917     {
6918         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6919         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6920                                GEN_INT (aarch64_stack_protector_guard_offset)));
6921
6922     }
6923     operands[1] = gen_rtx_MEM (mode, tmp_reg);
6924   }
6925
6926   emit_insn ((mode == DImode
6927               ? gen_stack_protect_set_di
6928               : gen_stack_protect_set_si) (operands[0], operands[1]));
6929   DONE;
6930 })
6931
6932 (define_insn "reg_stack_protect_address_<mode>"
6933  [(set (match_operand:PTR 0 "register_operand" "=r")
6934        (unspec:PTR [(const_int 0)]
6935         UNSPEC_SSP_SYSREG))]
6936  "aarch64_stack_protector_guard != SSP_GLOBAL"
6937  {
6938    char buf[150];
6939    snprintf (buf, 150, "mrs\\t%%<w>0, %s",
6940             aarch64_stack_protector_guard_reg_str);
6941    output_asm_insn (buf, operands);
6942    return "";
6943  }
6944  [(set_attr "type" "mrs")])
6945
6946 (define_insn "stack_protect_set_<mode>"
6947   [(set (match_operand:PTR 0 "memory_operand" "=m")
6948         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6949          UNSPEC_SP_SET))
6950    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6951   ""
6952   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6953   [(set_attr "length" "12")
6954    (set_attr "type" "multiple")])
6955
6956 (define_expand "stack_protect_test"
6957   [(match_operand 0 "memory_operand")
6958    (match_operand 1 "memory_operand")
6959    (match_operand 2)]
6960   ""
6961 {
6962   rtx result;
6963   machine_mode mode = GET_MODE (operands[0]);
6964
6965   result = gen_reg_rtx(mode);
6966   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6967   {
6968     /* Generate access through the system register. The
6969        sequence we want here is the access
6970        of the stack offset to come with
6971        mrs scratch_reg, <system_register>
6972        add scratch_reg, scratch_reg, :lo12:offset. */
6973     rtx tmp_reg = gen_reg_rtx (mode);
6974     if (mode == DImode)
6975     {
6976        emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6977        emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6978                               GEN_INT (aarch64_stack_protector_guard_offset)));
6979     }
6980     else
6981     {
6982         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6983         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6984                                GEN_INT (aarch64_stack_protector_guard_offset)));
6985
6986     }
6987     operands[1] = gen_rtx_MEM (mode, tmp_reg);
6988   }
6989   emit_insn ((mode == DImode
6990                   ? gen_stack_protect_test_di
6991                   : gen_stack_protect_test_si) (result,
6992                                                 operands[0],
6993                                                 operands[1]));
6994
6995   if (mode == DImode)
6996     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6997                                     result, const0_rtx, operands[2]));
6998   else
6999     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7000                                     result, const0_rtx, operands[2]));
7001   DONE;
7002 })
7003
7004 (define_insn "stack_protect_test_<mode>"
7005   [(set (match_operand:PTR 0 "register_operand" "=r")
7006         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7007                      (match_operand:PTR 2 "memory_operand" "m")]
7008          UNSPEC_SP_TEST))
7009    (clobber (match_scratch:PTR 3 "=&r"))]
7010   ""
7011   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7012   [(set_attr "length" "12")
7013    (set_attr "type" "multiple")])
7014
7015 ;; Write Floating-point Control Register.
7016 (define_insn "set_fpcr"
7017   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7018   ""
7019   "msr\\tfpcr, %0"
7020   [(set_attr "type" "mrs")])
7021
7022 ;; Read Floating-point Control Register.
7023 (define_insn "get_fpcr"
7024   [(set (match_operand:SI 0 "register_operand" "=r")
7025         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7026   ""
7027   "mrs\\t%0, fpcr"
7028   [(set_attr "type" "mrs")])
7029
7030 ;; Write Floating-point Status Register.
7031 (define_insn "set_fpsr"
7032   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7033   ""
7034   "msr\\tfpsr, %0"
7035   [(set_attr "type" "mrs")])
7036
7037 ;; Read Floating-point Status Register.
7038 (define_insn "get_fpsr"
7039   [(set (match_operand:SI 0 "register_operand" "=r")
7040         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7041   ""
7042   "mrs\\t%0, fpsr"
7043   [(set_attr "type" "mrs")])
7044
7045
7046 ;; Define the subtract-one-and-jump insns so loop.c
7047 ;; knows what to generate.
7048 (define_expand "doloop_end"
7049   [(use (match_operand 0 "" ""))      ; loop pseudo
7050    (use (match_operand 1 "" ""))]     ; label
7051   "optimize > 0 && flag_modulo_sched"
7052 {
7053   rtx s0;
7054   rtx bcomp;
7055   rtx loc_ref;
7056   rtx cc_reg;
7057   rtx insn;
7058   rtx cmp;
7059
7060   /* Currently SMS relies on the do-loop pattern to recognize loops
7061      where (1) the control part consists of all insns defining and/or
7062      using a certain 'count' register and (2) the loop count can be
7063      adjusted by modifying this register prior to the loop.
7064      ??? The possible introduction of a new block to initialize the
7065      new IV can potentially affect branch optimizations.  */
7066
7067   if (GET_MODE (operands[0]) != DImode)
7068     FAIL;
7069
7070   s0 = operands [0];
7071   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7072
7073   cmp = XVECEXP (PATTERN (insn), 0, 0);
7074   cc_reg = SET_DEST (cmp);
7075   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7076   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7077   emit_jump_insn (gen_rtx_SET (pc_rtx,
7078                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7079                                                      loc_ref, pc_rtx)));
7080   DONE;
7081 })
7082
7083 ;; Track speculation through conditional branches.  We assume that
7084 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7085 (define_insn "speculation_tracker"
7086   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7087         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7088          UNSPEC_SPECULATION_TRACKER))]
7089   ""
7090   {
7091     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7092     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7093     return "";
7094   }
7095   [(set_attr "type" "csel")]
7096 )
7097
7098 ;; BTI <target> instructions
7099 (define_insn "bti_noarg"
7100   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7101   ""
7102   "hint\t32 // bti"
7103   [(set_attr "type" "no_insn")]
7104 )
7105
7106 (define_insn "bti_c"
7107   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7108   ""
7109   "hint\t34 // bti c"
7110   [(set_attr "type" "no_insn")]
7111 )
7112
7113 (define_insn "bti_j"
7114   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7115   ""
7116   "hint\t36 // bti j"
7117   [(set_attr "type" "no_insn")]
7118 )
7119
7120 (define_insn "bti_jc"
7121   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7122   ""
7123   "hint\t38 // bti jc"
7124   [(set_attr "type" "no_insn")]
7125 )
7126
7127 ;; Helper for aarch64.c code.
7128 (define_expand "set_clobber_cc_nzc"
7129   [(parallel [(set (match_operand 0)
7130                    (match_operand 1))
7131               (clobber (reg:CC_NZC CC_REGNUM))])])
7132
7133 ;; Hard speculation barrier.
7134 (define_insn "speculation_barrier"
7135   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7136   ""
7137   "isb\;dsb\\tsy"
7138   [(set_attr "length" "8")
7139    (set_attr "type" "block")
7140    (set_attr "speculation_barrier" "true")]
7141 )
7142
7143 ;; Support for __builtin_speculation_safe_value when we have speculation
7144 ;; tracking enabled.  Use the speculation tracker to decide whether to
7145 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7146 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7147   [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
7148         (unspec_volatile:ALLI_TI
7149          [(match_operand:ALLI_TI 1 "register_operand" "r")
7150           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
7151           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7152           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7153   ""
7154   "
7155   {
7156     if (operands[2] == const0_rtx)
7157       {
7158         rtx tracker;
7159         if (<MODE>mode == TImode)
7160           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7161         else
7162           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7163
7164         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7165                                                  tracker));
7166         DONE;
7167       }
7168   }
7169   "
7170 )
7171
7172 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
7173 ;; encoding for CSDB, but will work in older versions of the assembler.
7174 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7175   [(set (match_operand:ALLI 0 "register_operand" "=r")
7176         (unspec_volatile:ALLI
7177          [(match_operand:ALLI 1 "register_operand" "r")
7178           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7179           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7180           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7181   ""
7182   {
7183     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7184     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7185                      operands);
7186     return "";
7187   }
7188   [(set_attr "length" "12")
7189    (set_attr "type" "block")
7190    (set_attr "speculation_barrier" "true")]
7191 )
7192
7193 ;; Pattern to match despeculate_copyti
7194 (define_insn "*despeculate_copyti_insn"
7195   [(set (match_operand:TI 0 "register_operand" "=r")
7196         (unspec_volatile:TI
7197          [(match_operand:TI 1 "register_operand" "r")
7198           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7199           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7200           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7201   ""
7202   {
7203     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7204     output_asm_insn
7205       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7206        operands);
7207     return "";
7208   }
7209   [(set_attr "length" "16")
7210    (set_attr "type" "block")
7211    (set_attr "speculation_barrier" "true")]
7212 )
7213
7214 (define_insn "despeculate_simple<ALLI:mode>"
7215   [(set (match_operand:ALLI 0 "register_operand" "=r")
7216         (unspec_volatile:ALLI
7217          [(match_operand:ALLI 1 "register_operand" "r")
7218           (use (match_operand:ALLI 2 "register_operand" ""))]
7219          UNSPECV_SPECULATION_BARRIER))]
7220   ""
7221   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7222   [(set_attr "type" "block")
7223    (set_attr "length" "8")
7224    (set_attr "speculation_barrier" "true")]
7225 )
7226
7227 (define_insn "despeculate_simpleti"
7228   [(set (match_operand:TI 0 "register_operand" "=r")
7229         (unspec_volatile:TI
7230          [(match_operand:TI 1 "register_operand" "r")
7231           (use (match_operand:DI 2 "register_operand" ""))]
7232          UNSPECV_SPECULATION_BARRIER))]
7233   ""
7234   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7235   [(set_attr "type" "block")
7236    (set_attr "length" "12")
7237    (set_attr "speculation_barrier" "true")]
7238 )
7239
7240 ;; AdvSIMD Stuff
7241 (include "aarch64-simd.md")
7242
7243 ;; Atomic Operations
7244 (include "atomics.md")
7245
7246 ;; ldp/stp peephole patterns
7247 (include "aarch64-ldpstp.md")
7248
7249 ;; SVE.
7250 (include "aarch64-sve.md")
7251
7252 ;; SVE2.
7253 (include "aarch64-sve2.md")