]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/aarch64/aarch64.md
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update copyright years.
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64.md
CommitLineData
43e9d192 1;; Machine description for AArch64 architecture.
99dee823 2;; Copyright (C) 2009-2021 Free Software Foundation, Inc.
43e9d192
IB		 3;; Contributed by ARM Ltd.
4;;
5;; This file is part of GCC.
6;;
7;; GCC is free software; you can redistribute it and/or modify it
8;; under the terms of the GNU General Public License as published by
9;; the Free Software Foundation; either version 3, or (at your option)
10;; any later version.
11;;
12;; GCC is distributed in the hope that it will be useful, but
13;; WITHOUT ANY WARRANTY; without even the implied warranty of
14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15;; General Public License for more details.
16;;
17;; You should have received a copy of the GNU General Public License
18;; along with GCC; see the file COPYING3. If not see
19;; <http://www.gnu.org/licenses/>.
20
21;; Register numbers
22(define_constants
23 [
24 (R0_REGNUM 0)
25 (R1_REGNUM 1)
26 (R2_REGNUM 2)
27 (R3_REGNUM 3)
28 (R4_REGNUM 4)
29 (R5_REGNUM 5)
30 (R6_REGNUM 6)
31 (R7_REGNUM 7)
32 (R8_REGNUM 8)
33 (R9_REGNUM 9)
34 (R10_REGNUM 10)
35 (R11_REGNUM 11)
36 (R12_REGNUM 12)
37 (R13_REGNUM 13)
38 (R14_REGNUM 14)
39 (R15_REGNUM 15)
40 (R16_REGNUM 16)
43e9d192 41 (R17_REGNUM 17)
43e9d192
IB		 42 (R18_REGNUM 18)
43 (R19_REGNUM 19)
44 (R20_REGNUM 20)
45 (R21_REGNUM 21)
46 (R22_REGNUM 22)
47 (R23_REGNUM 23)
48 (R24_REGNUM 24)
49 (R25_REGNUM 25)
50 (R26_REGNUM 26)
51 (R27_REGNUM 27)
52 (R28_REGNUM 28)
53 (R29_REGNUM 29)
54 (R30_REGNUM 30)
43e9d192
IB		 55 (SP_REGNUM 31)
56 (V0_REGNUM 32)
234c9000
AH		 57 (V1_REGNUM 33)
58 (V2_REGNUM 34)
59 (V3_REGNUM 35)
80c13ac5 60 (V4_REGNUM 36)
234c9000
AH		 61 (V5_REGNUM 37)
62 (V6_REGNUM 38)
63 (V7_REGNUM 39)
80c13ac5 64 (V8_REGNUM 40)
234c9000
AH		 65 (V9_REGNUM 41)
66 (V10_REGNUM 42)
67 (V11_REGNUM 43)
80c13ac5 68 (V12_REGNUM 44)
234c9000
AH		 69 (V13_REGNUM 45)
70 (V14_REGNUM 46)
43e9d192 71 (V15_REGNUM 47)
80c13ac5 72 (V16_REGNUM 48)
234c9000
AH		 73 (V17_REGNUM 49)
74 (V18_REGNUM 50)
75 (V19_REGNUM 51)
80c13ac5 76 (V20_REGNUM 52)
234c9000
AH		 77 (V21_REGNUM 53)
78 (V22_REGNUM 54)
79 (V23_REGNUM 55)
80c13ac5 80 (V24_REGNUM 56)
234c9000
AH		 81 (V25_REGNUM 57)
82 (V26_REGNUM 58)
83 (V27_REGNUM 59)
80c13ac5 84 (V28_REGNUM 60)
234c9000
AH		 85 (V29_REGNUM 61)
86 (V30_REGNUM 62)
43e9d192
IB		 87 (V31_REGNUM 63)
88 (SFP_REGNUM 64)
89 (AP_REGNUM 65)
90 (CC_REGNUM 66)
43cacb12
RS		 91 ;; Defined only to make the DWARF description simpler.
92 (VG_REGNUM 67)
93 (P0_REGNUM 68)
80c13ac5
RS		 94 (P1_REGNUM 69)
95 (P2_REGNUM 70)
96 (P3_REGNUM 71)
97 (P4_REGNUM 72)
98 (P5_REGNUM 73)
99 (P6_REGNUM 74)
43cacb12 100 (P7_REGNUM 75)
80c13ac5
RS		 101 (P8_REGNUM 76)
102 (P9_REGNUM 77)
103 (P10_REGNUM 78)
104 (P11_REGNUM 79)
105 (P12_REGNUM 80)
106 (P13_REGNUM 81)
107 (P14_REGNUM 82)
43cacb12 108 (P15_REGNUM 83)
c600df9a 109 (LAST_SAVED_REGNUM 83)
183bfdaf
RS		 110 (FFR_REGNUM 84)
111 ;; "FFR token": a fake register used for representing the scheduling
112 ;; restrictions on FFR-related operations.
113 (FFRT_REGNUM 85)
5773855c
OH		 114 ;; The pair of scratch registers used for stack probing with -fstack-check.
115 ;; Leave R9 alone as a possible choice for the static chain.
116 ;; Note that the use of these registers is mutually exclusive with the use
117 ;; of STACK_CLASH_SVE_CFA_REGNUM, which is for -fstack-clash-protection
118 ;; rather than -fstack-check.
119 (PROBE_STACK_FIRST_REGNUM 10)
120 (PROBE_STACK_SECOND_REGNUM 11)
143d3b15
TC		 121 ;; Scratch register used by stack clash protection to calculate
122 ;; SVE CFA offsets during probing.
123 (STACK_CLASH_SVE_CFA_REGNUM 11)
124 ;; Scratch registers for prologue/epilogue use.
125 (EP0_REGNUM 12)
126 (EP1_REGNUM 13)
3751345d
RE		 127 ;; A couple of call-clobbered registers that we need to reserve when
128 ;; tracking speculation this is not ABI, so is subject to change.
3751345d 129 (SPECULATION_SCRATCH_REGNUM 14)
143d3b15
TC		 130 (SPECULATION_TRACKER_REGNUM 15)
131 ;; Scratch registers used in frame layout.
132 (IP0_REGNUM 16)
133 (IP1_REGNUM 17)
0e510d18 134 (FP_REGNUM 29)
143d3b15 135 (LR_REGNUM 30)
43e9d192
IB		 136 ]
137)
138
139(define_c_enum "unspec" [
8fc16d72
ST		 140 UNSPEC_AUTIA1716
141 UNSPEC_AUTIB1716
142 UNSPEC_AUTIASP
143 UNSPEC_AUTIBSP
08cc4d92 144 UNSPEC_CALLEE_ABI
43e9d192 145 UNSPEC_CASESI
5d357f26
KT		 146 UNSPEC_CRC32B
147 UNSPEC_CRC32CB
148 UNSPEC_CRC32CH
149 UNSPEC_CRC32CW
150 UNSPEC_CRC32CX
151 UNSPEC_CRC32H
152 UNSPEC_CRC32W
153 UNSPEC_CRC32X
3f598afe
JW		 154 UNSPEC_FCVTZS
155 UNSPEC_FCVTZU
e1d5d19e 156 UNSPEC_FJCVTZS
10bd1d96
KT		 157 UNSPEC_FRINT32Z
158 UNSPEC_FRINT32X
159 UNSPEC_FRINT64Z
160 UNSPEC_FRINT64X
58a3bd25 161 UNSPEC_URECPE
0050faf8
JG		 162 UNSPEC_FRECPE
163 UNSPEC_FRECPS
164 UNSPEC_FRECPX
43e9d192
IB		 165 UNSPEC_FRINTA
166 UNSPEC_FRINTI
167 UNSPEC_FRINTM
0659ce6f 168 UNSPEC_FRINTN
43e9d192
IB		 169 UNSPEC_FRINTP
170 UNSPEC_FRINTX
171 UNSPEC_FRINTZ
172 UNSPEC_GOTSMALLPIC
1b1e81f8 173 UNSPEC_GOTSMALLPIC28K
43e9d192 174 UNSPEC_GOTSMALLTLS
87dd8ab0 175 UNSPEC_GOTTINYPIC
5ae7caad 176 UNSPEC_GOTTINYTLS
89b4515c 177 UNSPEC_LD1
43e9d192 178 UNSPEC_LD2
fca7d0a4 179 UNSPEC_LD2_DREG
77efea31 180 UNSPEC_LD2_DUP
43e9d192 181 UNSPEC_LD3
fca7d0a4 182 UNSPEC_LD3_DREG
77efea31 183 UNSPEC_LD3_DUP
43e9d192 184 UNSPEC_LD4
fca7d0a4 185 UNSPEC_LD4_DREG
77efea31 186 UNSPEC_LD4_DUP
3ec1be97
CB		 187 UNSPEC_LD2_LANE
188 UNSPEC_LD3_LANE
189 UNSPEC_LD4_LANE
43e9d192
IB		 190 UNSPEC_MB
191 UNSPEC_NOP
8fc16d72
ST		 192 UNSPEC_PACIA1716
193 UNSPEC_PACIB1716
194 UNSPEC_PACIASP
195 UNSPEC_PACIBSP
43e9d192 196 UNSPEC_PRLG_STK
43cacb12 197 UNSPEC_REV
43e9d192 198 UNSPEC_RBIT
75add2d0
KT		 199 UNSPEC_SABAL
200 UNSPEC_SABDL2
201 UNSPEC_SADALP
3f598afe 202 UNSPEC_SCVTF
2e100703
VP		 203 UNSPEC_SISD_NEG
204 UNSPEC_SISD_SSHL
205 UNSPEC_SISD_USHL
206 UNSPEC_SSHL_2S
89b4515c 207 UNSPEC_ST1
43e9d192
IB		 208 UNSPEC_ST2
209 UNSPEC_ST3
210 UNSPEC_ST4
ba081b77
JG		 211 UNSPEC_ST2_LANE
212 UNSPEC_ST3_LANE
213 UNSPEC_ST4_LANE
43e9d192
IB		 214 UNSPEC_TLS
215 UNSPEC_TLSDESC
cbf5629e
JW		 216 UNSPEC_TLSLE12
217 UNSPEC_TLSLE24
218 UNSPEC_TLSLE32
219 UNSPEC_TLSLE48
75add2d0
KT		 220 UNSPEC_UABAL
221 UNSPEC_UABDL2
222 UNSPEC_UADALP
3f598afe 223 UNSPEC_UCVTF
2e100703 224 UNSPEC_USHL_2S
43e9d192 225 UNSPEC_VSTRUCTDUMMY
cd0b2d36 226 UNSPEC_SSP_SYSREG
36e17020
VK		 227 UNSPEC_SP_SET
228 UNSPEC_SP_TEST
a6fc00da
BH		 229 UNSPEC_RSQRT
230 UNSPEC_RSQRTE
231 UNSPEC_RSQRTS
d9c981c8 232 UNSPEC_NZCV
db58fd89 233 UNSPEC_XPACLRI
43cacb12
RS		 234 UNSPEC_LD1_SVE
235 UNSPEC_ST1_SVE
624d0f07
RS		 236 UNSPEC_LDNT1_SVE
237 UNSPEC_STNT1_SVE
43cacb12 238 UNSPEC_LD1RQ
bfaa08b7 239 UNSPEC_LD1_GATHER
624d0f07 240 UNSPEC_LDFF1_GATHER
0a09a948 241 UNSPEC_LDNT1_GATHER
f307441a 242 UNSPEC_ST1_SCATTER
0a09a948 243 UNSPEC_STNT1_SCATTER
06308276 244 UNSPEC_PRED_X
00fa90d9 245 UNSPEC_PRED_Z
34467289 246 UNSPEC_PTEST
624d0f07 247 UNSPEC_PTRUE
43cacb12
RS		 248 UNSPEC_UNPACKSHI
249 UNSPEC_UNPACKUHI
250 UNSPEC_UNPACKSLO
251 UNSPEC_UNPACKULO
252 UNSPEC_PACK
0a09a948
RS		 253 UNSPEC_WHILEGE
254 UNSPEC_WHILEGT
255 UNSPEC_WHILEHI
256 UNSPEC_WHILEHS
6ad9571b
RS		 257 UNSPEC_WHILELE
258 UNSPEC_WHILELO
259 UNSPEC_WHILELS
260 UNSPEC_WHILELT
58c036c8
RS		 261 UNSPEC_WHILERW
262 UNSPEC_WHILEWR
9f4cbab8
RS		 263 UNSPEC_LDN
264 UNSPEC_STN
f1739b48 265 UNSPEC_INSR
624d0f07 266 UNSPEC_CLASTA
bb6c2b68 267 UNSPEC_CLASTB
b781a135 268 UNSPEC_FADDA
002092be 269 UNSPEC_REV_SUBREG
4aeb1ba7 270 UNSPEC_REINTERPRET
3751345d 271 UNSPEC_SPECULATION_TRACKER
2214053f 272 UNSPEC_SPECULATION_TRACKER_REV
315fdae8 273 UNSPEC_COPYSIGN
89626179 274 UNSPEC_TTEST ; Represent transaction test.
624d0f07
RS		 275 UNSPEC_UPDATE_FFR
276 UNSPEC_UPDATE_FFRT
277 UNSPEC_RDFFR
278 UNSPEC_WRFFR
279 ;; Represents an SVE-style lane index, in which the indexing applies
280 ;; within the containing 128-bit block.
281 UNSPEC_SVE_LANE_SELECT
282 UNSPEC_SVE_CNT_PAT
283 UNSPEC_SVE_PREFETCH
284 UNSPEC_SVE_PREFETCH_GATHER
285 UNSPEC_SVE_COMPACT
286 UNSPEC_SVE_SPLICE
ef01e6bb
DZ		 287 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag.
288 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag.
289 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space.
9ceec73f 290 UNSPEC_LD1RO
74b27d8e 291 UNSPEC_SALT_ADDR
43e9d192
IB		 292])
293
294(define_c_enum "unspecv" [
295 UNSPECV_EH_RETURN ; Represent EH_RETURN
aa87aced
KV		 296 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
297 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
298 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
299 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
a3eb8a52 300 UNSPECV_BLOCKAGE ; Represent a blockage
f58101cf 301 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
def745b2 302 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
b5f794b4
SD		 303 UNSPECV_BTI_NOARG ; Represent BTI.
304 UNSPECV_BTI_C ; Represent BTI c.
305 UNSPECV_BTI_J ; Represent BTI j.
306 UNSPECV_BTI_JC ; Represent BTI jc.
89626179
SD		 307 UNSPECV_TSTART ; Represent transaction start.
308 UNSPECV_TCOMMIT ; Represent transaction commit.
309 UNSPECV_TCANCEL ; Represent transaction cancel.
c5dc215d
KT		 310 UNSPEC_RNDR ; Represent RNDR
311 UNSPEC_RNDRRS ; Represent RNDRRS
43e9d192
IB		 312 ]
313)
314
34467289
RS		 315;; These constants are used as a const_int in various SVE unspecs
316;; to indicate whether the governing predicate is known to be a PTRUE.
317(define_constants
318 [; Indicates that the predicate might not be a PTRUE.
319 (SVE_MAYBE_NOT_PTRUE 0)
320
321 ; Indicates that the predicate is known to be a PTRUE.
322 (SVE_KNOWN_PTRUE 1)])
323
c9c5a809
RS		 324;; These constants are used as a const_int in predicated SVE FP arithmetic
325;; to indicate whether the operation is allowed to make additional lanes
326;; active without worrying about the effect on faulting behavior.
327(define_constants
328 [; Indicates either that all lanes are active or that the instruction may
329 ; operate on inactive inputs even if doing so could induce a fault.
330 (SVE_RELAXED_GP 0)
331
332 ; Indicates that some lanes might be inactive and that the instruction
333 ; must not operate on inactive inputs if doing so could induce a fault.
334 (SVE_STRICT_GP 1)])
335
43e9d192
IB		 336;; If further include files are added the defintion of MD_INCLUDES
337;; must be updated.
338
339(include "constraints.md")
340(include "predicates.md")
341(include "iterators.md")
342
43e9d192
IB		 343;; -------------------------------------------------------------------
344;; Instruction types and attributes
345;; -------------------------------------------------------------------
346
026c3cfd 347; The "type" attribute is included here from AArch32 backend to be able
34b512f5
SN		 348; to share pipeline descriptions.
349(include "../arm/types.md")
43e9d192 350
a636bb40
KT		 351;; It is important to set the fp or simd attributes to yes when a pattern
352;; alternative uses the FP or SIMD register files, usually signified by use of
353;; the 'w' constraint. This will ensure that the alternative will be
354;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
355;; architecture extensions. If all the alternatives in a pattern use the
356;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
357;; or TARGET_SIMD.
358
488461d8
MM		 359;; Attributes of the architecture required to support the instruction (or
360;; alternative). This attribute is used to compute attribute "enabled", use type
361;; "any" to enable an alternative in all cases.
43e9d192 362
3c5af608 363(define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
90e6443f 364
488461d8 365(define_enum_attr "arch" "arches" (const_string "any"))
43e9d192 366
488461d8
MM		 367;; [For compatibility with Arm in pipeline models]
368;; Attribute that specifies whether or not the instruction touches fp
369;; registers.
370;; Note that this attribute is not used anywhere in either the arm or aarch64
371;; backends except in the scheduling description for xgene1. In that
372;; scheduling description this attribute is used to subclass the load_4 and
373;; load_8 types.
374(define_attr "fp" "no,yes"
375 (if_then_else
376 (eq_attr "arch" "fp")
377 (const_string "yes")
378 (const_string "no")))
379
380(define_attr "arch_enabled" "no,yes"
381 (if_then_else
382 (ior
383 (eq_attr "arch" "any")
384
3c5af608
MM		 385 (and (eq_attr "arch" "rcpc8_4")
386 (match_test "AARCH64_ISA_RCPC8_4"))
387
488461d8
MM		 388 (and (eq_attr "arch" "fp")
389 (match_test "TARGET_FLOAT"))
390
391 (and (eq_attr "arch" "simd")
392 (match_test "TARGET_SIMD"))
393
394 (and (eq_attr "arch" "fp16")
395 (match_test "TARGET_FP_F16INST"))
396
397 (and (eq_attr "arch" "sve")
398 (match_test "TARGET_SVE")))
399 (const_string "yes")
400 (const_string "no")))
43cacb12 401
43e9d192
IB		 402;; Attribute that controls whether an alternative is enabled or not.
403;; Currently it is only used to disable alternatives which touch fp or simd
488461d8
MM		 404;; registers when -mgeneral-regs-only is specified or to require a special
405;; architecture support.
406(define_attr "enabled" "no,yes" (attr "arch_enabled"))
43e9d192 407
973d2e01
TP		 408;; Attribute that specifies whether we are dealing with a branch to a
409;; label that is far away, i.e. further away than the maximum/minimum
410;; representable in a signed 21-bits number.
411;; 0 :=: no
412;; 1 :=: yes
413(define_attr "far_branch" "" (const_int 0))
414
a08acce8
RH		 415;; Attribute that specifies whether the alternative uses MOVPRFX.
416(define_attr "movprfx" "no,yes" (const_string "no"))
417
be178ecd
MM		 418;; Attribute to specify that an alternative has the length of a single
419;; instruction plus a speculation barrier.
420(define_attr "sls_length" "none,retbr,casesi" (const_string "none"))
421
a08acce8
RH		 422(define_attr "length" ""
423 (cond [(eq_attr "movprfx" "yes")
424 (const_int 8)
be178ecd
MM		 425
426 (eq_attr "sls_length" "retbr")
427 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4)
428 (match_test "TARGET_SB") (const_int 8)]
429 (const_int 12))
430
431 (eq_attr "sls_length" "casesi")
432 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16)
433 (match_test "TARGET_SB") (const_int 20)]
434 (const_int 24))
435 ]
436 (const_int 4)))
a08acce8 437
4adcec71
EM		 438;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
439;; no predicated insns.
440(define_attr "predicated" "yes,no" (const_string "no"))
441
3751345d
RE		 442;; Set to true on an insn that requires the speculation tracking state to be
443;; in the tracking register before the insn issues. Otherwise the compiler
444;; may chose to hold the tracking state encoded in SP.
445(define_attr "speculation_barrier" "true,false" (const_string "false"))
446
43e9d192
IB		 447;; -------------------------------------------------------------------
448;; Pipeline descriptions and scheduling
449;; -------------------------------------------------------------------
450
451;; Processor types.
452(include "aarch64-tune.md")
453
454;; Scheduling
c451f4d6 455(include "../arm/cortex-a53.md")
e0ae0288 456(include "../arm/cortex-a57.md")
b2ca46df 457(include "../arm/exynos-m1.md")
825ada35 458(include "falkor.md")
2e9d500e 459(include "saphira.md")
2d41ed58 460(include "thunderx.md")
a0f06fc9 461(include "../arm/xgene1.md")
f192030d 462(include "thunderx2t99.md")
8108dfde 463(include "tsv110.md")
fa477e45 464(include "thunderx3t110.md")
43e9d192
IB		 465
466;; -------------------------------------------------------------------
467;; Jumps and other miscellaneous insns
468;; -------------------------------------------------------------------
469
470(define_insn "indirect_jump"
471 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
472 ""
be178ecd
MM		 473 {
474 output_asm_insn ("br\\t%0", operands);
475 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
476 }
477 [(set_attr "type" "branch")
478 (set_attr "sls_length" "retbr")]
43e9d192
IB		 479)
480
481(define_insn "jump"
482 [(set (pc) (label_ref (match_operand 0 "" "")))]
483 ""
484 "b\\t%l0"
c1dfcec7 485 [(set_attr "type" "branch")]
43e9d192
IB		 486)
487
488(define_expand "cbranch<mode>4"
489 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 490 [(match_operand:GPI 1 "register_operand")
491 (match_operand:GPI 2 "aarch64_plus_operand")])
43e9d192
IB		 492 (label_ref (match_operand 3 "" ""))
493 (pc)))]
494 ""
495 "
496 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
497 operands[2]);
498 operands[2] = const0_rtx;
499 "
500)
501
502(define_expand "cbranch<mode>4"
503 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 504 [(match_operand:GPF 1 "register_operand")
505 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
43e9d192
IB		 506 (label_ref (match_operand 3 "" ""))
507 (pc)))]
508 ""
509 "
510 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
511 operands[2]);
512 operands[2] = const0_rtx;
513 "
514)
515
cf670503
ZC		 516(define_expand "cbranchcc4"
517 [(set (pc) (if_then_else
518 (match_operator 0 "aarch64_comparison_operator"
1bbffb87 519 [(match_operand 1 "cc_register")
cf670503
ZC		 520 (match_operand 2 "const0_operand")])
521 (label_ref (match_operand 3 "" ""))
522 (pc)))]
523 ""
524 "")
525
865257c4
RS		 526(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>"
527 [(set (match_operand:CC_ONLY 1 "cc_register" "")
528 (if_then_else:CC_ONLY
cf670503 529 (match_operator 4 "aarch64_comparison_operator"
c8012fbc 530 [(match_operand 0 "cc_register" "")
cf670503 531 (const_int 0)])
865257c4 532 (compare:CC_ONLY
c8012fbc
WD		 533 (match_operand:GPI 2 "register_operand" "r,r,r")
534 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
865257c4
RS		 535 (unspec:CC_ONLY
536 [(match_operand 5 "immediate_operand")]
537 UNSPEC_NZCV)))]
c8012fbc 538 ""
cf670503
ZC		 539 "@
540 ccmp\\t%<w>2, %<w>3, %k5, %m4
e2b691c4 541 ccmp\\t%<w>2, %3, %k5, %m4
cf670503
ZC		 542 ccmn\\t%<w>2, #%n3, %k5, %m4"
543 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
544)
545
865257c4
RS		 546(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>"
547 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
548 (if_then_else:CCFP_CCFPE
786e3c06
WD		 549 (match_operator 4 "aarch64_comparison_operator"
550 [(match_operand 0 "cc_register" "")
551 (const_int 0)])
865257c4 552 (compare:CCFP_CCFPE
786e3c06
WD		 553 (match_operand:GPF 2 "register_operand" "w")
554 (match_operand:GPF 3 "register_operand" "w"))
865257c4
RS		 555 (unspec:CCFP_CCFPE
556 [(match_operand 5 "immediate_operand")]
557 UNSPEC_NZCV)))]
786e3c06 558 "TARGET_FLOAT"
865257c4 559 "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4"
c297d256 560 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 561)
562
865257c4
RS		 563(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev"
564 [(set (match_operand:CC_ONLY 1 "cc_register" "")
565 (if_then_else:CC_ONLY
786e3c06
WD		 566 (match_operator 4 "aarch64_comparison_operator"
567 [(match_operand 0 "cc_register" "")
865257c4
RS		 568 (const_int 0)])
569 (unspec:CC_ONLY
570 [(match_operand 5 "immediate_operand")]
571 UNSPEC_NZCV)
572 (compare:CC_ONLY
573 (match_operand:GPI 2 "register_operand" "r,r,r")
574 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))))]
575 ""
576 "@
577 ccmp\\t%<w>2, %<w>3, %k5, %M4
578 ccmp\\t%<w>2, %3, %k5, %M4
579 ccmn\\t%<w>2, #%n3, %k5, %M4"
580 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
581)
582
583(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev"
584 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
585 (if_then_else:CCFP_CCFPE
586 (match_operator 4 "aarch64_comparison_operator"
587 [(match_operand 0 "cc_register" "")
588 (const_int 0)])
589 (unspec:CCFP_CCFPE
590 [(match_operand 5 "immediate_operand")]
591 UNSPEC_NZCV)
592 (compare:CCFP_CCFPE
786e3c06 593 (match_operand:GPF 2 "register_operand" "w")
865257c4 594 (match_operand:GPF 3 "register_operand" "w"))))]
786e3c06 595 "TARGET_FLOAT"
865257c4 596 "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4"
c297d256 597 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 598)
599
4f58fe36
KT		 600;; Expansion of signed mod by a power of 2 using CSNEG.
601;; For x0 % n where n is a power of 2 produce:
602;; negs x1, x0
603;; and x0, x0, #(n - 1)
604;; and x1, x1, #(n - 1)
605;; csneg x0, x0, x1, mi
606
607(define_expand "mod<mode>3"
1bbffb87
DZ		 608 [(match_operand:GPI 0 "register_operand")
609 (match_operand:GPI 1 "register_operand")
610 (match_operand:GPI 2 "const_int_operand")]
4f58fe36
KT		 611 ""
612 {
613 HOST_WIDE_INT val = INTVAL (operands[2]);
614
615 if (val <= 0
616 || exact_log2 (val) <= 0
617 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
618 FAIL;
619
620 rtx mask = GEN_INT (val - 1);
621
622 /* In the special case of x0 % 2 we can do the even shorter:
623 cmp x0, xzr
624 and x0, x0, 1
625 cneg x0, x0, lt. */
626 if (val == 2)
627 {
628 rtx masked = gen_reg_rtx (<MODE>mode);
629 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
630 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
631 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
632 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
633 DONE;
634 }
635
636 rtx neg_op = gen_reg_rtx (<MODE>mode);
637 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
638
639 /* Extract the condition register and mode. */
640 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
641 rtx cc_reg = SET_DEST (cmp);
642 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
643
644 rtx masked_pos = gen_reg_rtx (<MODE>mode);
645 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
646
647 rtx masked_neg = gen_reg_rtx (<MODE>mode);
648 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
649
650 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
651 masked_neg, masked_pos));
652 DONE;
653 }
654)
655
714e1b3b 656(define_insn "condjump"
43e9d192
IB		 657 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
658 [(match_operand 1 "cc_register" "") (const_int 0)])
659 (label_ref (match_operand 2 "" ""))
660 (pc)))]
661 ""
973d2e01 662 {
57d6f4d0
RS		 663 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
664 but the "." is required for SVE conditions. */
665 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
973d2e01 666 if (get_attr_length (insn) == 8)
57d6f4d0
RS		 667 return aarch64_gen_far_branch (operands, 2, "Lbcond",
668 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
973d2e01 669 else
57d6f4d0 670 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
973d2e01
TP		 671 }
672 [(set_attr "type" "branch")
673 (set (attr "length")
674 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
675 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
676 (const_int 4)
677 (const_int 8)))
678 (set (attr "far_branch")
679 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
680 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
681 (const_int 0)
682 (const_int 1)))]
43e9d192
IB		 683)
684
714e1b3b
KT		 685;; For a 24-bit immediate CST we can optimize the compare for equality
686;; and branch sequence from:
687;; mov x0, #imm1
688;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
689;; cmp x1, x0
690;; b<ne,eq> .Label
691;; into the shorter:
692;; sub x0, x1, #(CST & 0xfff000)
693;; subs x0, x0, #(CST & 0x000fff)
694;; b<ne,eq> .Label
e7ba492a 695(define_insn_and_split "*compare_condjump<GPI:mode>"
714e1b3b
KT		 696 [(set (pc) (if_then_else (EQL
697 (match_operand:GPI 0 "register_operand" "r")
698 (match_operand:GPI 1 "aarch64_imm24" "n"))
699 (label_ref:P (match_operand 2 "" ""))
700 (pc)))]
e7ba492a
RS		 701 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
702 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
714e1b3b
KT		 703 && !reload_completed"
704 "#"
705 "&& true"
706 [(const_int 0)]
707 {
708 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
709 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
e7ba492a
RS		 710 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
711 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
712 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
714e1b3b 713 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
e7ba492a
RS		 714 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
715 cc_reg, const0_rtx);
714e1b3b
KT		 716 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
717 DONE;
718 }
719)
720
43e9d192 721(define_expand "casesi"
1bbffb87
DZ		 722 [(match_operand:SI 0 "register_operand") ; Index
723 (match_operand:SI 1 "const_int_operand") ; Lower bound
724 (match_operand:SI 2 "const_int_operand") ; Total range
43e9d192
IB		 725 (match_operand:DI 3 "" "") ; Table label
726 (match_operand:DI 4 "" "")] ; Out of range label
727 ""
728 {
729 if (operands[1] != const0_rtx)
730 {
731 rtx reg = gen_reg_rtx (SImode);
732
733 /* Canonical RTL says that if you have:
734
735 (minus (X) (CONST))
736
737 then this should be emitted as:
738
739 (plus (X) (-CONST))
740
741 The use of trunc_int_for_mode ensures that the resulting
742 constant can be represented in SImode, this is important
743 for the corner case where operand[1] is INT_MIN. */
744
745 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
746
747 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
748 (operands[1], SImode))
749 operands[1] = force_reg (SImode, operands[1]);
750 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
751 operands[0] = reg;
752 }
753
754 if (!aarch64_plus_operand (operands[2], SImode))
755 operands[2] = force_reg (SImode, operands[2]);
756 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
757 const0_rtx),
758 operands[0], operands[2], operands[4]));
759
4033e62c 760 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
ec302910
JJ		 761 operands[2]
762 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
763 UNSPEC_CASESI);
764 operands[2] = gen_rtx_MEM (DImode, operands[2]);
765 MEM_READONLY_P (operands[2]) = 1;
766 MEM_NOTRAP_P (operands[2]) = 1;
767 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
43e9d192
IB		 768 DONE;
769 }
770)
771
ec302910
JJ		 772(define_expand "casesi_dispatch"
773 [(parallel
774 [(set (pc) (match_operand:DI 0 ""))
775 (clobber (reg:CC CC_REGNUM))
776 (clobber (match_scratch:DI 2))
777 (clobber (match_scratch:DI 3))
778 (use (label_ref:DI (match_operand 1 "")))])]
779 "")
780
781(define_insn "*casesi_dispatch"
43e9d192
IB		 782 [(parallel
783 [(set (pc)
784 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
785 (match_operand:SI 1 "register_operand" "r")]
786 UNSPEC_CASESI)))
787 (clobber (reg:CC CC_REGNUM))
788 (clobber (match_scratch:DI 3 "=r"))
789 (clobber (match_scratch:DI 4 "=r"))
ec302910 790 (use (label_ref:DI (match_operand 2 "" "")))])]
43e9d192
IB		 791 ""
792 "*
793 return aarch64_output_casesi (operands);
794 "
be178ecd 795 [(set_attr "sls_length" "casesi")
34b512f5 796 (set_attr "type" "branch")]
43e9d192
IB		 797)
798
799(define_insn "nop"
800 [(unspec[(const_int 0)] UNSPEC_NOP)]
801 ""
802 "nop"
c1dfcec7 803 [(set_attr "type" "no_insn")]
43e9d192
IB		 804)
805
79c6bb5f 806(define_insn "prefetch"
9bf2f779 807 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
79c6bb5f
GG		 808 (match_operand:QI 1 "const_int_operand" "")
809 (match_operand:QI 2 "const_int_operand" ""))]
810 ""
811 {
9bf2f779 812 const char * pftype[2][4] =
79c6bb5f 813 {
9bf2f779
KT		 814 {"prfm\\tPLDL1STRM, %0",
815 "prfm\\tPLDL3KEEP, %0",
816 "prfm\\tPLDL2KEEP, %0",
817 "prfm\\tPLDL1KEEP, %0"},
818 {"prfm\\tPSTL1STRM, %0",
819 "prfm\\tPSTL3KEEP, %0",
820 "prfm\\tPSTL2KEEP, %0",
821 "prfm\\tPSTL1KEEP, %0"},
79c6bb5f
GG		 822 };
823
824 int locality = INTVAL (operands[2]);
825
826 gcc_assert (IN_RANGE (locality, 0, 3));
827
9bf2f779
KT		 828 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
829 the address into a DImode MEM so that aarch64_print_operand knows
830 how to print it. */
831 operands[0] = gen_rtx_MEM (DImode, operands[0]);
79c6bb5f
GG		 832 return pftype[INTVAL(operands[1])][locality];
833 }
89b2133e 834 [(set_attr "type" "load_4")]
79c6bb5f
GG		 835)
836
cd93c60e
AP		 837(define_insn "trap"
838 [(trap_if (const_int 1) (const_int 8))]
839 ""
840 "brk #1000"
841 [(set_attr "type" "trap")])
842
43e9d192
IB		 843(define_expand "prologue"
844 [(clobber (const_int 0))]
845 ""
846 "
847 aarch64_expand_prologue ();
848 DONE;
849 "
850)
851
852(define_expand "epilogue"
853 [(clobber (const_int 0))]
854 ""
855 "
856 aarch64_expand_epilogue (false);
857 DONE;
858 "
859)
860
861(define_expand "sibcall_epilogue"
862 [(clobber (const_int 0))]
863 ""
864 "
865 aarch64_expand_epilogue (true);
866 DONE;
867 "
868)
869
870(define_insn "*do_return"
871 [(return)]
872 ""
db58fd89 873 {
be178ecd 874 const char *ret = NULL;
db58fd89 875 if (aarch64_return_address_signing_enabled ()
ef33047a 876 && (TARGET_PAUTH)
db58fd89 877 && !crtl->calls_eh_return)
8fc16d72
ST		 878 {
879 if (aarch64_ra_sign_key == AARCH64_KEY_B)
be178ecd 880 ret = "retab";
8fc16d72 881 else
be178ecd 882 ret = "retaa";
8fc16d72 883 }
be178ecd
MM		 884 else
885 ret = "ret";
886 output_asm_insn (ret, operands);
887 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
db58fd89 888 }
be178ecd
MM		 889 [(set_attr "type" "branch")
890 (set_attr "sls_length" "retbr")]
43e9d192
IB		 891)
892
4f942779
RL		 893(define_expand "return"
894 [(simple_return)]
895 "aarch64_use_return_insn_p ()"
896 ""
897)
898
899(define_insn "simple_return"
900 [(simple_return)]
ce9d2a37 901 ""
be178ecd
MM		 902 {
903 output_asm_insn ("ret", operands);
904 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
905 }
906 [(set_attr "type" "branch")
907 (set_attr "sls_length" "retbr")]
4f942779
RL		 908)
909
43e9d192
IB		 910(define_insn "*cb<optab><mode>1"
911 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
912 (const_int 0))
913 (label_ref (match_operand 1 "" ""))
914 (pc)))]
6e1eaca9 915 "!aarch64_track_speculation"
973d2e01
TP		 916 {
917 if (get_attr_length (insn) == 8)
918 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
919 else
920 return "<cbz>\\t%<w>0, %l1";
921 }
922 [(set_attr "type" "branch")
923 (set (attr "length")
924 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
925 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
926 (const_int 4)
927 (const_int 8)))
928 (set (attr "far_branch")
929 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
930 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
931 (const_int 0)
932 (const_int 1)))]
43e9d192
IB		 933)
934
935(define_insn "*tb<optab><mode>1"
936 [(set (pc) (if_then_else
937 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
938 (const_int 1)
e27bfda2
KT		 939 (match_operand 1
940 "aarch64_simd_shift_imm_<mode>" "n"))
43e9d192
IB		 941 (const_int 0))
942 (label_ref (match_operand 2 "" ""))
943 (pc)))
096e8448 944 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 945 "!aarch64_track_speculation"
096e8448
JW		 946 {
947 if (get_attr_length (insn) == 8)
948 {
973d2e01
TP		 949 if (get_attr_far_branch (insn) == 1)
950 return aarch64_gen_far_branch (operands, 2, "Ltb",
951 "<inv_tb>\\t%<w>0, %1, ");
952 else
953 {
954 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
955 return "tst\t%<w>0, %1\;<bcond>\t%l2";
956 }
096e8448
JW		 957 }
958 else
959 return "<tbz>\t%<w>0, %1, %l2";
960 }
c1dfcec7 961 [(set_attr "type" "branch")
43e9d192
IB		 962 (set (attr "length")
963 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
964 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
965 (const_int 4)
973d2e01
TP		 966 (const_int 8)))
967 (set (attr "far_branch")
968 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
969 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
970 (const_int 0)
971 (const_int 1)))]
972
43e9d192
IB		 973)
974
975(define_insn "*cb<optab><mode>1"
976 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
977 (const_int 0))
978 (label_ref (match_operand 1 "" ""))
979 (pc)))
096e8448 980 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 981 "!aarch64_track_speculation"
096e8448
JW		 982 {
983 if (get_attr_length (insn) == 8)
984 {
973d2e01
TP		 985 if (get_attr_far_branch (insn) == 1)
986 return aarch64_gen_far_branch (operands, 1, "Ltb",
987 "<inv_tb>\\t%<w>0, <sizem1>, ");
988 else
989 {
990 char buf[64];
991 uint64_t val = ((uint64_t) 1)
992 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
993 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
994 output_asm_insn (buf, operands);
995 return "<bcond>\t%l1";
996 }
096e8448
JW		 997 }
998 else
999 return "<tbz>\t%<w>0, <sizem1>, %l1";
1000 }
c1dfcec7 1001 [(set_attr "type" "branch")
43e9d192
IB		 1002 (set (attr "length")
1003 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
1004 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
1005 (const_int 4)
973d2e01
TP		 1006 (const_int 8)))
1007 (set (attr "far_branch")
1008 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
1009 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
1010 (const_int 0)
1011 (const_int 1)))]
43e9d192
IB		 1012)
1013
1014;; -------------------------------------------------------------------
1015;; Subroutine calls and sibcalls
1016;; -------------------------------------------------------------------
1017
1018(define_expand "call"
08cc4d92
RS		 1019 [(parallel
1020 [(call (match_operand 0 "memory_operand")
1021 (match_operand 1 "general_operand"))
1022 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1023 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 1024 ""
1025 "
1026 {
08cc4d92 1027 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
78607708 1028 DONE;
43e9d192
IB		 1029 }"
1030)
1031
47210a04 1032(define_insn "*call_insn"
96b7f495 1033 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucr, Usf"))
43e9d192 1034 (match_operand 1 "" ""))
08cc4d92 1035 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 1036 (clobber (reg:DI LR_REGNUM))]
1037 ""
47210a04 1038 "@
96b7f495 1039 * return aarch64_indirect_call_asm (operands[0]);
d6591257 1040 bl\\t%c0"
96b7f495 1041 [(set_attr "type" "call, call")])
43e9d192
IB		 1042
1043(define_expand "call_value"
08cc4d92
RS		 1044 [(parallel
1045 [(set (match_operand 0 "")
1046 (call (match_operand 1 "memory_operand")
1047 (match_operand 2 "general_operand")))
1048 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1049 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 1050 ""
1051 "
1052 {
08cc4d92 1053 aarch64_expand_call (operands[0], operands[1], operands[3], false);
78607708 1054 DONE;
43e9d192
IB		 1055 }"
1056)
1057
47210a04 1058(define_insn "*call_value_insn"
43e9d192 1059 [(set (match_operand 0 "" "")
96b7f495 1060 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "Ucr, Usf"))
43e9d192 1061 (match_operand 2 "" "")))
08cc4d92 1062 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 1063 (clobber (reg:DI LR_REGNUM))]
1064 ""
47210a04 1065 "@
96b7f495 1066 * return aarch64_indirect_call_asm (operands[1]);
d6591257 1067 bl\\t%c1"
47210a04 1068 [(set_attr "type" "call, call")]
43e9d192
IB		 1069)
1070
1071(define_expand "sibcall"
08cc4d92
RS		 1072 [(parallel
1073 [(call (match_operand 0 "memory_operand")
1074 (match_operand 1 "general_operand"))
1075 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1076 (return)])]
43e9d192
IB		 1077 ""
1078 {
08cc4d92 1079 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
78607708 1080 DONE;
43e9d192
IB		 1081 }
1082)
1083
1084(define_expand "sibcall_value"
08cc4d92
RS		 1085 [(parallel
1086 [(set (match_operand 0 "")
1087 (call (match_operand 1 "memory_operand")
1088 (match_operand 2 "general_operand")))
1089 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1090 (return)])]
43e9d192
IB		 1091 ""
1092 {
08cc4d92 1093 aarch64_expand_call (operands[0], operands[1], operands[3], true);
78607708 1094 DONE;
43e9d192
IB		 1095 }
1096)
1097
1098(define_insn "*sibcall_insn"
fee9ba42 1099 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1100 (match_operand 1 ""))
1101 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1102 (return)]
fee9ba42 1103 "SIBLING_CALL_P (insn)"
be178ecd
MM		 1104 {
1105 if (which_alternative == 0)
1106 {
1107 output_asm_insn ("br\\t%0", operands);
1108 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1109 }
1110 return "b\\t%c0";
1111 }
1112 [(set_attr "type" "branch, branch")
1113 (set_attr "sls_length" "retbr,none")]
43e9d192
IB		 1114)
1115
1116(define_insn "*sibcall_value_insn"
08cc4d92 1117 [(set (match_operand 0 "")
aaea00aa
JG		 1118 (call (mem:DI
1119 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1120 (match_operand 2 "")))
1121 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1122 (return)]
fee9ba42 1123 "SIBLING_CALL_P (insn)"
be178ecd
MM		 1124 {
1125 if (which_alternative == 0)
1126 {
1127 output_asm_insn ("br\\t%1", operands);
1128 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1129 }
1130 return "b\\t%c1";
1131 }
1132 [(set_attr "type" "branch, branch")
1133 (set_attr "sls_length" "retbr,none")]
43e9d192
IB		 1134)
1135
1136;; Call subroutine returning any type.
1137
1138(define_expand "untyped_call"
1139 [(parallel [(call (match_operand 0 "")
1140 (const_int 0))
1141 (match_operand 1 "")
1142 (match_operand 2 "")])]
1143 ""
1144{
1145 int i;
1146
08cc4d92
RS		 1147 /* Untyped calls always use the default ABI. It's only possible to use
1148 ABI variants if we know the type of the target function. */
1149 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
43e9d192
IB		 1150
1151 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1152 {
1153 rtx set = XVECEXP (operands[2], 0, i);
1154 emit_move_insn (SET_DEST (set), SET_SRC (set));
1155 }
1156
1157 /* The optimizer does not know that the call sets the function value
1158 registers we stored in the result block. We avoid problems by
1159 claiming that all hard registers are used and clobbered at this
1160 point. */
1161 emit_insn (gen_blockage ());
1162 DONE;
1163})
1164
1165;; -------------------------------------------------------------------
1166;; Moves
1167;; -------------------------------------------------------------------
1168
1169(define_expand "mov<mode>"
1bbffb87
DZ		 1170 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1171 (match_operand:SHORT 1 "general_operand"))]
43e9d192
IB		 1172 ""
1173 "
1174 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1175 operands[1] = force_reg (<MODE>mode, operands[1]);
43cacb12
RS		 1176
1177 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1178 {
1179 aarch64_expand_mov_immediate (operands[0], operands[1]);
1180 DONE;
1181 }
43e9d192
IB		 1182 "
1183)
1184
1185(define_insn "*mov<mode>_aarch64"
43d0a8ee
WD		 1186 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1187 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
43e9d192
IB		 1188 "(register_operand (operands[0], <MODE>mode)
1189 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
b7342d25
IB		 1190{
1191 switch (which_alternative)
1192 {
1193 case 0:
1194 return "mov\t%w0, %w1";
1195 case 1:
1196 return "mov\t%w0, %1";
1197 case 2:
1198 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1199 <MODE>mode);
1200 case 3:
43cacb12 1201 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
b7342d25 1202 case 4:
43cacb12 1203 return "ldr<size>\t%w0, %1";
b7342d25 1204 case 5:
43cacb12 1205 return "ldr\t%<size>0, %1";
b7342d25 1206 case 6:
43cacb12 1207 return "str<size>\t%w1, %0";
b7342d25 1208 case 7:
43cacb12 1209 return "str\t%<size>1, %0";
b7342d25 1210 case 8:
43cacb12 1211 return "umov\t%w0, %1.<v>[0]";
b7342d25 1212 case 9:
43cacb12
RS		 1213 return "dup\t%0.<Vallxd>, %w1";
1214 case 10:
21216998 1215 return "dup\t%<Vetype>0, %1.<v>[0]";
b7342d25
IB		 1216 default:
1217 gcc_unreachable ();
1218 }
1219}
43cacb12
RS		 1220 ;; The "mov_imm" type for CNT is just a placeholder.
1221 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1222 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
488461d8 1223 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
43e9d192
IB		 1224)
1225
1226(define_expand "mov<mode>"
1bbffb87
DZ		 1227 [(set (match_operand:GPI 0 "nonimmediate_operand")
1228 (match_operand:GPI 1 "general_operand"))]
43e9d192
IB		 1229 ""
1230 "
01b9402c
KT		 1231 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1232 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
141a3ccf
KT		 1233 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1234 DONE;
1235
43e9d192
IB		 1236 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1237 operands[1] = force_reg (<MODE>mode, operands[1]);
1238
82614948
RR		 1239 /* FIXME: RR we still need to fix up what we are doing with
1240 symbol_refs and other types of constants. */
1241 if (CONSTANT_P (operands[1])
1242 && !CONST_INT_P (operands[1]))
1243 {
1244 aarch64_expand_mov_immediate (operands[0], operands[1]);
1245 DONE;
1246 }
43e9d192
IB		 1247 "
1248)
1249
82614948 1250(define_insn_and_split "*movsi_aarch64"
43cacb12 1251 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
43d0a8ee 1252 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
43e9d192
IB		 1253 "(register_operand (operands[0], SImode)
1254 || aarch64_reg_or_zero (operands[1], SImode))"
1255 "@
28514dda
YZ		 1256 mov\\t%w0, %w1
1257 mov\\t%w0, %w1
43e9d192
IB		 1258 mov\\t%w0, %w1
1259 mov\\t%w0, %1
82614948 1260 #
43cacb12 1261 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1262 ldr\\t%w0, %1
0b064172 1263 ldr\\t%s0, %1
43e9d192 1264 str\\t%w1, %0
0b064172 1265 str\\t%s1, %0
d6591257 1266 adr\\t%x0, %c1
28514dda 1267 adrp\\t%x0, %A1
43e9d192
IB		 1268 fmov\\t%s0, %w1
1269 fmov\\t%w0, %s1
a2170965
TC		 1270 fmov\\t%s0, %s1
1271 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1272 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
7263fa9f 1273 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1274 [(const_int 0)]
1275 "{
1276 aarch64_expand_mov_immediate (operands[0], operands[1]);
1277 DONE;
1278 }"
43cacb12
RS		 1279 ;; The "mov_imm" type for CNT is just a placeholder.
1280 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1281 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
488461d8 1282 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
43e9d192
IB		 1283)
1284
82614948 1285(define_insn_and_split "*movdi_aarch64"
43cacb12
RS		 1286 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1287 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
43e9d192
IB		 1288 "(register_operand (operands[0], DImode)
1289 || aarch64_reg_or_zero (operands[1], DImode))"
1290 "@
1291 mov\\t%x0, %x1
1292 mov\\t%0, %x1
1293 mov\\t%x0, %1
1294 mov\\t%x0, %1
9de00935 1295 mov\\t%w0, %1
82614948 1296 #
43cacb12 1297 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1298 ldr\\t%x0, %1
0b064172 1299 ldr\\t%d0, %1
43e9d192 1300 str\\t%x1, %0
0b064172 1301 str\\t%d1, %0
d6591257 1302 adr\\t%x0, %c1
43e9d192
IB		 1303 adrp\\t%x0, %A1
1304 fmov\\t%d0, %x1
1305 fmov\\t%x0, %d1
1306 fmov\\t%d0, %d1
a2170965 1307 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
f2a03a72 1308 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
7263fa9f 1309 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1310 [(const_int 0)]
1311 "{
1312 aarch64_expand_mov_immediate (operands[0], operands[1]);
1313 DONE;
1314 }"
43cacb12
RS		 1315 ;; The "mov_imm" type for CNTD is just a placeholder.
1316 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1317 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1318 neon_move")
488461d8 1319 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
43e9d192
IB		 1320)
1321
1322(define_insn "insv_imm<mode>"
1323 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1324 (const_int 16)
0b81f26c
JG		 1325 (match_operand:GPI 1 "const_int_operand" "n"))
1326 (match_operand:GPI 2 "const_int_operand" "n"))]
50d38551
IB		 1327 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1328 && UINTVAL (operands[1]) % 16 == 0"
a05c0ddf 1329 "movk\\t%<w>0, %X2, lsl %1"
99e6af15 1330 [(set_attr "type" "mov_imm")]
43e9d192
IB		 1331)
1332
bba0c624
RS		 1333;; Match MOVK as a normal AND and IOR operation.
1334(define_insn "aarch64_movk<mode>"
1335 [(set (match_operand:GPI 0 "register_operand" "=r")
1336 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
1337 (match_operand:GPI 2 "const_int_operand"))
1338 (match_operand:GPI 3 "const_int_operand")))]
1339 "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1340 rtx_mode_t (operands[3], <MODE>mode)) >= 0"
1341 {
1342 int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1343 rtx_mode_t (operands[3], <MODE>mode));
1344 operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode);
1345 operands[3] = gen_int_mode (shift, SImode);
1346 return "movk\\t%<w>0, #%X2, lsl %3";
1347 }
1ccdd460 1348 [(set_attr "type" "mov_imm")]
bba0c624
RS		 1349)
1350
43e9d192 1351(define_expand "movti"
1bbffb87
DZ		 1352 [(set (match_operand:TI 0 "nonimmediate_operand")
1353 (match_operand:TI 1 "general_operand"))]
43e9d192
IB		 1354 ""
1355 "
1356 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1357 operands[1] = force_reg (TImode, operands[1]);
43cacb12
RS		 1358
1359 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1360 {
1361 emit_move_insn (gen_lowpart (DImode, operands[0]),
1362 gen_lowpart (DImode, operands[1]));
1363 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1364 DONE;
1365 }
43e9d192
IB		 1366 "
1367)
1368
1369(define_insn "*movti_aarch64"
1370 [(set (match_operand:TI 0
d4f9e819 1371 "nonimmediate_operand" "= r,w,w, r,w,r,m,m,w,m")
43e9d192 1372 (match_operand:TI 1
d4f9e819 1373 "aarch64_movti_operand" " rUti,Z,r, w,w,m,r,Z,m,w"))]
43e9d192
IB		 1374 "(register_operand (operands[0], TImode)
1375 || aarch64_reg_or_zero (operands[1], TImode))"
1376 "@
1377 #
d4f9e819 1378 movi\\t%0.2d, #0
43e9d192
IB		 1379 #
1380 #
9fe9816b 1381 mov\\t%0.16b, %1.16b
43e9d192
IB		 1382 ldp\\t%0, %H0, %1
1383 stp\\t%1, %H1, %0
1384 stp\\txzr, xzr, %0
1385 ldr\\t%q0, %1
1386 str\\t%q1, %0"
d4f9e819 1387 [(set_attr "type" "multiple,neon_move,f_mcr,f_mrc,neon_logic_q, \
db46a2e6
JG		 1388 load_16,store_16,store_16,\
1389 load_16,store_16")
d4f9e819
RS		 1390 (set_attr "length" "8,4,8,8,4,4,4,4,4,4")
1391 (set_attr "arch" "*,simd,*,*,simd,*,*,*,fp,fp")]
78ec3036 1392)
43e9d192
IB		 1393
1394;; Split a TImode register-register or register-immediate move into
1395;; its component DImode pieces, taking care to handle overlapping
1396;; source and dest registers.
1397(define_split
1398 [(set (match_operand:TI 0 "register_operand" "")
1399 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1400 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1401 [(const_int 0)]
1402{
1403 aarch64_split_128bit_move (operands[0], operands[1]);
1404 DONE;
1405})
1406
1407(define_expand "mov<mode>"
abbe1ed2
SMW		 1408 [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand")
1409 (match_operand:GPF_TF_F16_MOV 1 "general_operand"))]
43e9d192 1410 ""
261fb553 1411 {
43e9d192 1412 if (!TARGET_FLOAT)
261fb553 1413 {
fc29dfc9 1414 aarch64_err_no_fpadvsimd (<MODE>mode);
248e357f
FY		 1415 machine_mode intmode
1416 = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require ();
1417 emit_move_insn (gen_lowpart (intmode, operands[0]),
1418 gen_lowpart (intmode, operands[1]));
1419 DONE;
261fb553 1420 }
43e9d192 1421
490948ca
JW		 1422 if (GET_CODE (operands[0]) == MEM
1423 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1424 && aarch64_float_const_zero_rtx_p (operands[1])))
43e9d192 1425 operands[1] = force_reg (<MODE>mode, operands[1]);
261fb553 1426 }
43e9d192
IB		 1427)
1428
abbe1ed2
SMW		 1429(define_insn "*mov<mode>_aarch64"
1430 [(set (match_operand:HFBF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1431 (match_operand:HFBF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1432 "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode)
1433 || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))"
c2ec330c 1434 "@
523d7207 1435 movi\\t%0.4h, #0
90e6443f 1436 fmov\\t%h0, %w1
d89fe63e 1437 dup\\t%w0.4h, %w1
c2ec330c
AL		 1438 umov\\t%w0, %1.h[0]
1439 mov\\t%0.h[0], %1.h[0]
90e6443f 1440 fmov\\t%h0, %1
980902fb 1441 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
c2ec330c
AL		 1442 ldr\\t%h0, %1
1443 str\\t%h1, %0
1444 ldrh\\t%w0, %1
1445 strh\\t%w1, %0
1446 mov\\t%w0, %w1"
d89fe63e 1447 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
89b2133e 1448 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
488461d8 1449 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
c2ec330c
AL		 1450)
1451
43e9d192 1452(define_insn "*movsf_aarch64"
90e6443f
TC		 1453 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1454 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
43e9d192 1455 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
701170a4 1456 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
43e9d192 1457 "@
523d7207 1458 movi\\t%0.2s, #0
43e9d192
IB		 1459 fmov\\t%s0, %w1
1460 fmov\\t%w0, %s1
1461 fmov\\t%s0, %s1
3520f7cc 1462 fmov\\t%s0, %1
90e6443f 1463 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
43e9d192
IB		 1464 ldr\\t%s0, %1
1465 str\\t%s1, %0
1466 ldr\\t%w0, %1
1467 str\\t%w1, %0
90e6443f
TC		 1468 mov\\t%w0, %w1
1469 mov\\t%w0, %1"
1470 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
89b2133e 1471 f_loads,f_stores,load_4,store_4,mov_reg,\
90e6443f 1472 fconsts")
488461d8 1473 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1474)
1475
1476(define_insn "*movdf_aarch64"
90e6443f
TC		 1477 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1478 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
43e9d192 1479 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
701170a4 1480 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
43e9d192 1481 "@
523d7207 1482 movi\\t%d0, #0
43e9d192
IB		 1483 fmov\\t%d0, %x1
1484 fmov\\t%x0, %d1
1485 fmov\\t%d0, %d1
3520f7cc 1486 fmov\\t%d0, %1
90e6443f 1487 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
43e9d192
IB		 1488 ldr\\t%d0, %1
1489 str\\t%d1, %0
1490 ldr\\t%x0, %1
1491 str\\t%x1, %0
90e6443f
TC		 1492 mov\\t%x0, %x1
1493 mov\\t%x0, %1"
1494 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
db46a2e6 1495 f_loadd,f_stored,load_8,store_8,mov_reg,\
90e6443f 1496 fconstd")
488461d8 1497 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
90e6443f
TC		 1498)
1499
1500(define_split
1501 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1502 (match_operand:GPF_HF 1 "general_operand"))]
1503 "can_create_pseudo_p ()
1504 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1505 && !aarch64_float_const_representable_p (operands[1])
1506 && aarch64_float_const_rtx_p (operands[1])"
1507 [(const_int 0)]
1508 {
1509 unsigned HOST_WIDE_INT ival;
1510 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1511 FAIL;
1512
1513 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1514 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1515 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1516 DONE;
1517 }
43e9d192
IB		 1518)
1519
43e9d192
IB		 1520(define_insn "*movtf_aarch64"
1521 [(set (match_operand:TF 0
d4f9e819 1522 "nonimmediate_operand" "=w,?r ,w ,?r,w,?w,w,m,?r,m ,m")
43e9d192 1523 (match_operand:TF 1
d4f9e819 1524 "general_operand" " w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))]
43e9d192 1525 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
6a0f8c01 1526 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
43e9d192 1527 "@
9fe9816b 1528 mov\\t%0.16b, %1.16b
12dc6974
SN		 1529 #
1530 #
1531 #
43e9d192
IB		 1532 movi\\t%0.2d, #0
1533 fmov\\t%s0, wzr
1534 ldr\\t%q0, %1
1535 str\\t%q1, %0
1536 ldp\\t%0, %H0, %1
6a0f8c01
JW		 1537 stp\\t%1, %H1, %0
1538 stp\\txzr, xzr, %0"
04e34811 1539 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
db46a2e6 1540 f_loadd,f_stored,load_16,store_16,store_16")
6a0f8c01 1541 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
488461d8 1542 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1543)
1544
12dc6974
SN		 1545(define_split
1546 [(set (match_operand:TF 0 "register_operand" "")
d4f9e819 1547 (match_operand:TF 1 "nonmemory_operand" ""))]
12dc6974
SN		 1548 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1549 [(const_int 0)]
1550 {
1551 aarch64_split_128bit_move (operands[0], operands[1]);
1552 DONE;
1553 }
1554)
1555
e2c75eea
JG		 1556;; 0 is dst
1557;; 1 is src
76715c32 1558;; 2 is size of copy in bytes
e2c75eea
JG		 1559;; 3 is alignment
1560
76715c32 1561(define_expand "cpymemdi"
e2c75eea
JG		 1562 [(match_operand:BLK 0 "memory_operand")
1563 (match_operand:BLK 1 "memory_operand")
1564 (match_operand:DI 2 "immediate_operand")
1565 (match_operand:DI 3 "immediate_operand")]
1566 "!STRICT_ALIGNMENT"
1567{
76715c32 1568 if (aarch64_expand_cpymem (operands))
e2c75eea
JG		 1569 DONE;
1570 FAIL;
1571}
1572)
1573
54bbde55
SD		 1574;; 0 is dst
1575;; 1 is val
1576;; 2 is size of copy in bytes
1577;; 3 is alignment
1578
1579(define_expand "setmemdi"
1580 [(set (match_operand:BLK 0 "memory_operand") ;; Dest
1581 (match_operand:QI 2 "nonmemory_operand")) ;; Value
1582 (use (match_operand:DI 1 "immediate_operand")) ;; Length
1583 (match_operand 3 "immediate_operand")] ;; Align
1584 "TARGET_SIMD"
1585{
1586 if (aarch64_expand_setmem (operands))
1587 DONE;
1588
1589 FAIL;
1590})
1591
43e9d192
IB		 1592;; Operands 1 and 3 are tied together by the final condition; so we allow
1593;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1594(define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1595 [(set (match_operand:SX 0 "register_operand" "=r,w")
1596 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1597 (set (match_operand:SX2 2 "register_operand" "=r,w")
1598 (match_operand:SX2 3 "memory_operand" "m,m"))]
1599 "rtx_equal_p (XEXP (operands[3], 0),
1600 plus_constant (Pmode,
1601 XEXP (operands[1], 0),
1602 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc 1603 "@
2c272091
RS		 1604 ldp\\t%w0, %w2, %z1
1605 ldp\\t%s0, %s2, %z1"
89b2133e 1606 [(set_attr "type" "load_8,neon_load1_2reg")
488461d8 1607 (set_attr "arch" "*,fp")]
43e9d192
IB		 1608)
1609
dfe1da23
JW		 1610;; Storing different modes that can still be merged
1611(define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1612 [(set (match_operand:DX 0 "register_operand" "=r,w")
1613 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1614 (set (match_operand:DX2 2 "register_operand" "=r,w")
1615 (match_operand:DX2 3 "memory_operand" "m,m"))]
1616 "rtx_equal_p (XEXP (operands[3], 0),
1617 plus_constant (Pmode,
1618 XEXP (operands[1], 0),
1619 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc 1620 "@
2c272091
RS		 1621 ldp\\t%x0, %x2, %z1
1622 ldp\\t%d0, %d2, %z1"
db46a2e6 1623 [(set_attr "type" "load_16,neon_load1_2reg")
488461d8 1624 (set_attr "arch" "*,fp")]
350013bc
BC		 1625)
1626
a0d0b980
SE		 1627(define_insn "load_pair_dw_tftf"
1628 [(set (match_operand:TF 0 "register_operand" "=w")
1629 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1630 (set (match_operand:TF 2 "register_operand" "=w")
1631 (match_operand:TF 3 "memory_operand" "m"))]
1632 "TARGET_SIMD
1633 && rtx_equal_p (XEXP (operands[3], 0),
1634 plus_constant (Pmode,
1635 XEXP (operands[1], 0),
1636 GET_MODE_SIZE (TFmode)))"
2c272091 1637 "ldp\\t%q0, %q2, %z1"
a0d0b980
SE		 1638 [(set_attr "type" "neon_ldp_q")
1639 (set_attr "fp" "yes")]
1640)
1641
43e9d192
IB		 1642;; Operands 0 and 2 are tied together by the final condition; so we allow
1643;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1644(define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1645 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1646 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1647 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1648 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1649 "rtx_equal_p (XEXP (operands[2], 0),
1650 plus_constant (Pmode,
1651 XEXP (operands[0], 0),
1652 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc 1653 "@
2c272091
RS		 1654 stp\\t%w1, %w3, %z0
1655 stp\\t%s1, %s3, %z0"
89b2133e 1656 [(set_attr "type" "store_8,neon_store1_2reg")
488461d8 1657 (set_attr "arch" "*,fp")]
350013bc
BC		 1658)
1659
dfe1da23
JW		 1660;; Storing different modes that can still be merged
1661(define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1662 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1663 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1664 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1665 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1666 "rtx_equal_p (XEXP (operands[2], 0),
1667 plus_constant (Pmode,
1668 XEXP (operands[0], 0),
1669 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc 1670 "@
2c272091
RS		 1671 stp\\t%x1, %x3, %z0
1672 stp\\t%d1, %d3, %z0"
db46a2e6 1673 [(set_attr "type" "store_16,neon_store1_2reg")
488461d8 1674 (set_attr "arch" "*,fp")]
43e9d192
IB		 1675)
1676
a0d0b980
SE		 1677(define_insn "store_pair_dw_tftf"
1678 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1679 (match_operand:TF 1 "register_operand" "w"))
1680 (set (match_operand:TF 2 "memory_operand" "=m")
1681 (match_operand:TF 3 "register_operand" "w"))]
1682 "TARGET_SIMD &&
1683 rtx_equal_p (XEXP (operands[2], 0),
1684 plus_constant (Pmode,
1685 XEXP (operands[0], 0),
1686 GET_MODE_SIZE (TFmode)))"
2c272091 1687 "stp\\t%q1, %q3, %z0"
a0d0b980
SE		 1688 [(set_attr "type" "neon_stp_q")
1689 (set_attr "fp" "yes")]
1690)
1691
3e322b3f
JW		 1692;; Load pair with post-index writeback. This is primarily used in function
1693;; epilogues.
28514dda 1694(define_insn "loadwb_pair<GPI:mode>_<P:mode>"
43e9d192 1695 [(parallel
28514dda
YZ		 1696 [(set (match_operand:P 0 "register_operand" "=k")
1697 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1698 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
43e9d192 1699 (set (match_operand:GPI 2 "register_operand" "=r")
3e322b3f 1700 (mem:GPI (match_dup 1)))
43e9d192 1701 (set (match_operand:GPI 3 "register_operand" "=r")
28514dda
YZ		 1702 (mem:GPI (plus:P (match_dup 1)
1703 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1704 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1705 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1706 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
43e9d192
IB		 1707)
1708
159313d9
JW		 1709(define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1710 [(parallel
1711 [(set (match_operand:P 0 "register_operand" "=k")
1712 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1713 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
159313d9 1714 (set (match_operand:GPF 2 "register_operand" "=w")
3e322b3f 1715 (mem:GPF (match_dup 1)))
159313d9
JW		 1716 (set (match_operand:GPF 3 "register_operand" "=w")
1717 (mem:GPF (plus:P (match_dup 1)
1718 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1719 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1720 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
159313d9
JW		 1721 [(set_attr "type" "neon_load1_2reg")]
1722)
1723
a0d0b980
SE		 1724(define_insn "loadwb_pair<TX:mode>_<P:mode>"
1725 [(parallel
1726 [(set (match_operand:P 0 "register_operand" "=k")
1727 (plus:P (match_operand:P 1 "register_operand" "0")
1728 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1729 (set (match_operand:TX 2 "register_operand" "=w")
1730 (mem:TX (match_dup 1)))
1731 (set (match_operand:TX 3 "register_operand" "=w")
1732 (mem:TX (plus:P (match_dup 1)
1733 (match_operand:P 5 "const_int_operand" "n"))))])]
1734 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1735 "ldp\\t%q2, %q3, [%1], %4"
1736 [(set_attr "type" "neon_ldp_q")]
1737)
1738
3e322b3f
JW		 1739;; Store pair with pre-index writeback. This is primarily used in function
1740;; prologues.
28514dda 1741(define_insn "storewb_pair<GPI:mode>_<P:mode>"
43e9d192 1742 [(parallel
28514dda
YZ		 1743 [(set (match_operand:P 0 "register_operand" "=&k")
1744 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1745 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
28514dda 1746 (set (mem:GPI (plus:P (match_dup 0)
43e9d192
IB		 1747 (match_dup 4)))
1748 (match_operand:GPI 2 "register_operand" "r"))
28514dda
YZ		 1749 (set (mem:GPI (plus:P (match_dup 0)
1750 (match_operand:P 5 "const_int_operand" "n")))
43e9d192
IB		 1751 (match_operand:GPI 3 "register_operand" "r"))])]
1752 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1753 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1754 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
80c11907
JW		 1755)
1756
1757(define_insn "storewb_pair<GPF:mode>_<P:mode>"
1758 [(parallel
1759 [(set (match_operand:P 0 "register_operand" "=&k")
1760 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1761 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
80c11907
JW		 1762 (set (mem:GPF (plus:P (match_dup 0)
1763 (match_dup 4)))
1764 (match_operand:GPF 2 "register_operand" "w"))
1765 (set (mem:GPF (plus:P (match_dup 0)
1766 (match_operand:P 5 "const_int_operand" "n")))
1767 (match_operand:GPF 3 "register_operand" "w"))])]
1768 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1769 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
80c11907 1770 [(set_attr "type" "neon_store1_2reg<q>")]
43e9d192
IB		 1771)
1772
a0d0b980
SE		 1773(define_insn "storewb_pair<TX:mode>_<P:mode>"
1774 [(parallel
1775 [(set (match_operand:P 0 "register_operand" "=&k")
1776 (plus:P (match_operand:P 1 "register_operand" "0")
1777 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1778 (set (mem:TX (plus:P (match_dup 0)
1779 (match_dup 4)))
1780 (match_operand:TX 2 "register_operand" "w"))
1781 (set (mem:TX (plus:P (match_dup 0)
1782 (match_operand:P 5 "const_int_operand" "n")))
1783 (match_operand:TX 3 "register_operand" "w"))])]
1784 "TARGET_SIMD
1785 && INTVAL (operands[5])
1786 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1787 "stp\\t%q2, %q3, [%0, %4]!"
1788 [(set_attr "type" "neon_stp_q")]
1789)
1790
43e9d192
IB		 1791;; -------------------------------------------------------------------
1792;; Sign/Zero extension
1793;; -------------------------------------------------------------------
1794
1795(define_expand "<optab>sidi2"
1796 [(set (match_operand:DI 0 "register_operand")
1797 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1798 ""
1799)
1800
1801(define_insn "*extendsidi2_aarch64"
1802 [(set (match_operand:DI 0 "register_operand" "=r,r")
1803 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1804 ""
1805 "@
1806 sxtw\t%0, %w1
1807 ldrsw\t%0, %1"
89b2133e 1808 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1809)
1810
350013bc
BC		 1811(define_insn "*load_pair_extendsidi2_aarch64"
1812 [(set (match_operand:DI 0 "register_operand" "=r")
1813 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1814 (set (match_operand:DI 2 "register_operand" "=r")
1815 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1816 "rtx_equal_p (XEXP (operands[3], 0),
1817 plus_constant (Pmode,
1818 XEXP (operands[1], 0),
1819 GET_MODE_SIZE (SImode)))"
2c272091 1820 "ldpsw\\t%0, %2, %z1"
89b2133e 1821 [(set_attr "type" "load_8")]
350013bc
BC		 1822)
1823
43e9d192 1824(define_insn "*zero_extendsidi2_aarch64"
0cfc095c
WD		 1825 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1826 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
43e9d192
IB		 1827 ""
1828 "@
1829 uxtw\t%0, %w1
0cfc095c
WD		 1830 ldr\t%w0, %1
1831 fmov\t%s0, %w1
1832 ldr\t%s0, %1
1833 fmov\t%w0, %s1
1834 fmov\t%s0, %s1"
e6b85350 1835 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
0cfc095c 1836 (set_attr "arch" "*,*,fp,fp,fp,fp")]
43e9d192
IB		 1837)
1838
350013bc 1839(define_insn "*load_pair_zero_extendsidi2_aarch64"
0cfc095c
WD		 1840 [(set (match_operand:DI 0 "register_operand" "=r,w")
1841 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1842 (set (match_operand:DI 2 "register_operand" "=r,w")
1843 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
350013bc
BC		 1844 "rtx_equal_p (XEXP (operands[3], 0),
1845 plus_constant (Pmode,
1846 XEXP (operands[1], 0),
1847 GET_MODE_SIZE (SImode)))"
0cfc095c 1848 "@
2c272091
RS		 1849 ldp\t%w0, %w2, %z1
1850 ldp\t%s0, %s2, %z1"
0cfc095c
WD		 1851 [(set_attr "type" "load_8,neon_load1_2reg")
1852 (set_attr "arch" "*,fp")]
350013bc
BC		 1853)
1854
43e9d192
IB		 1855(define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1856 [(set (match_operand:GPI 0 "register_operand")
1857 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1858 ""
1859)
1860
1861(define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1862 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1863 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1864 ""
1865 "@
1866 sxt<SHORT:size>\t%<GPI:w>0, %w1
1867 ldrs<SHORT:size>\t%<GPI:w>0, %1"
89b2133e 1868 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1869)
1870
1871(define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
43d0a8ee 1872 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
c902d3c8 1873 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
43e9d192
IB		 1874 ""
1875 "@
63715e5e 1876 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
c902d3c8
SN		 1877 ldr<SHORT:size>\t%w0, %1
1878 ldr\t%<SHORT:size>0, %1"
0cfc095c
WD		 1879 [(set_attr "type" "logic_imm,load_4,f_loads")
1880 (set_attr "arch" "*,*,fp")]
43e9d192
IB		 1881)
1882
1883(define_expand "<optab>qihi2"
1884 [(set (match_operand:HI 0 "register_operand")
1885 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1886 ""
1887)
1888
63715e5e 1889(define_insn "*extendqihi2_aarch64"
43e9d192 1890 [(set (match_operand:HI 0 "register_operand" "=r,r")
63715e5e 1891 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
43e9d192
IB		 1892 ""
1893 "@
63715e5e
WD		 1894 sxtb\t%w0, %w1
1895 ldrsb\t%w0, %1"
89b2133e 1896 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1897)
1898
63715e5e
WD		 1899(define_insn "*zero_extendqihi2_aarch64"
1900 [(set (match_operand:HI 0 "register_operand" "=r,r")
1901 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1902 ""
1903 "@
1904 and\t%w0, %w1, 255
1905 ldrb\t%w0, %1"
89b2133e 1906 [(set_attr "type" "logic_imm,load_4")]
63715e5e
WD		 1907)
1908
43e9d192
IB		 1909;; -------------------------------------------------------------------
1910;; Simple arithmetic
1911;; -------------------------------------------------------------------
1912
1913(define_expand "add<mode>3"
1914 [(set
1bbffb87
DZ		 1915 (match_operand:GPI 0 "register_operand")
1916 (plus:GPI (match_operand:GPI 1 "register_operand")
1917 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
43e9d192 1918 ""
89f0a62e 1919{
35323bd8
KT		 1920 /* If operands[1] is a subreg extract the inner RTX. */
1921 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1922
45a6c1e3
WD		 1923 /* If the constant is too large for a single instruction and isn't frame
1924 based, split off the immediate so it is available for CSE. */
1925 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
0fdc30bc 1926 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
45a6c1e3 1927 && can_create_pseudo_p ()
35323bd8
KT		 1928 && (!REG_P (op1)
1929 || !REGNO_PTR_FRAME_P (REGNO (op1))))
45a6c1e3 1930 operands[2] = force_reg (<MODE>mode, operands[2]);
43cacb12
RS		 1931 /* Expand polynomial additions now if the destination is the stack
1932 pointer, since we don't want to use that as a temporary. */
1933 else if (operands[0] == stack_pointer_rtx
1934 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1935 {
1936 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1937 operands[2], NULL_RTX, NULL_RTX);
1938 DONE;
1939 }
89f0a62e 1940})
43e9d192 1941
89f0a62e 1942(define_insn "*add<mode>3_aarch64"
43e9d192 1943 [(set
0fdc30bc 1944 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
89f0a62e 1945 (plus:GPI
0fdc30bc
RS		 1946 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1947 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
43e9d192
IB		 1948 ""
1949 "@
89f0a62e
RH		 1950 add\\t%<w>0, %<w>1, %2
1951 add\\t%<w>0, %<w>1, %<w>2
1952 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1953 sub\\t%<w>0, %<w>1, #%n2
43cacb12 1954 #
0fdc30bc
RS		 1955 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1956 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1957 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1958 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1959 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
43e9d192
IB		 1960)
1961
3b585938
IB		 1962;; zero_extend version of above
1963(define_insn "*addsi3_aarch64_uxtw"
1964 [(set
89f0a62e 1965 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
3b585938 1966 (zero_extend:DI
89f0a62e 1967 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
ff1335e2 1968 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
3b585938
IB		 1969 ""
1970 "@
1971 add\\t%w0, %w1, %2
1972 add\\t%w0, %w1, %w2
89f0a62e
RH		 1973 sub\\t%w0, %w1, #%n2
1974 #"
1975 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1976)
1977
1978;; If there's a free register, and we can load the constant with a
1979;; single instruction, do so. This has a chance to improve scheduling.
1980(define_peephole2
1981 [(match_scratch:GPI 3 "r")
1982 (set (match_operand:GPI 0 "register_operand")
1983 (plus:GPI
1984 (match_operand:GPI 1 "register_operand")
1985 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1986 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1987 [(set (match_dup 3) (match_dup 2))
1988 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1989)
1990
1991(define_peephole2
1992 [(match_scratch:SI 3 "r")
1993 (set (match_operand:DI 0 "register_operand")
1994 (zero_extend:DI
1995 (plus:SI
1996 (match_operand:SI 1 "register_operand")
1997 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1998 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1999 [(set (match_dup 3) (match_dup 2))
2000 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
2001)
2002
2003;; After peephole2 has had a chance to run, split any remaining long
2004;; additions into two add immediates.
2005(define_split
2006 [(set (match_operand:GPI 0 "register_operand")
2007 (plus:GPI
2008 (match_operand:GPI 1 "register_operand")
2009 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
2010 "epilogue_completed"
2011 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
2012 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
2013 {
2014 HOST_WIDE_INT i = INTVAL (operands[2]);
2015 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2016 operands[3] = GEN_INT (i - s);
2017 operands[4] = GEN_INT (s);
2018 }
3b585938
IB		 2019)
2020
43cacb12
RS		 2021;; Match addition of polynomial offsets that require one temporary, for which
2022;; we can use the early-clobbered destination register. This is a separate
2023;; pattern so that the early clobber doesn't affect register allocation
2024;; for other forms of addition. However, we still need to provide an
2025;; all-register alternative, in case the offset goes out of range after
2026;; elimination. For completeness we might as well provide all GPR-based
2027;; alternatives from the main pattern.
2028;;
2029;; We don't have a pattern for additions requiring two temporaries since at
2030;; present LRA doesn't allow new scratches to be added during elimination.
2031;; Such offsets should be rare anyway.
2032;;
2033;; ??? But if we added LRA support for new scratches, much of the ugliness
2034;; here would go away. We could just handle all polynomial constants in
2035;; this pattern.
2036(define_insn_and_split "*add<mode>3_poly_1"
2037 [(set
0fdc30bc 2038 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
43cacb12 2039 (plus:GPI
0fdc30bc
RS		 2040 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
2041 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
43cacb12
RS		 2042 "TARGET_SVE && operands[0] != stack_pointer_rtx"
2043 "@
2044 add\\t%<w>0, %<w>1, %2
2045 add\\t%<w>0, %<w>1, %<w>2
2046 sub\\t%<w>0, %<w>1, #%n2
2047 #
0fdc30bc
RS		 2048 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
2049 * return aarch64_output_sve_addvl_addpl (operands[2]);
43cacb12
RS		 2050 #"
2051 "&& epilogue_completed
2052 && !reg_overlap_mentioned_p (operands[0], operands[1])
2053 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
2054 [(const_int 0)]
2055 {
2056 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2057 operands[2], operands[0], NULL_RTX);
2058 DONE;
2059 }
0fdc30bc
RS		 2060 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2061 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
43cacb12
RS		 2062)
2063
89f0a62e
RH		 2064(define_split
2065 [(set (match_operand:DI 0 "register_operand")
2066 (zero_extend:DI
2067 (plus:SI
2068 (match_operand:SI 1 "register_operand")
2069 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2070 "epilogue_completed"
2071 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
2072 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
2073 {
2074 HOST_WIDE_INT i = INTVAL (operands[2]);
2075 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2076 operands[3] = GEN_INT (i - s);
2077 operands[4] = GEN_INT (s);
2078 operands[5] = gen_lowpart (SImode, operands[0]);
2079 }
43e9d192
IB		 2080)
2081
30c46053
MC		 2082(define_expand "addv<mode>4"
2083 [(match_operand:GPI 0 "register_operand")
2084 (match_operand:GPI 1 "register_operand")
a58fe3c5 2085 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 2086 (label_ref (match_operand 3 "" ""))]
2087 ""
2088{
a58fe3c5
RE		 2089 if (CONST_INT_P (operands[2]))
2090 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
2091 operands[2]));
2092 else
2093 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
30c46053
MC		 2094 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2095
2096 DONE;
2097})
2098
2099(define_expand "uaddv<mode>4"
2100 [(match_operand:GPI 0 "register_operand")
2101 (match_operand:GPI 1 "register_operand")
2102 (match_operand:GPI 2 "register_operand")
2103 (label_ref (match_operand 3 "" ""))]
2104 ""
2105{
2106 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
f7343f20 2107 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
30c46053
MC		 2108
2109 DONE;
2110})
2111
2195867f 2112(define_expand "addti3"
1bbffb87
DZ		 2113 [(set (match_operand:TI 0 "register_operand")
2114 (plus:TI (match_operand:TI 1 "register_operand")
2115 (match_operand:TI 2 "aarch64_reg_or_imm")))]
30c46053
MC		 2116 ""
2117{
2118 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2119
2120 aarch64_addti_scratch_regs (operands[1], operands[2],
2121 &low_dest, &op1_low, &op2_low,
2122 &high_dest, &op1_high, &op2_high);
2123
2124 if (op2_low == const0_rtx)
2125 {
2126 low_dest = op1_low;
2127 if (!aarch64_pluslong_operand (op2_high, DImode))
2128 op2_high = force_reg (DImode, op2_high);
2129 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2130 }
2131 else
2132 {
2133 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2134 force_reg (DImode, op2_low)));
2135 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2136 force_reg (DImode, op2_high)));
2137 }
2138
2139 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2140 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2141
2142 DONE;
2143})
2144
2145(define_expand "addvti4"
1bbffb87
DZ		 2146 [(match_operand:TI 0 "register_operand")
2147 (match_operand:TI 1 "register_operand")
2148 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053 2149 (label_ref (match_operand 3 "" ""))]
2195867f
RH		 2150 ""
2151{
30c46053 2152 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2195867f 2153
30c46053
MC		 2154 aarch64_addti_scratch_regs (operands[1], operands[2],
2155 &low_dest, &op1_low, &op2_low,
2156 &high_dest, &op1_high, &op2_high);
2195867f 2157
30c46053
MC		 2158 if (op2_low == const0_rtx)
2159 {
2160 low_dest = op1_low;
2161 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2162 force_reg (DImode, op2_high)));
2163 }
2164 else
2165 {
2166 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2167 force_reg (DImode, op2_low)));
2168 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2169 force_reg (DImode, op2_high)));
2170 }
2171
2172 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2173 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2174
2175 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2195867f
RH		 2176 DONE;
2177})
2178
30c46053 2179(define_expand "uaddvti4"
1bbffb87
DZ		 2180 [(match_operand:TI 0 "register_operand")
2181 (match_operand:TI 1 "register_operand")
2182 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053
MC		 2183 (label_ref (match_operand 3 "" ""))]
2184 ""
2185{
2186 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2187
2188 aarch64_addti_scratch_regs (operands[1], operands[2],
2189 &low_dest, &op1_low, &op2_low,
2190 &high_dest, &op1_high, &op2_high);
2191
2192 if (op2_low == const0_rtx)
2193 {
2194 low_dest = op1_low;
2195 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2196 force_reg (DImode, op2_high)));
2197 }
2198 else
2199 {
2200 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2201 force_reg (DImode, op2_low)));
2202 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2203 force_reg (DImode, op2_high)));
2204 }
2205
2206 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2207 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2208
f7343f20 2209 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
30c46053
MC		 2210 DONE;
2211 })
2212
2195867f 2213(define_insn "add<mode>3_compare0"
43e9d192
IB		 2214 [(set (reg:CC_NZ CC_REGNUM)
2215 (compare:CC_NZ
341fca1b 2216 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
34b512f5 2217 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
43e9d192 2218 (const_int 0)))
34b512f5 2219 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
43e9d192
IB		 2220 (plus:GPI (match_dup 1) (match_dup 2)))]
2221 ""
2222 "@
2223 adds\\t%<w>0, %<w>1, %<w>2
e2b691c4 2224 adds\\t%<w>0, %<w>1, %2
43e9d192 2225 subs\\t%<w>0, %<w>1, #%n2"
1d61feeb 2226 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2227)
2228
3b585938
IB		 2229;; zero_extend version of above
2230(define_insn "*addsi3_compare0_uxtw"
2231 [(set (reg:CC_NZ CC_REGNUM)
2232 (compare:CC_NZ
341fca1b 2233 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
34b512f5 2234 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
3b585938 2235 (const_int 0)))
34b512f5 2236 (set (match_operand:DI 0 "register_operand" "=r,r,r")
3b585938
IB		 2237 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2238 ""
2239 "@
2240 adds\\t%w0, %w1, %w2
e2b691c4 2241 adds\\t%w0, %w1, %2
3b585938 2242 subs\\t%w0, %w1, #%n2"
1d61feeb 2243 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3b585938
IB		 2244)
2245
ef22810a
RH		 2246(define_insn "*add<mode>3_compareC_cconly"
2247 [(set (reg:CC_C CC_REGNUM)
f7343f20
RE		 2248 (compare:CC_C
2249 (plus:GPI
2250 (match_operand:GPI 0 "register_operand" "r,r,r")
2251 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2252 (match_dup 0)))]
ef22810a 2253 ""
ef22810a 2254 "@
f7343f20
RE		 2255 cmn\\t%<w>0, %<w>1
2256 cmn\\t%<w>0, %1
2257 cmp\\t%<w>0, #%n1"
2258 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a 2259)
954224d7 2260
ef22810a
RH		 2261(define_insn "add<mode>3_compareC"
2262 [(set (reg:CC_C CC_REGNUM)
30c46053 2263 (compare:CC_C
f7343f20 2264 (plus:GPI
341fca1b 2265 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
f7343f20
RE		 2266 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2267 (match_dup 1)))
2268 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
ef22810a
RH		 2269 (plus:GPI (match_dup 1) (match_dup 2)))]
2270 ""
f7343f20
RE		 2271 "@
2272 adds\\t%<w>0, %<w>1, %<w>2
2273 adds\\t%<w>0, %<w>1, %2
2274 subs\\t%<w>0, %<w>1, #%n2"
2275 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a
RH		 2276)
2277
30c46053
MC		 2278(define_insn "*add<mode>3_compareV_cconly_imm"
2279 [(set (reg:CC_V CC_REGNUM)
2280 (compare:CC_V
2281 (plus:<DWI>
2282 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2283 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2284 (sign_extend:<DWI>
2285 (plus:GPI
2286 (match_dup 0)
2287 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2288 "INTVAL (operands[1]) == INTVAL (operands[2])"
2289 "@
2290 cmn\\t%<w>0, %<w>1
2291 cmp\\t%<w>0, #%n1"
2292 [(set_attr "type" "alus_imm")]
2293)
2294
2295(define_insn "*add<mode>3_compareV_cconly"
2296 [(set (reg:CC_V CC_REGNUM)
2297 (compare:CC_V
2298 (plus:<DWI>
2299 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2300 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2301 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2302 ""
2303 "cmn\\t%<w>0, %<w>1"
2304 [(set_attr "type" "alus_sreg")]
2305)
2306
a58fe3c5 2307(define_insn "add<mode>3_compareV_imm"
30c46053
MC		 2308 [(set (reg:CC_V CC_REGNUM)
2309 (compare:CC_V
2310 (plus:<DWI>
2311 (sign_extend:<DWI>
341fca1b 2312 (match_operand:GPI 1 "register_operand" "rk,rk"))
30c46053
MC		 2313 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2314 (sign_extend:<DWI>
2315 (plus:GPI (match_dup 1) (match_dup 2)))))
2316 (set (match_operand:GPI 0 "register_operand" "=r,r")
2317 (plus:GPI (match_dup 1) (match_dup 2)))]
2318 ""
2319 "@
2320 adds\\t%<w>0, %<w>1, %<w>2
2321 subs\\t%<w>0, %<w>1, #%n2"
2322 [(set_attr "type" "alus_imm,alus_imm")]
2323)
2324
2325(define_insn "add<mode>3_compareV"
2326 [(set (reg:CC_V CC_REGNUM)
2327 (compare:CC_V
2328 (plus:<DWI>
341fca1b 2329 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
30c46053
MC		 2330 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2331 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2332 (set (match_operand:GPI 0 "register_operand" "=r")
2333 (plus:GPI (match_dup 1) (match_dup 2)))]
2334 ""
2335 "adds\\t%<w>0, %<w>1, %<w>2"
2336 [(set_attr "type" "alus_sreg")]
2337)
2338
dd1bdce4
VK		 2339(define_insn "*adds_shift_imm_<mode>"
2340 [(set (reg:CC_NZ CC_REGNUM)
2341 (compare:CC_NZ
30c46053 2342 (plus:GPI (ASHIFT:GPI
dd1bdce4
VK		 2343 (match_operand:GPI 1 "register_operand" "r")
2344 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2345 (match_operand:GPI 3 "register_operand" "r"))
2346 (const_int 0)))
2347 (set (match_operand:GPI 0 "register_operand" "=r")
2348 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2349 (match_dup 3)))]
2350 ""
2351 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2352 [(set_attr "type" "alus_shift_imm")]
2353)
2354
2355(define_insn "*subs_shift_imm_<mode>"
2356 [(set (reg:CC_NZ CC_REGNUM)
2357 (compare:CC_NZ
2358 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2359 (ASHIFT:GPI
2360 (match_operand:GPI 2 "register_operand" "r")
2361 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2362 (const_int 0)))
2363 (set (match_operand:GPI 0 "register_operand" "=r")
2364 (minus:GPI (match_dup 1)
2365 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2366 ""
2367 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2368 [(set_attr "type" "alus_shift_imm")]
2369)
2370
f3d096b0
N		 2371(define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2372 [(set (reg:CC_NZ CC_REGNUM)
2373 (compare:CC_NZ
2374 (plus:GPI
2375 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
341fca1b 2376 (match_operand:GPI 2 "register_operand" "rk"))
f3d096b0
N		 2377 (const_int 0)))
2378 (set (match_operand:GPI 0 "register_operand" "=r")
2379 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2380 ""
d4febc75 2381 "adds\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
99e6af15 2382 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2383)
2384
2385(define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2386 [(set (reg:CC_NZ CC_REGNUM)
2387 (compare:CC_NZ
341fca1b 2388 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
f3d096b0
N		 2389 (ANY_EXTEND:GPI
2390 (match_operand:ALLX 2 "register_operand" "r")))
2391 (const_int 0)))
2392 (set (match_operand:GPI 0 "register_operand" "=r")
2393 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2394 ""
d4febc75 2395 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
99e6af15 2396 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2397)
2398
dd1bdce4
VK		 2399(define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2400 [(set (reg:CC_NZ CC_REGNUM)
2401 (compare:CC_NZ
2402 (plus:GPI (ashift:GPI
2403 (ANY_EXTEND:GPI
2404 (match_operand:ALLX 1 "register_operand" "r"))
2405 (match_operand 2 "aarch64_imm3" "Ui3"))
341fca1b 2406 (match_operand:GPI 3 "register_operand" "rk"))
dd1bdce4
VK		 2407 (const_int 0)))
2408 (set (match_operand:GPI 0 "register_operand" "=rk")
2409 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2410 (match_dup 2))
2411 (match_dup 3)))]
2412 ""
d4febc75 2413 "adds\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
dd1bdce4
VK		 2414 [(set_attr "type" "alus_ext")]
2415)
2416
2417(define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2418 [(set (reg:CC_NZ CC_REGNUM)
2419 (compare:CC_NZ
341fca1b 2420 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
dd1bdce4
VK		 2421 (ashift:GPI
2422 (ANY_EXTEND:GPI
2423 (match_operand:ALLX 2 "register_operand" "r"))
2424 (match_operand 3 "aarch64_imm3" "Ui3")))
2425 (const_int 0)))
2426 (set (match_operand:GPI 0 "register_operand" "=rk")
2427 (minus:GPI (match_dup 1)
2428 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2429 (match_dup 3))))]
2430 ""
d4febc75 2431 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
dd1bdce4
VK		 2432 [(set_attr "type" "alus_ext")]
2433)
2434
43e9d192
IB		 2435(define_insn "*add<mode>3nr_compare0"
2436 [(set (reg:CC_NZ CC_REGNUM)
2437 (compare:CC_NZ
34b512f5
SN		 2438 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2439 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
43e9d192
IB		 2440 (const_int 0)))]
2441 ""
2442 "@
2443 cmn\\t%<w>0, %<w>1
e2b691c4 2444 cmn\\t%<w>0, %1
43e9d192 2445 cmp\\t%<w>0, #%n1"
1d61feeb 2446 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2447)
2448
2f65ab2e
MC		 2449(define_insn "aarch64_sub<mode>_compare0"
2450 [(set (reg:CC_NZ CC_REGNUM)
2451 (compare:CC_NZ
2452 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2453 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2454 (const_int 0)))]
2455 ""
2456 "cmp\\t%<w>0, %<w>1"
2457 [(set_attr "type" "alus_sreg")]
2458)
2459
f5313c63 2460(define_insn "*compare_neg<mode>"
1c992d1e
RE		 2461 [(set (reg:CC_Z CC_REGNUM)
2462 (compare:CC_Z
274b2532
KT		 2463 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2464 (match_operand:GPI 1 "register_operand" "r")))]
f5313c63 2465 ""
274b2532 2466 "cmn\\t%<w>1, %<w>0"
1d61feeb 2467 [(set_attr "type" "alus_sreg")]
f5313c63
IB		 2468)
2469
43e9d192 2470(define_insn "*add_<shift>_<mode>"
24c56925 2471 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 2472 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2473 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2474 (match_operand:GPI 3 "register_operand" "r")))]
2475 ""
2476 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
ae27ce51 2477 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 2478)
2479
3b585938
IB		 2480;; zero_extend version of above
2481(define_insn "*add_<shift>_si_uxtw"
24c56925 2482 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 2483 (zero_extend:DI
2484 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2a9704d0
MS		 2485 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2486 (match_operand:SI 3 "register_operand" "r"))))]
3b585938
IB		 2487 ""
2488 "add\\t%w0, %w3, %w1, <shift> %2"
ae27ce51 2489 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 2490)
2491
43e9d192
IB		 2492(define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2493 [(set (match_operand:GPI 0 "register_operand" "=rk")
2494 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2495 (match_operand:GPI 2 "register_operand" "r")))]
2496 ""
d4febc75 2497 "add\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
99e6af15 2498 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2499)
2500
3b585938
IB		 2501;; zero_extend version of above
2502(define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2503 [(set (match_operand:DI 0 "register_operand" "=rk")
2504 (zero_extend:DI
2505 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2506 (match_operand:GPI 2 "register_operand" "r"))))]
2507 ""
2508 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
99e6af15 2509 [(set_attr "type" "alu_ext")]
3b585938
IB		 2510)
2511
43e9d192
IB		 2512(define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2513 [(set (match_operand:GPI 0 "register_operand" "=rk")
2514 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2515 (match_operand:ALLX 1 "register_operand" "r"))
2516 (match_operand 2 "aarch64_imm3" "Ui3"))
2517 (match_operand:GPI 3 "register_operand" "r")))]
2518 ""
d4febc75 2519 "add\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
99e6af15 2520 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2521)
2522
3b585938
IB		 2523;; zero_extend version of above
2524(define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2525 [(set (match_operand:DI 0 "register_operand" "=rk")
2526 (zero_extend:DI
2527 (plus:SI (ashift:SI (ANY_EXTEND:SI
2528 (match_operand:SHORT 1 "register_operand" "r"))
2529 (match_operand 2 "aarch64_imm3" "Ui3"))
2530 (match_operand:SI 3 "register_operand" "r"))))]
2531 ""
2532 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
99e6af15 2533 [(set_attr "type" "alu_ext")]
3b585938
IB		 2534)
2535
ef22810a
RH		 2536(define_expand "add<mode>3_carryin"
2537 [(set (match_operand:GPI 0 "register_operand")
2538 (plus:GPI
2539 (plus:GPI
f7343f20 2540 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
ef22810a
RH		 2541 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2542 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
3b585938 2543 ""
43e9d192 2544 ""
43e9d192
IB		 2545)
2546
ef22810a
RH		 2547;; Note that add with carry with two zero inputs is matched by cset,
2548;; and that add with carry with one zero input is matched by cinc.
3b585938 2549
ef22810a
RH		 2550(define_insn "*add<mode>3_carryin"
2551 [(set (match_operand:GPI 0 "register_operand" "=r")
2552 (plus:GPI
2553 (plus:GPI
2554 (match_operand:GPI 3 "aarch64_carry_operation" "")
2555 (match_operand:GPI 1 "register_operand" "r"))
2556 (match_operand:GPI 2 "register_operand" "r")))]
43e9d192
IB		 2557 ""
2558 "adc\\t%<w>0, %<w>1, %<w>2"
99e6af15 2559 [(set_attr "type" "adc_reg")]
43e9d192
IB		 2560)
2561
3b585938 2562;; zero_extend version of above
ef22810a
RH		 2563(define_insn "*addsi3_carryin_uxtw"
2564 [(set (match_operand:DI 0 "register_operand" "=r")
2565 (zero_extend:DI
2566 (plus:SI
2567 (plus:SI
2568 (match_operand:SI 3 "aarch64_carry_operation" "")
2569 (match_operand:SI 1 "register_operand" "r"))
2570 (match_operand:SI 2 "register_operand" "r"))))]
3b585938
IB		 2571 ""
2572 "adc\\t%w0, %w1, %w2"
99e6af15 2573 [(set_attr "type" "adc_reg")]
3b585938
IB		 2574)
2575
30c46053
MC		 2576(define_expand "add<mode>3_carryinC"
2577 [(parallel
2578 [(set (match_dup 3)
f7343f20 2579 (compare:CC_ADC
30c46053
MC		 2580 (plus:<DWI>
2581 (plus:<DWI>
2582 (match_dup 4)
2583 (zero_extend:<DWI>
1bbffb87 2584 (match_operand:GPI 1 "register_operand")))
30c46053 2585 (zero_extend:<DWI>
1bbffb87 2586 (match_operand:GPI 2 "register_operand")))
f7343f20 2587 (match_dup 6)))
30c46053
MC		 2588 (set (match_operand:GPI 0 "register_operand")
2589 (plus:GPI
2590 (plus:GPI (match_dup 5) (match_dup 1))
2591 (match_dup 2)))])]
2592 ""
2593{
f7343f20
RE		 2594 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2595 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2596 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2597 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2598 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2599 << GET_MODE_BITSIZE (<MODE>mode),
2600 TImode);
30c46053
MC		 2601})
2602
2603(define_insn "*add<mode>3_carryinC_zero"
f7343f20
RE		 2604 [(set (reg:CC_ADC CC_REGNUM)
2605 (compare:CC_ADC
30c46053
MC		 2606 (plus:<DWI>
2607 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2608 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
f7343f20 2609 (match_operand 4 "const_scalar_int_operand" "")))
16621f0d 2610 (set (match_operand:GPI 0 "register_operand" "=r")
f7343f20
RE		 2611 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2612 (match_dup 1)))]
2613 "rtx_mode_t (operands[4], <DWI>mode)
2614 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2615 "adcs\\t%<w>0, %<w>1, <w>zr"
2616 [(set_attr "type" "adc_reg")]
2617)
2618
2619(define_insn "*add<mode>3_carryinC"
f7343f20
RE		 2620 [(set (reg:CC_ADC CC_REGNUM)
2621 (compare:CC_ADC
30c46053
MC		 2622 (plus:<DWI>
2623 (plus:<DWI>
2624 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2625 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2626 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
f7343f20 2627 (match_operand 5 "const_scalar_int_operand" "")))
16621f0d 2628 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053 2629 (plus:GPI
f7343f20
RE		 2630 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2631 (match_dup 1))
30c46053 2632 (match_dup 2)))]
f7343f20
RE		 2633 "rtx_mode_t (operands[5], <DWI>mode)
2634 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2635 "adcs\\t%<w>0, %<w>1, %<w>2"
2636 [(set_attr "type" "adc_reg")]
2637)
2638
2639(define_expand "add<mode>3_carryinV"
2640 [(parallel
2641 [(set (reg:CC_V CC_REGNUM)
2642 (compare:CC_V
2643 (plus:<DWI>
2644 (plus:<DWI>
2645 (match_dup 3)
2646 (sign_extend:<DWI>
1bbffb87 2647 (match_operand:GPI 1 "register_operand")))
30c46053 2648 (sign_extend:<DWI>
1bbffb87 2649 (match_operand:GPI 2 "register_operand")))
30c46053
MC		 2650 (sign_extend:<DWI>
2651 (plus:GPI
2652 (plus:GPI (match_dup 4) (match_dup 1))
2653 (match_dup 2)))))
2654 (set (match_operand:GPI 0 "register_operand")
2655 (plus:GPI
2656 (plus:GPI (match_dup 4) (match_dup 1))
2657 (match_dup 2)))])]
2658 ""
2659{
2660 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
f7343f20
RE		 2661 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2662 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
30c46053
MC		 2663})
2664
2665(define_insn "*add<mode>3_carryinV_zero"
2666 [(set (reg:CC_V CC_REGNUM)
2667 (compare:CC_V
2668 (plus:<DWI>
2669 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2670 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2671 (sign_extend:<DWI>
2672 (plus:GPI
2673 (match_operand:GPI 3 "aarch64_carry_operation" "")
2674 (match_dup 1)))))
16621f0d 2675 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2676 (plus:GPI (match_dup 3) (match_dup 1)))]
2677 ""
2678 "adcs\\t%<w>0, %<w>1, <w>zr"
2679 [(set_attr "type" "adc_reg")]
2680)
2681
2682(define_insn "*add<mode>3_carryinV"
2683 [(set (reg:CC_V CC_REGNUM)
2684 (compare:CC_V
2685 (plus:<DWI>
2686 (plus:<DWI>
2687 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2688 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2689 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2690 (sign_extend:<DWI>
2691 (plus:GPI
2692 (plus:GPI
2693 (match_operand:GPI 4 "aarch64_carry_operation" "")
2694 (match_dup 1))
2695 (match_dup 2)))))
16621f0d 2696 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2697 (plus:GPI
2698 (plus:GPI (match_dup 4) (match_dup 1))
2699 (match_dup 2)))]
2700 ""
2701 "adcs\\t%<w>0, %<w>1, %<w>2"
2702 [(set_attr "type" "adc_reg")]
2703)
2704
dd1bdce4
VK		 2705(define_insn "*add_uxt<mode>_shift2"
2706 [(set (match_operand:GPI 0 "register_operand" "=rk")
2707 (plus:GPI (and:GPI
2708 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2709 (match_operand 2 "aarch64_imm3" "Ui3"))
2710 (match_operand 3 "const_int_operand" "n"))
2711 (match_operand:GPI 4 "register_operand" "r")))]
2712 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2713 "*
2714 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2715 INTVAL (operands[3])));
d4febc75 2716 return \"add\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
dd1bdce4
VK		 2717 [(set_attr "type" "alu_ext")]
2718)
2719
2720;; zero_extend version of above
2721(define_insn "*add_uxtsi_shift2_uxtw"
2722 [(set (match_operand:DI 0 "register_operand" "=rk")
2723 (zero_extend:DI
2724 (plus:SI (and:SI
2725 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2726 (match_operand 2 "aarch64_imm3" "Ui3"))
2727 (match_operand 3 "const_int_operand" "n"))
2728 (match_operand:SI 4 "register_operand" "r"))))]
2729 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2730 "*
2731 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2732 INTVAL (operands[3])));
2733 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2734 [(set_attr "type" "alu_ext")]
2735)
2736
43e9d192
IB		 2737(define_insn "subsi3"
2738 [(set (match_operand:SI 0 "register_operand" "=rk")
7eb8fb77
RS		 2739 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2740 (match_operand:SI 2 "register_operand" "r")))]
43e9d192
IB		 2741 ""
2742 "sub\\t%w0, %w1, %w2"
1d61feeb 2743 [(set_attr "type" "alu_sreg")]
43e9d192
IB		 2744)
2745
3b585938
IB		 2746;; zero_extend version of above
2747(define_insn "*subsi3_uxtw"
2748 [(set (match_operand:DI 0 "register_operand" "=rk")
2749 (zero_extend:DI
7eb8fb77 2750 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 2751 (match_operand:SI 2 "register_operand" "r"))))]
2752 ""
2753 "sub\\t%w0, %w1, %w2"
1d61feeb 2754 [(set_attr "type" "alu_sreg")]
3b585938
IB		 2755)
2756
43e9d192 2757(define_insn "subdi3"
4f2962fd 2758 [(set (match_operand:DI 0 "register_operand" "=rk,w")
7eb8fb77
RS		 2759 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2760 (match_operand:DI 2 "register_operand" "r,w")))]
43e9d192
IB		 2761 ""
2762 "@
2763 sub\\t%x0, %x1, %x2
2764 sub\\t%d0, %d1, %d2"
1d61feeb 2765 [(set_attr "type" "alu_sreg, neon_sub")
488461d8 2766 (set_attr "arch" "*,simd")]
43e9d192
IB		 2767)
2768
a58fe3c5 2769(define_expand "subv<GPI:mode>4"
30c46053 2770 [(match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 2771 (match_operand:GPI 1 "register_operand")
2772 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 2773 (label_ref (match_operand 3 "" ""))]
2774 ""
2775{
a58fe3c5
RE		 2776 if (CONST_INT_P (operands[2]))
2777 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2778 else
2779 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
30c46053
MC		 2780 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2781
2782 DONE;
2783})
2784
a58fe3c5
RE		 2785(define_insn "subv<GPI:mode>_insn"
2786 [(set (reg:CC_V CC_REGNUM)
2787 (compare:CC_V
2788 (sign_extend:<DWI>
2789 (minus:GPI
2790 (match_operand:GPI 1 "register_operand" "rk")
2791 (match_operand:GPI 2 "register_operand" "r")))
2792 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2793 (sign_extend:<DWI> (match_dup 2)))))
2794 (set (match_operand:GPI 0 "register_operand" "=r")
2795 (minus:GPI (match_dup 1) (match_dup 2)))]
2796 ""
2797 "subs\\t%<w>0, %<w>1, %<w>2"
2798 [(set_attr "type" "alus_sreg")]
2799)
2800
2801(define_insn "subv<GPI:mode>_imm"
2802 [(set (reg:CC_V CC_REGNUM)
2803 (compare:CC_V
2804 (sign_extend:<DWI>
2805 (minus:GPI
2806 (match_operand:GPI 1 "register_operand" "rk,rk")
2807 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2808 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2809 (match_dup 2))))
2810 (set (match_operand:GPI 0 "register_operand" "=r,r")
2811 (minus:GPI (match_dup 1) (match_dup 2)))]
2812 ""
2813 "@
2814 subs\\t%<w>0, %<w>1, %2
2815 adds\\t%<w>0, %<w>1, #%n2"
2816 [(set_attr "type" "alus_sreg")]
2817)
2818
2819(define_expand "negv<GPI:mode>3"
2820 [(match_operand:GPI 0 "register_operand")
2821 (match_operand:GPI 1 "register_operand")
2822 (label_ref (match_operand 2 "" ""))]
2823 ""
2824 {
2825 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2826 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2827
2828 DONE;
2829 }
2830)
2831
2832(define_insn "negv<GPI:mode>_insn"
2833 [(set (reg:CC_V CC_REGNUM)
2834 (compare:CC_V
2835 (sign_extend:<DWI>
2836 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2837 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2838 (set (match_operand:GPI 0 "register_operand" "=r")
2839 (neg:GPI (match_dup 1)))]
2840 ""
2841 "negs\\t%<w>0, %<w>1"
2842 [(set_attr "type" "alus_sreg")]
2843)
2844
2845(define_insn "negv<GPI:mode>_cmp_only"
2846 [(set (reg:CC_V CC_REGNUM)
2847 (compare:CC_V
2848 (sign_extend:<DWI>
2849 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2850 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2851 ""
2852 "negs\\t%<w>zr, %<w>0"
2853 [(set_attr "type" "alus_sreg")]
2854)
2855
2856(define_insn "*cmpv<GPI:mode>_insn"
2857 [(set (reg:CC_V CC_REGNUM)
2858 (compare:CC_V
2859 (sign_extend:<DWI>
2860 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2861 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2862 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2863 (sign_extend:<DWI> (match_dup 1)))))]
2864 ""
2865 "@
2866 cmp\\t%<w>0, %<w>1
2867 cmp\\t%<w>0, %1
2868 cmp\\t%<w>0, #%n1"
2869 [(set_attr "type" "alus_sreg")]
2870)
2871
30c46053
MC		 2872(define_expand "usubv<mode>4"
2873 [(match_operand:GPI 0 "register_operand")
2874 (match_operand:GPI 1 "aarch64_reg_or_zero")
2875 (match_operand:GPI 2 "aarch64_reg_or_zero")
2876 (label_ref (match_operand 3 "" ""))]
2877 ""
2878{
2879 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2880 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2881
2882 DONE;
2883})
2884
2195867f 2885(define_expand "subti3"
1bbffb87
DZ		 2886 [(set (match_operand:TI 0 "register_operand")
2887 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2888 (match_operand:TI 2 "register_operand")))]
2195867f
RH		 2889 ""
2890{
30c46053
MC		 2891 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2892
2893 aarch64_subvti_scratch_regs (operands[1], operands[2],
2894 &low_dest, &op1_low, &op2_low,
2895 &high_dest, &op1_high, &op2_high);
2195867f 2896
30c46053
MC		 2897 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2898 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2195867f 2899
30c46053
MC		 2900 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2901 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2902 DONE;
2903})
2904
2905(define_expand "subvti4"
2906 [(match_operand:TI 0 "register_operand")
a58fe3c5 2907 (match_operand:TI 1 "register_operand")
30c46053
MC		 2908 (match_operand:TI 2 "aarch64_reg_or_imm")
2909 (label_ref (match_operand 3 "" ""))]
2910 ""
2911{
2912 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2913
2914 aarch64_subvti_scratch_regs (operands[1], operands[2],
2915 &low_dest, &op1_low, &op2_low,
2916 &high_dest, &op1_high, &op2_high);
2917 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 2918 high_dest, op1_high, op2_high, false);
30c46053
MC		 2919
2920 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2921 DONE;
2922})
2923
2924(define_expand "usubvti4"
2925 [(match_operand:TI 0 "register_operand")
a58fe3c5 2926 (match_operand:TI 1 "register_operand")
30c46053
MC		 2927 (match_operand:TI 2 "aarch64_reg_or_imm")
2928 (label_ref (match_operand 3 "" ""))]
2929 ""
2930{
2931 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2932
2933 aarch64_subvti_scratch_regs (operands[1], operands[2],
2934 &low_dest, &op1_low, &op2_low,
2935 &high_dest, &op1_high, &op2_high);
2936 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 2937 high_dest, op1_high, op2_high, true);
30c46053
MC		 2938
2939 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2195867f
RH		 2940 DONE;
2941})
43e9d192 2942
a58fe3c5
RE		 2943(define_expand "negvti3"
2944 [(match_operand:TI 0 "register_operand")
2945 (match_operand:TI 1 "register_operand")
2946 (label_ref (match_operand 2 "" ""))]
2947 ""
2948 {
2949 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2950 gen_lowpart (DImode, operands[1])));
2951 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2952 gen_highpart (DImode, operands[1])));
2953 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2954
2955 DONE;
2956 }
2957)
2958
2959(define_insn "negdi_carryout"
2960 [(set (reg:CC CC_REGNUM)
2961 (compare:CC
2962 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2963 (set (match_operand:DI 0 "register_operand" "=r")
2964 (neg:DI (match_dup 1)))]
2965 ""
2966 "negs\\t%0, %1"
2967 [(set_attr "type" "alus_sreg")]
2968)
2969
2970(define_insn "negvdi_carryinV"
2971 [(set (reg:CC_V CC_REGNUM)
2972 (compare:CC_V
2973 (neg:TI (plus:TI
2974 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2975 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2976 (sign_extend:TI
2977 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2978 (match_dup 1))))))
2979 (set (match_operand:DI 0 "register_operand" "=r")
2980 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2981 (match_dup 1))))]
2982 ""
2983 "ngcs\\t%0, %1"
2984 [(set_attr "type" "alus_sreg")]
2985)
2986
ef22810a 2987(define_insn "*sub<mode>3_compare0"
43e9d192 2988 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 2989 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 2990 (match_operand:GPI 2 "register_operand" "r"))
2991 (const_int 0)))
2992 (set (match_operand:GPI 0 "register_operand" "=r")
2993 (minus:GPI (match_dup 1) (match_dup 2)))]
2994 ""
2995 "subs\\t%<w>0, %<w>1, %<w>2"
1d61feeb 2996 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 2997)
2998
3b585938
IB		 2999;; zero_extend version of above
3000(define_insn "*subsi3_compare0_uxtw"
3001 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 3002 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3003 (match_operand:SI 2 "register_operand" "r"))
3004 (const_int 0)))
3005 (set (match_operand:DI 0 "register_operand" "=r")
3006 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3007 ""
3008 "subs\\t%w0, %w1, %w2"
1d61feeb 3009 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3010)
3011
a58fe3c5 3012(define_insn "sub<mode>3_compare1_imm"
30c46053
MC		 3013 [(set (reg:CC CC_REGNUM)
3014 (compare:CC
341fca1b 3015 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
30c46053
MC		 3016 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3017 (set (match_operand:GPI 0 "register_operand" "=r,r")
3018 (plus:GPI
3019 (match_dup 1)
3020 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3021 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3022 "@
a58fe3c5
RE		 3023 subs\\t%<w>0, %<w>1, %2
3024 adds\\t%<w>0, %<w>1, #%n2"
30c46053
MC		 3025 [(set_attr "type" "alus_imm")]
3026)
3027
ef22810a
RH		 3028(define_insn "sub<mode>3_compare1"
3029 [(set (reg:CC CC_REGNUM)
3030 (compare:CC
341fca1b 3031 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
ef22810a
RH		 3032 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3033 (set (match_operand:GPI 0 "register_operand" "=r")
3034 (minus:GPI (match_dup 1) (match_dup 2)))]
3035 ""
3036 "subs\\t%<w>0, %<w>1, %<w>2"
3037 [(set_attr "type" "alus_sreg")]
3038)
3039
e49d8b68 3040(define_peephole2
341fca1b 3041 [(set (match_operand:GPI 0 "aarch64_general_reg")
e49d8b68
KT		 3042 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3043 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3044 (set (reg:CC CC_REGNUM)
3045 (compare:CC
3046 (match_dup 1)
3047 (match_dup 2)))]
3048 "!reg_overlap_mentioned_p (operands[0], operands[1])
3049 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3050 [(const_int 0)]
3051 {
3052 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3053 operands[2]));
3054 DONE;
3055 }
3056)
3057
de47f61f
KT		 3058;; Same as the above peephole but with the compare and minus in
3059;; swapped order. The restriction on overlap between operand 0
3060;; and operands 1 and 2 doesn't apply here.
3061(define_peephole2
3062 [(set (reg:CC CC_REGNUM)
3063 (compare:CC
3064 (match_operand:GPI 1 "aarch64_reg_or_zero")
3065 (match_operand:GPI 2 "aarch64_reg_or_zero")))
341fca1b 3066 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f
KT		 3067 (minus:GPI (match_dup 1)
3068 (match_dup 2)))]
3069 ""
3070 [(const_int 0)]
3071 {
3072 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3073 operands[2]));
3074 DONE;
3075 }
3076)
3077
279dc4b4 3078(define_peephole2
341fca1b 3079 [(set (match_operand:GPI 0 "aarch64_general_reg")
279dc4b4 3080 (plus:GPI (match_operand:GPI 1 "register_operand")
a58fe3c5 3081 (match_operand:GPI 2 "aarch64_plus_immediate")))
279dc4b4
KT		 3082 (set (reg:CC CC_REGNUM)
3083 (compare:CC
3084 (match_dup 1)
3085 (match_operand:GPI 3 "const_int_operand")))]
3086 "!reg_overlap_mentioned_p (operands[0], operands[1])
3087 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3088 [(const_int 0)]
3089 {
3090 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3091 operands[3], operands[2]));
279dc4b4
KT		 3092 DONE;
3093 }
3094)
3095
de47f61f
KT		 3096;; Same as the above peephole but with the compare and minus in
3097;; swapped order. The restriction on overlap between operand 0
3098;; and operands 1 doesn't apply here.
3099(define_peephole2
3100 [(set (reg:CC CC_REGNUM)
3101 (compare:CC
3102 (match_operand:GPI 1 "register_operand")
3103 (match_operand:GPI 3 "const_int_operand")))
341fca1b 3104 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f 3105 (plus:GPI (match_dup 1)
a58fe3c5 3106 (match_operand:GPI 2 "aarch64_plus_immediate")))]
de47f61f
KT		 3107 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3108 [(const_int 0)]
3109 {
3110 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3111 operands[3], operands[2]));
de47f61f
KT		 3112 DONE;
3113 }
3114)
3115
43e9d192 3116(define_insn "*sub_<shift>_<mode>"
24c56925 3117 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 3118 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3119 (ASHIFT:GPI
3120 (match_operand:GPI 1 "register_operand" "r")
3121 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3122 ""
3123 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
ae27ce51 3124 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 3125)
3126
3b585938
IB		 3127;; zero_extend version of above
3128(define_insn "*sub_<shift>_si_uxtw"
24c56925 3129 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 3130 (zero_extend:DI
3131 (minus:SI (match_operand:SI 3 "register_operand" "r")
3132 (ASHIFT:SI
3133 (match_operand:SI 1 "register_operand" "r")
3134 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3135 ""
3136 "sub\\t%w0, %w3, %w1, <shift> %2"
ae27ce51 3137 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 3138)
3139
43e9d192
IB		 3140(define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3141 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3142 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3143 (ANY_EXTEND:GPI
3144 (match_operand:ALLX 2 "register_operand" "r"))))]
3145 ""
d4febc75 3146 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
99e6af15 3147 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3148)
3149
3b585938
IB		 3150;; zero_extend version of above
3151(define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3152 [(set (match_operand:DI 0 "register_operand" "=rk")
3153 (zero_extend:DI
7eb8fb77 3154 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3155 (ANY_EXTEND:SI
3156 (match_operand:SHORT 2 "register_operand" "r")))))]
3157 ""
3158 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
99e6af15 3159 [(set_attr "type" "alu_ext")]
3b585938
IB		 3160)
3161
43e9d192
IB		 3162(define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3163 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3164 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3165 (ashift:GPI (ANY_EXTEND:GPI
3166 (match_operand:ALLX 2 "register_operand" "r"))
3167 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3168 ""
d4febc75 3169 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
99e6af15 3170 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3171)
3172
3b585938
IB		 3173;; zero_extend version of above
3174(define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3175 [(set (match_operand:DI 0 "register_operand" "=rk")
3176 (zero_extend:DI
7eb8fb77 3177 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3178 (ashift:SI (ANY_EXTEND:SI
3179 (match_operand:SHORT 2 "register_operand" "r"))
3180 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3181 ""
3182 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
99e6af15 3183 [(set_attr "type" "alu_ext")]
3b585938
IB		 3184)
3185
ef22810a
RH		 3186;; The hardware description is op1 + ~op2 + C.
3187;; = op1 + (-op2 + 1) + (1 - !C)
3188;; = op1 - op2 - 1 + 1 - !C
3189;; = op1 - op2 - !C.
3190;; We describe the latter.
3191
3192(define_insn "*sub<mode>3_carryin0"
3193 [(set (match_operand:GPI 0 "register_operand" "=r")
3194 (minus:GPI
3195 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3196 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3197 ""
3198 "sbc\\t%<w>0, %<w>1, <w>zr"
3199 [(set_attr "type" "adc_reg")]
3200)
3201
3202;; zero_extend version of the above
3203(define_insn "*subsi3_carryin_uxtw"
3204 [(set (match_operand:DI 0 "register_operand" "=r")
3205 (zero_extend:DI
3206 (minus:SI
3207 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3208 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3209 ""
3210 "sbc\\t%w0, %w1, wzr"
3211 [(set_attr "type" "adc_reg")]
3212)
3213
3214(define_expand "sub<mode>3_carryin"
3215 [(set (match_operand:GPI 0 "register_operand")
3216 (minus:GPI
3217 (minus:GPI
3218 (match_operand:GPI 1 "aarch64_reg_or_zero")
3219 (match_operand:GPI 2 "register_operand"))
3220 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3221 ""
3222 ""
3223)
3224
3225(define_insn "*sub<mode>3_carryin"
3226 [(set (match_operand:GPI 0 "register_operand" "=r")
3227 (minus:GPI
3228 (minus:GPI
3229 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3230 (match_operand:GPI 2 "register_operand" "r"))
3231 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3232
0b8cdc58
IB		 3233 ""
3234 "sbc\\t%<w>0, %<w>1, %<w>2"
99e6af15 3235 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3236)
3237
3238;; zero_extend version of the above
3239(define_insn "*subsi3_carryin_uxtw"
ef22810a
RH		 3240 [(set (match_operand:DI 0 "register_operand" "=r")
3241 (zero_extend:DI
3242 (minus:SI
3243 (minus:SI
3244 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3245 (match_operand:SI 2 "register_operand" "r"))
3246 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3247
3248 ""
3249 "sbc\\t%w0, %w1, %w2"
3250 [(set_attr "type" "adc_reg")]
3251)
3252
3253(define_insn "*sub<mode>3_carryin_alt"
3254 [(set (match_operand:GPI 0 "register_operand" "=r")
3255 (minus:GPI
3256 (minus:GPI
3257 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3258 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3259 (match_operand:GPI 2 "register_operand" "r")))]
3260 ""
3261 "sbc\\t%<w>0, %<w>1, %<w>2"
3262 [(set_attr "type" "adc_reg")]
3263)
3264
3265;; zero_extend version of the above
3266(define_insn "*subsi3_carryin_alt_uxtw"
3267 [(set (match_operand:DI 0 "register_operand" "=r")
3268 (zero_extend:DI
3269 (minus:SI
3270 (minus:SI
3271 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3272 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3273 (match_operand:SI 2 "register_operand" "r"))))]
0b8cdc58
IB		 3274 ""
3275 "sbc\\t%w0, %w1, %w2"
99e6af15 3276 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3277)
3278
a58fe3c5 3279(define_expand "usub<GPI:mode>3_carryinC"
30c46053
MC		 3280 [(parallel
3281 [(set (reg:CC CC_REGNUM)
3282 (compare:CC
a58fe3c5 3283 (zero_extend:<DWI>
1bbffb87 3284 (match_operand:GPI 1 "aarch64_reg_or_zero"))
30c46053 3285 (plus:<DWI>
a58fe3c5 3286 (zero_extend:<DWI>
1bbffb87 3287 (match_operand:GPI 2 "register_operand"))
30c46053 3288 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
1bbffb87 3289 (set (match_operand:GPI 0 "register_operand")
30c46053
MC		 3290 (minus:GPI
3291 (minus:GPI (match_dup 1) (match_dup 2))
3292 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3293 ""
3294)
3295
a58fe3c5 3296(define_insn "*usub<GPI:mode>3_carryinC_z1"
30c46053
MC		 3297 [(set (reg:CC CC_REGNUM)
3298 (compare:CC
3299 (const_int 0)
3300 (plus:<DWI>
a58fe3c5 3301 (zero_extend:<DWI>
30c46053
MC		 3302 (match_operand:GPI 1 "register_operand" "r"))
3303 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3304 (set (match_operand:GPI 0 "register_operand" "=r")
3305 (minus:GPI
3306 (neg:GPI (match_dup 1))
3307 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3308 ""
3309 "sbcs\\t%<w>0, <w>zr, %<w>1"
3310 [(set_attr "type" "adc_reg")]
3311)
3312
a58fe3c5 3313(define_insn "*usub<GPI:mode>3_carryinC_z2"
30c46053
MC		 3314 [(set (reg:CC CC_REGNUM)
3315 (compare:CC
a58fe3c5 3316 (zero_extend:<DWI>
30c46053
MC		 3317 (match_operand:GPI 1 "register_operand" "r"))
3318 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3319 (set (match_operand:GPI 0 "register_operand" "=r")
3320 (minus:GPI
3321 (match_dup 1)
3322 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3323 ""
3324 "sbcs\\t%<w>0, %<w>1, <w>zr"
3325 [(set_attr "type" "adc_reg")]
3326)
3327
a58fe3c5 3328(define_insn "*usub<GPI:mode>3_carryinC"
30c46053
MC		 3329 [(set (reg:CC CC_REGNUM)
3330 (compare:CC
a58fe3c5 3331 (zero_extend:<DWI>
30c46053
MC		 3332 (match_operand:GPI 1 "register_operand" "r"))
3333 (plus:<DWI>
a58fe3c5 3334 (zero_extend:<DWI>
30c46053
MC		 3335 (match_operand:GPI 2 "register_operand" "r"))
3336 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3337 (set (match_operand:GPI 0 "register_operand" "=r")
3338 (minus:GPI
3339 (minus:GPI (match_dup 1) (match_dup 2))
3340 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3341 ""
3342 "sbcs\\t%<w>0, %<w>1, %<w>2"
3343 [(set_attr "type" "adc_reg")]
3344)
3345
a58fe3c5
RE		 3346(define_expand "sub<GPI:mode>3_carryinV"
3347 [(parallel
3348 [(set (reg:CC_V CC_REGNUM)
3349 (compare:CC_V
3350 (minus:<DWI>
3351 (sign_extend:<DWI>
1bbffb87 3352 (match_operand:GPI 1 "aarch64_reg_or_zero"))
a58fe3c5
RE		 3353 (plus:<DWI>
3354 (sign_extend:<DWI>
1bbffb87 3355 (match_operand:GPI 2 "register_operand"))
a58fe3c5
RE		 3356 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3357 (sign_extend:<DWI>
3358 (minus:GPI (match_dup 1)
3359 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3360 (match_dup 2))))))
1bbffb87 3361 (set (match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 3362 (minus:GPI
3363 (minus:GPI (match_dup 1) (match_dup 2))
3364 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3365 ""
3366)
3367
3368(define_insn "*sub<mode>3_carryinV_z2"
3369 [(set (reg:CC_V CC_REGNUM)
3370 (compare:CC_V
3371 (minus:<DWI>
3372 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3373 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3374 (sign_extend:<DWI>
3375 (minus:GPI (match_dup 1)
3376 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3377 (set (match_operand:GPI 0 "register_operand" "=r")
3378 (minus:GPI
3379 (match_dup 1) (match_dup 3)))]
3380 ""
3381 "sbcs\\t%<w>0, %<w>1, <w>zr"
3382 [(set_attr "type" "adc_reg")]
3383)
3384
3385(define_insn "*sub<mode>3_carryinV"
3386 [(set (reg:CC_V CC_REGNUM)
3387 (compare:CC_V
3388 (minus:<DWI>
3389 (sign_extend:<DWI>
3390 (match_operand:GPI 1 "register_operand" "r"))
3391 (plus:<DWI>
3392 (sign_extend:<DWI>
3393 (match_operand:GPI 2 "register_operand" "r"))
3394 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3395 (sign_extend:<DWI>
3396 (minus:GPI
3397 (match_dup 1)
3398 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3399 (match_dup 2))))))
3400 (set (match_operand:GPI 0 "register_operand" "=r")
3401 (minus:GPI
3402 (minus:GPI (match_dup 1) (match_dup 2))
3403 (match_dup 4)))]
3404 ""
3405 "sbcs\\t%<w>0, %<w>1, %<w>2"
3406 [(set_attr "type" "adc_reg")]
3407)
3408
dd1bdce4
VK		 3409(define_insn "*sub_uxt<mode>_shift2"
3410 [(set (match_operand:GPI 0 "register_operand" "=rk")
3411 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3412 (and:GPI
3413 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3414 (match_operand 2 "aarch64_imm3" "Ui3"))
3415 (match_operand 3 "const_int_operand" "n"))))]
3416 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3417 "*
3418 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3419 INTVAL (operands[3])));
d4febc75 3420 return \"sub\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
dd1bdce4
VK		 3421 [(set_attr "type" "alu_ext")]
3422)
3423
3424;; zero_extend version of above
3425(define_insn "*sub_uxtsi_shift2_uxtw"
3426 [(set (match_operand:DI 0 "register_operand" "=rk")
3427 (zero_extend:DI
3428 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3429 (and:SI
3430 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3431 (match_operand 2 "aarch64_imm3" "Ui3"))
3432 (match_operand 3 "const_int_operand" "n")))))]
3433 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3434 "*
3435 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3436 INTVAL (operands[3])));
3437 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3438 [(set_attr "type" "alu_ext")]
3439)
3440
fc8b587d 3441(define_expand "abs<mode>2"
1bbffb87
DZ		 3442 [(match_operand:GPI 0 "register_operand")
3443 (match_operand:GPI 1 "register_operand")]
26366d28 3444 ""
26366d28 3445 {
fc8b587d
WD		 3446 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3447 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3448 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
26366d28
IB		 3449 DONE;
3450 }
26366d28
IB		 3451)
3452
43e9d192 3453(define_insn "neg<mode>2"
040d8a1c
IB		 3454 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3455 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 3456 ""
040d8a1c
IB		 3457 "@
3458 neg\\t%<w>0, %<w>1
3459 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
1d61feeb 3460 [(set_attr "type" "alu_sreg, neon_neg<q>")
488461d8 3461 (set_attr "arch" "*,simd")]
43e9d192
IB		 3462)
3463
3b585938
IB		 3464;; zero_extend version of above
3465(define_insn "*negsi2_uxtw"
3466 [(set (match_operand:DI 0 "register_operand" "=r")
3467 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3468 ""
3469 "neg\\t%w0, %w1"
1d61feeb 3470 [(set_attr "type" "alu_sreg")]
3b585938
IB		 3471)
3472
7e0228bf
N		 3473(define_insn "*ngc<mode>"
3474 [(set (match_operand:GPI 0 "register_operand" "=r")
ef22810a
RH		 3475 (minus:GPI
3476 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3477 (match_operand:GPI 1 "register_operand" "r")))]
7e0228bf
N		 3478 ""
3479 "ngc\\t%<w>0, %<w>1"
99e6af15 3480 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3481)
3482
3483(define_insn "*ngcsi_uxtw"
3484 [(set (match_operand:DI 0 "register_operand" "=r")
3485 (zero_extend:DI
ef22810a
RH		 3486 (minus:SI
3487 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3488 (match_operand:SI 1 "register_operand" "r"))))]
7e0228bf
N		 3489 ""
3490 "ngc\\t%w0, %w1"
99e6af15 3491 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3492)
3493
4f58fe36 3494(define_insn "neg<mode>2_compare0"
43e9d192
IB		 3495 [(set (reg:CC_NZ CC_REGNUM)
3496 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3497 (const_int 0)))
3498 (set (match_operand:GPI 0 "register_operand" "=r")
3499 (neg:GPI (match_dup 1)))]
3500 ""
3501 "negs\\t%<w>0, %<w>1"
1d61feeb 3502 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 3503)
3504
3b585938
IB		 3505;; zero_extend version of above
3506(define_insn "*negsi2_compare0_uxtw"
3507 [(set (reg:CC_NZ CC_REGNUM)
3508 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3509 (const_int 0)))
3510 (set (match_operand:DI 0 "register_operand" "=r")
3511 (zero_extend:DI (neg:SI (match_dup 1))))]
3512 ""
3513 "negs\\t%w0, %w1"
1d61feeb 3514 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3515)
3516
b056c910
N		 3517(define_insn "*neg_<shift><mode>3_compare0"
3518 [(set (reg:CC_NZ CC_REGNUM)
3519 (compare:CC_NZ
3520 (neg:GPI (ASHIFT:GPI
3521 (match_operand:GPI 1 "register_operand" "r")
3522 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3523 (const_int 0)))
3524 (set (match_operand:GPI 0 "register_operand" "=r")
3525 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3526 ""
3527 "negs\\t%<w>0, %<w>1, <shift> %2"
99e6af15 3528 [(set_attr "type" "alus_shift_imm")]
b056c910
N		 3529)
3530
43e9d192
IB		 3531(define_insn "*neg_<shift>_<mode>2"
3532 [(set (match_operand:GPI 0 "register_operand" "=r")
3533 (neg:GPI (ASHIFT:GPI
3534 (match_operand:GPI 1 "register_operand" "r")
3535 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3536 ""
3537 "neg\\t%<w>0, %<w>1, <shift> %2"
ae27ce51 3538 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 3539)
3540
3b585938
IB		 3541;; zero_extend version of above
3542(define_insn "*neg_<shift>_si2_uxtw"
3543 [(set (match_operand:DI 0 "register_operand" "=r")
3544 (zero_extend:DI
3545 (neg:SI (ASHIFT:SI
3546 (match_operand:SI 1 "register_operand" "r")
3547 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3548 ""
3549 "neg\\t%w0, %w1, <shift> %2"
ae27ce51 3550 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 3551)
3552
43e9d192
IB		 3553(define_insn "mul<mode>3"
3554 [(set (match_operand:GPI 0 "register_operand" "=r")
3555 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3556 (match_operand:GPI 2 "register_operand" "r")))]
3557 ""
3558 "mul\\t%<w>0, %<w>1, %<w>2"
99e6af15 3559 [(set_attr "type" "mul")]
43e9d192
IB		 3560)
3561
3b585938
IB		 3562;; zero_extend version of above
3563(define_insn "*mulsi3_uxtw"
3564 [(set (match_operand:DI 0 "register_operand" "=r")
3565 (zero_extend:DI
3566 (mult:SI (match_operand:SI 1 "register_operand" "r")
3567 (match_operand:SI 2 "register_operand" "r"))))]
3568 ""
3569 "mul\\t%w0, %w1, %w2"
99e6af15 3570 [(set_attr "type" "mul")]
3b585938
IB		 3571)
3572
2195867f 3573(define_insn "madd<mode>"
43e9d192
IB		 3574 [(set (match_operand:GPI 0 "register_operand" "=r")
3575 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3576 (match_operand:GPI 2 "register_operand" "r"))
3577 (match_operand:GPI 3 "register_operand" "r")))]
3578 ""
3579 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3580 [(set_attr "type" "mla")]
43e9d192
IB		 3581)
3582
3b585938
IB		 3583;; zero_extend version of above
3584(define_insn "*maddsi_uxtw"
3585 [(set (match_operand:DI 0 "register_operand" "=r")
3586 (zero_extend:DI
3587 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3588 (match_operand:SI 2 "register_operand" "r"))
3589 (match_operand:SI 3 "register_operand" "r"))))]
3590 ""
3591 "madd\\t%w0, %w1, %w2, %w3"
99e6af15 3592 [(set_attr "type" "mla")]
3b585938
IB		 3593)
3594
43e9d192
IB		 3595(define_insn "*msub<mode>"
3596 [(set (match_operand:GPI 0 "register_operand" "=r")
3597 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3598 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3599 (match_operand:GPI 2 "register_operand" "r"))))]
3600
3601 ""
3602 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3603 [(set_attr "type" "mla")]
43e9d192
IB		 3604)
3605
3b585938
IB		 3606;; zero_extend version of above
3607(define_insn "*msubsi_uxtw"
3608 [(set (match_operand:DI 0 "register_operand" "=r")
3609 (zero_extend:DI
3610 (minus:SI (match_operand:SI 3 "register_operand" "r")
3611 (mult:SI (match_operand:SI 1 "register_operand" "r")
3612 (match_operand:SI 2 "register_operand" "r")))))]
3613
3614 ""
3615 "msub\\t%w0, %w1, %w2, %w3"
99e6af15 3616 [(set_attr "type" "mla")]
3b585938
IB		 3617)
3618
43e9d192
IB		 3619(define_insn "*mul<mode>_neg"
3620 [(set (match_operand:GPI 0 "register_operand" "=r")
3621 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3622 (match_operand:GPI 2 "register_operand" "r")))]
3623
3624 ""
3625 "mneg\\t%<w>0, %<w>1, %<w>2"
99e6af15 3626 [(set_attr "type" "mul")]
43e9d192
IB		 3627)
3628
3b585938
IB		 3629;; zero_extend version of above
3630(define_insn "*mulsi_neg_uxtw"
3631 [(set (match_operand:DI 0 "register_operand" "=r")
3632 (zero_extend:DI
3633 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3634 (match_operand:SI 2 "register_operand" "r"))))]
3635
3636 ""
3637 "mneg\\t%w0, %w1, %w2"
99e6af15 3638 [(set_attr "type" "mul")]
3b585938
IB		 3639)
3640
43e9d192
IB		 3641(define_insn "<su_optab>mulsidi3"
3642 [(set (match_operand:DI 0 "register_operand" "=r")
3643 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3644 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3645 ""
3646 "<su>mull\\t%0, %w1, %w2"
99e6af15 3647 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3648)
3649
3650(define_insn "<su_optab>maddsidi4"
3651 [(set (match_operand:DI 0 "register_operand" "=r")
3652 (plus:DI (mult:DI
3653 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3654 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3655 (match_operand:DI 3 "register_operand" "r")))]
3656 ""
3657 "<su>maddl\\t%0, %w1, %w2, %3"
99e6af15 3658 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3659)
3660
3661(define_insn "<su_optab>msubsidi4"
3662 [(set (match_operand:DI 0 "register_operand" "=r")
3663 (minus:DI
3664 (match_operand:DI 3 "register_operand" "r")
3665 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3666 (ANY_EXTEND:DI
3667 (match_operand:SI 2 "register_operand" "r")))))]
3668 ""
3669 "<su>msubl\\t%0, %w1, %w2, %3"
99e6af15 3670 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3671)
3672
3673(define_insn "*<su_optab>mulsidi_neg"
3674 [(set (match_operand:DI 0 "register_operand" "=r")
3675 (mult:DI (neg:DI
3676 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3677 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3678 ""
3679 "<su>mnegl\\t%0, %w1, %w2"
99e6af15 3680 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3681)
3682
2195867f
RH		 3683(define_expand "<su_optab>mulditi3"
3684 [(set (match_operand:TI 0 "register_operand")
3685 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3686 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3687 ""
3688{
3689 rtx low = gen_reg_rtx (DImode);
3690 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3691
3692 rtx high = gen_reg_rtx (DImode);
3693 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3694
3695 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3696 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3697 DONE;
3698})
3699
3700;; The default expansion of multi3 using umuldi3_highpart will perform
3701;; the additions in an order that fails to combine into two madd insns.
3702(define_expand "multi3"
3703 [(set (match_operand:TI 0 "register_operand")
3704 (mult:TI (match_operand:TI 1 "register_operand")
3705 (match_operand:TI 2 "register_operand")))]
3706 ""
3707{
3708 rtx l0 = gen_reg_rtx (DImode);
3709 rtx l1 = gen_lowpart (DImode, operands[1]);
3710 rtx l2 = gen_lowpart (DImode, operands[2]);
3711 rtx h0 = gen_reg_rtx (DImode);
3712 rtx h1 = gen_highpart (DImode, operands[1]);
3713 rtx h2 = gen_highpart (DImode, operands[2]);
3714
3715 emit_insn (gen_muldi3 (l0, l1, l2));
3716 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3717 emit_insn (gen_madddi (h0, h1, l2, h0));
3718 emit_insn (gen_madddi (h0, l1, h2, h0));
3719
3720 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3721 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3722 DONE;
3723})
3724
43e9d192
IB		 3725(define_insn "<su>muldi3_highpart"
3726 [(set (match_operand:DI 0 "register_operand" "=r")
3727 (truncate:DI
3728 (lshiftrt:TI
3729 (mult:TI
3730 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3731 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3732 (const_int 64))))]
3733 ""
3734 "<su>mulh\\t%0, %1, %2"
99e6af15 3735 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3736)
3737
3738(define_insn "<su_optab>div<mode>3"
3739 [(set (match_operand:GPI 0 "register_operand" "=r")
3740 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3741 (match_operand:GPI 2 "register_operand" "r")))]
3742 ""
3743 "<su>div\\t%<w>0, %<w>1, %<w>2"
99e6af15 3744 [(set_attr "type" "<su>div")]
43e9d192
IB		 3745)
3746
3b585938
IB		 3747;; zero_extend version of above
3748(define_insn "*<su_optab>divsi3_uxtw"
3749 [(set (match_operand:DI 0 "register_operand" "=r")
3750 (zero_extend:DI
3751 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3752 (match_operand:SI 2 "register_operand" "r"))))]
3753 ""
3754 "<su>div\\t%w0, %w1, %w2"
99e6af15 3755 [(set_attr "type" "<su>div")]
3b585938
IB		 3756)
3757
43e9d192
IB		 3758;; -------------------------------------------------------------------
3759;; Comparison insns
3760;; -------------------------------------------------------------------
3761
c8012fbc 3762(define_insn "cmp<mode>"
43e9d192 3763 [(set (reg:CC CC_REGNUM)
3751345d 3764 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
34b512f5 3765 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
43e9d192
IB		 3766 ""
3767 "@
34b512f5 3768 cmp\\t%<w>0, %<w>1
e2b691c4 3769 cmp\\t%<w>0, %1
43e9d192 3770 cmn\\t%<w>0, #%n1"
1d61feeb 3771 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 3772)
3773
786e3c06 3774(define_insn "fcmp<mode>"
43e9d192
IB		 3775 [(set (reg:CCFP CC_REGNUM)
3776 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3777 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3778 "TARGET_FLOAT"
3779 "@
3780 fcmp\\t%<s>0, #0.0
3781 fcmp\\t%<s>0, %<s>1"
99e6af15 3782 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3783)
3784
786e3c06 3785(define_insn "fcmpe<mode>"
43e9d192
IB		 3786 [(set (reg:CCFPE CC_REGNUM)
3787 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3788 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3789 "TARGET_FLOAT"
3790 "@
3791 fcmpe\\t%<s>0, #0.0
3792 fcmpe\\t%<s>0, %<s>1"
99e6af15 3793 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3794)
3795
3796(define_insn "*cmp_swp_<shift>_reg<mode>"
3797 [(set (reg:CC_SWP CC_REGNUM)
3798 (compare:CC_SWP (ASHIFT:GPI
3799 (match_operand:GPI 0 "register_operand" "r")
3800 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3801 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3802 ""
3803 "cmp\\t%<w>2, %<w>0, <shift> %1"
99e6af15 3804 [(set_attr "type" "alus_shift_imm")]
43e9d192
IB		 3805)
3806
3807(define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3808 [(set (reg:CC_SWP CC_REGNUM)
3809 (compare:CC_SWP (ANY_EXTEND:GPI
3810 (match_operand:ALLX 0 "register_operand" "r"))
3811 (match_operand:GPI 1 "register_operand" "r")))]
3812 ""
d4febc75 3813 "cmp\\t%<GPI:w>1, %w0, <su>xt<ALLX:size>"
99e6af15 3814 [(set_attr "type" "alus_ext")]
43e9d192
IB		 3815)
3816
58cff58c
N		 3817(define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3818 [(set (reg:CC_SWP CC_REGNUM)
3819 (compare:CC_SWP (ashift:GPI
3820 (ANY_EXTEND:GPI
3821 (match_operand:ALLX 0 "register_operand" "r"))
4095f9fa 3822 (match_operand 1 "aarch64_imm3" "Ui3"))
58cff58c
N		 3823 (match_operand:GPI 2 "register_operand" "r")))]
3824 ""
d4febc75 3825 "cmp\\t%<GPI:w>2, %w0, <su>xt<ALLX:size> %1"
99e6af15 3826 [(set_attr "type" "alus_ext")]
58cff58c 3827)
43e9d192
IB		 3828
3829;; -------------------------------------------------------------------
3830;; Store-flag and conditional select insns
3831;; -------------------------------------------------------------------
3832
3833(define_expand "cstore<mode>4"
1bbffb87 3834 [(set (match_operand:SI 0 "register_operand")
43e9d192 3835 (match_operator:SI 1 "aarch64_comparison_operator"
1bbffb87
DZ		 3836 [(match_operand:GPI 2 "register_operand")
3837 (match_operand:GPI 3 "aarch64_plus_operand")]))]
43e9d192
IB		 3838 ""
3839 "
3840 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3841 operands[3]);
3842 operands[3] = const0_rtx;
3843 "
3844)
3845
cf670503
ZC		 3846(define_expand "cstorecc4"
3847 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 3848 (match_operator 1 "aarch64_comparison_operator_mode"
c8012fbc 3849 [(match_operand 2 "cc_register")
cf670503
ZC		 3850 (match_operand 3 "const0_operand")]))]
3851 ""
3852"{
f7df4a84 3853 emit_insn (gen_rtx_SET (operands[0], operands[1]));
cf670503
ZC		 3854 DONE;
3855}")
3856
3857
43e9d192 3858(define_expand "cstore<mode>4"
1bbffb87 3859 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 3860 (match_operator:SI 1 "aarch64_comparison_operator_mode"
1bbffb87
DZ		 3861 [(match_operand:GPF 2 "register_operand")
3862 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
43e9d192
IB		 3863 ""
3864 "
3865 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3866 operands[3]);
3867 operands[3] = const0_rtx;
3868 "
3869)
3870
714e1b3b 3871(define_insn "aarch64_cstore<mode>"
43e9d192 3872 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 3873 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 3874 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3875 ""
3876 "cset\\t%<w>0, %m1"
99e6af15 3877 [(set_attr "type" "csel")]
43e9d192
IB		 3878)
3879
714e1b3b
KT		 3880;; For a 24-bit immediate CST we can optimize the compare for equality
3881;; and branch sequence from:
3882;; mov x0, #imm1
3883;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
3884;; cmp x1, x0
3885;; cset x2, <ne,eq>
3886;; into the shorter:
3887;; sub x0, x1, #(CST & 0xfff000)
3888;; subs x0, x0, #(CST & 0x000fff)
3889;; cset x2, <ne, eq>.
3890(define_insn_and_split "*compare_cstore<mode>_insn"
3891 [(set (match_operand:GPI 0 "register_operand" "=r")
3892 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
c8574943
KT		 3893 (match_operand:GPI 2 "aarch64_imm24" "n")))
3894 (clobber (reg:CC CC_REGNUM))]
714e1b3b
KT		 3895 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3896 && !aarch64_plus_operand (operands[2], <MODE>mode)
3897 && !reload_completed"
3898 "#"
3899 "&& true"
3900 [(const_int 0)]
3901 {
3902 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3903 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3904 rtx tmp = gen_reg_rtx (<MODE>mode);
3905 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3906 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3907 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3908 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3909 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3910 DONE;
3911 }
3912 [(set_attr "type" "csel")]
3913)
3914
d0b6bb1b
IB		 3915;; zero_extend version of the above
3916(define_insn "*cstoresi_insn_uxtw"
3917 [(set (match_operand:DI 0 "register_operand" "=r")
3918 (zero_extend:DI
a44e0ebb 3919 (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 3920 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3921 ""
3922 "cset\\t%w0, %m1"
99e6af15 3923 [(set_attr "type" "csel")]
d0b6bb1b
IB		 3924)
3925
889b9412 3926(define_insn "cstore<mode>_neg"
43e9d192 3927 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 3928 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 3929 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3930 ""
3931 "csetm\\t%<w>0, %m1"
99e6af15 3932 [(set_attr "type" "csel")]
43e9d192
IB		 3933)
3934
d0b6bb1b
IB		 3935;; zero_extend version of the above
3936(define_insn "*cstoresi_neg_uxtw"
3937 [(set (match_operand:DI 0 "register_operand" "=r")
3938 (zero_extend:DI
a44e0ebb 3939 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 3940 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3941 ""
3942 "csetm\\t%w0, %m1"
99e6af15 3943 [(set_attr "type" "csel")]
d0b6bb1b
IB		 3944)
3945
43e9d192 3946(define_expand "cmov<mode>6"
1bbffb87 3947 [(set (match_operand:GPI 0 "register_operand")
43e9d192
IB		 3948 (if_then_else:GPI
3949 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 3950 [(match_operand:GPI 2 "register_operand")
3951 (match_operand:GPI 3 "aarch64_plus_operand")])
3952 (match_operand:GPI 4 "register_operand")
3953 (match_operand:GPI 5 "register_operand")))]
43e9d192
IB		 3954 ""
3955 "
3956 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3957 operands[3]);
3958 operands[3] = const0_rtx;
3959 "
3960)
3961
3962(define_expand "cmov<mode>6"
1bbffb87 3963 [(set (match_operand:GPF 0 "register_operand")
43e9d192
IB		 3964 (if_then_else:GPF
3965 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 3966 [(match_operand:GPF 2 "register_operand")
3967 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
3968 (match_operand:GPF 4 "register_operand")
3969 (match_operand:GPF 5 "register_operand")))]
43e9d192
IB		 3970 ""
3971 "
3972 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3973 operands[3]);
3974 operands[3] = const0_rtx;
3975 "
3976)
3977
3978(define_insn "*cmov<mode>_insn"
06b1198f 3979 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
43e9d192
IB		 3980 (if_then_else:ALLI
3981 (match_operator 1 "aarch64_comparison_operator"
3982 [(match_operand 2 "cc_register" "") (const_int 0)])
06b1198f
IB		 3983 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3984 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3985 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3986 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3987 ;; Final two alternatives should be unreachable, but included for completeness
43e9d192
IB		 3988 "@
3989 csel\\t%<w>0, %<w>3, %<w>4, %m1
3990 csinv\\t%<w>0, %<w>3, <w>zr, %m1
3991 csinv\\t%<w>0, %<w>4, <w>zr, %M1
06b1198f
IB		 3992 csinc\\t%<w>0, %<w>3, <w>zr, %m1
3993 csinc\\t%<w>0, %<w>4, <w>zr, %M1
3994 mov\\t%<w>0, -1
3995 mov\\t%<w>0, 1"
04e34811 3996 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
43e9d192
IB		 3997)
3998
d0b6bb1b
IB		 3999;; zero_extend version of above
4000(define_insn "*cmovsi_insn_uxtw"
4001 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4002 (zero_extend:DI
4003 (if_then_else:SI
4004 (match_operator 1 "aarch64_comparison_operator"
4005 [(match_operand 2 "cc_register" "") (const_int 0)])
4006 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4007 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4008 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4009 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4010 ;; Final two alternatives should be unreachable, but included for completeness
4011 "@
4012 csel\\t%w0, %w3, %w4, %m1
4013 csinv\\t%w0, %w3, wzr, %m1
4014 csinv\\t%w0, %w4, wzr, %M1
4015 csinc\\t%w0, %w3, wzr, %m1
4016 csinc\\t%w0, %w4, wzr, %M1
4017 mov\\t%w0, -1
4018 mov\\t%w0, 1"
04e34811 4019 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
d0b6bb1b
IB		 4020)
4021
1d3e6ed3
RL		 4022(define_insn "*cmovdi_insn_uxtw"
4023 [(set (match_operand:DI 0 "register_operand" "=r")
4024 (if_then_else:DI
4025 (match_operator 1 "aarch64_comparison_operator"
4026 [(match_operand 2 "cc_register" "") (const_int 0)])
4027 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4028 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4029 ""
4030 "csel\\t%w0, %w3, %w4, %m1"
4031 [(set_attr "type" "csel")]
4032)
4033
43e9d192
IB		 4034(define_insn "*cmov<mode>_insn"
4035 [(set (match_operand:GPF 0 "register_operand" "=w")
4036 (if_then_else:GPF
4037 (match_operator 1 "aarch64_comparison_operator"
4038 [(match_operand 2 "cc_register" "") (const_int 0)])
4039 (match_operand:GPF 3 "register_operand" "w")
4040 (match_operand:GPF 4 "register_operand" "w")))]
4041 "TARGET_FLOAT"
4042 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
99e6af15 4043 [(set_attr "type" "fcsel")]
43e9d192
IB		 4044)
4045
4046(define_expand "mov<mode>cc"
1bbffb87
DZ		 4047 [(set (match_operand:ALLI 0 "register_operand")
4048 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4049 (match_operand:ALLI 2 "register_operand")
4050 (match_operand:ALLI 3 "register_operand")))]
43e9d192
IB		 4051 ""
4052 {
c8012fbc 4053 rtx ccreg;
43e9d192
IB		 4054 enum rtx_code code = GET_CODE (operands[1]);
4055
4056 if (code == UNEQ || code == LTGT)
4057 FAIL;
4058
c8012fbc
WD		 4059 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4060 XEXP (operands[1], 1));
4061 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
43e9d192
IB		 4062 }
4063)
4064
4065(define_expand "mov<GPF:mode><GPI:mode>cc"
1bbffb87
DZ		 4066 [(set (match_operand:GPI 0 "register_operand")
4067 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4068 (match_operand:GPF 2 "register_operand")
4069 (match_operand:GPF 3 "register_operand")))]
43e9d192
IB		 4070 ""
4071 {
4072 rtx ccreg;
4073 enum rtx_code code = GET_CODE (operands[1]);
4074
4075 if (code == UNEQ || code == LTGT)
4076 FAIL;
4077
4078 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4079 XEXP (operands[1], 1));
4080 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4081 }
4082)
4083
d3a3182b 4084(define_expand "mov<mode>cc"
1bbffb87
DZ		 4085 [(set (match_operand:GPF 0 "register_operand")
4086 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4087 (match_operand:GPF 2 "register_operand")
4088 (match_operand:GPF 3 "register_operand")))]
d3a3182b
ZC		 4089 ""
4090 {
4091 rtx ccreg;
4092 enum rtx_code code = GET_CODE (operands[1]);
4093
4094 if (code == UNEQ || code == LTGT)
4095 FAIL;
4096
4097 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4098 XEXP (operands[1], 1));
4099 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4100 }
4101)
4102
3204ac98 4103(define_expand "<neg_not_op><mode>cc"
1bbffb87
DZ		 4104 [(set (match_operand:GPI 0 "register_operand")
4105 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4106 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4107 (match_operand:GPI 3 "register_operand")))]
3204ac98
KT		 4108 ""
4109 {
4110 rtx ccreg;
4111 enum rtx_code code = GET_CODE (operands[1]);
4112
4113 if (code == UNEQ || code == LTGT)
4114 FAIL;
4115
4116 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4117 XEXP (operands[1], 1));
4118 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4119 }
4120)
5d357f26
KT		 4121
4122;; CRC32 instructions.
4123(define_insn "aarch64_<crc_variant>"
4124 [(set (match_operand:SI 0 "register_operand" "=r")
4125 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4126 (match_operand:<crc_mode> 2 "register_operand" "r")]
4127 CRC))]
4128 "TARGET_CRC32"
4129 {
6a70badb 4130 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
5d357f26
KT		 4131 return "<crc_variant>\\t%w0, %w1, %x2";
4132 else
4133 return "<crc_variant>\\t%w0, %w1, %w2";
4134 }
4135 [(set_attr "type" "crc")]
4136)
4137
43e9d192
IB		 4138(define_insn "*csinc2<mode>_insn"
4139 [(set (match_operand:GPI 0 "register_operand" "=r")
cd5660ab
KT		 4140 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4141 (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4142 ""
49f88e43 4143 "cinc\\t%<w>0, %<w>1, %m2"
99e6af15
JG		 4144 [(set_attr "type" "csel")]
4145)
43e9d192
IB		 4146
4147(define_insn "csinc3<mode>_insn"
4148 [(set (match_operand:GPI 0 "register_operand" "=r")
4149 (if_then_else:GPI
cd5660ab
KT		 4150 (match_operand 1 "aarch64_comparison_operation" "")
4151 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
43e9d192 4152 (const_int 1))
cd5660ab 4153 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4154 ""
cd5660ab 4155 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15 4156 [(set_attr "type" "csel")]
43e9d192
IB		 4157)
4158
4159(define_insn "*csinv3<mode>_insn"
4160 [(set (match_operand:GPI 0 "register_operand" "=r")
4161 (if_then_else:GPI
cd5660ab
KT		 4162 (match_operand 1 "aarch64_comparison_operation" "")
4163 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4164 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4165 ""
cd5660ab 4166 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15
JG		 4167 [(set_attr "type" "csel")]
4168)
43e9d192 4169
0f6ca79c
RL		 4170(define_insn "csneg3_uxtw_insn"
4171 [(set (match_operand:DI 0 "register_operand" "=r")
4172 (zero_extend:DI
4173 (if_then_else:SI
4174 (match_operand 1 "aarch64_comparison_operation" "")
4175 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4176 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4177 ""
4178 "csneg\\t%w0, %w3, %w2, %M1"
4179 [(set_attr "type" "csel")]
4180)
4181
fc8b587d 4182(define_insn "csneg3<mode>_insn"
43e9d192
IB		 4183 [(set (match_operand:GPI 0 "register_operand" "=r")
4184 (if_then_else:GPI
cd5660ab
KT		 4185 (match_operand 1 "aarch64_comparison_operation" "")
4186 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4187 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4188 ""
cd5660ab 4189 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15 4190 [(set_attr "type" "csel")]
d572ad49
AC		 4191)
4192
4193(define_insn "*csinv3_uxtw_insn1"
4194 [(set (match_operand:DI 0 "register_operand" "=r")
4195 (if_then_else:DI
4196 (match_operand 1 "aarch64_comparison_operation" "")
4197 (zero_extend:DI
4198 (match_operand:SI 2 "register_operand" "r"))
4199 (zero_extend:DI
4200 (NEG_NOT:SI (match_operand:SI 3 "register_operand" "r")))))]
4201 ""
4202 "cs<neg_not_cs>\\t%w0, %w2, %w3, %m1"
4203 [(set_attr "type" "csel")]
4204)
4205
4206(define_insn "*csinv3_uxtw_insn2"
4207 [(set (match_operand:DI 0 "register_operand" "=r")
4208 (if_then_else:DI
4209 (match_operand 1 "aarch64_comparison_operation" "")
4210 (zero_extend:DI
4211 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4212 (zero_extend:DI
4213 (match_operand:SI 3 "register_operand" "r"))))]
4214 ""
4215 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
4216 [(set_attr "type" "csel")]
4217)
4218
4219(define_insn "*csinv3_uxtw_insn3"
4220 [(set (match_operand:DI 0 "register_operand" "=r")
4221 (if_then_else:DI
4222 (match_operand 1 "aarch64_comparison_operation" "")
4223 (zero_extend:DI
4224 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4225 (const_int 0)))]
4226 ""
4227 "cs<neg_not_cs>\\t%w0, wzr, %w2, %M1"
4228 [(set_attr "type" "csel")]
99e6af15 4229)
43e9d192 4230
7cfb4d93
RS		 4231;; If X can be loaded by a single CNT[BHWD] instruction,
4232;;
4233;; A = UMAX (B, X)
4234;;
4235;; is equivalent to:
4236;;
4237;; TMP = UQDEC[BHWD] (B, X)
4238;; A = TMP + X
4239;;
4240;; Defining the pattern this way means that:
4241;;
4242;; A = UMAX (B, X) - X
4243;;
4244;; becomes:
4245;;
4246;; TMP1 = UQDEC[BHWD] (B, X)
4247;; TMP2 = TMP1 + X
4248;; A = TMP2 - X
4249;;
4250;; which combine can optimize to:
4251;;
4252;; A = UQDEC[BHWD] (B, X)
4253;;
4254;; We don't use match_operand predicates because the order of the operands
4255;; can vary: the CNT[BHWD] constant will come first if the other operand is
4256;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4257(define_expand "umax<mode>3"
4258 [(set (match_operand:GPI 0 "register_operand")
4259 (umax:GPI (match_operand:GPI 1 "")
4260 (match_operand:GPI 2 "")))]
4261 "TARGET_SVE"
4262 {
4263 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4264 std::swap (operands[1], operands[2]);
4265 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4266 FAIL;
4267 rtx temp = gen_reg_rtx (<MODE>mode);
4268 operands[1] = force_reg (<MODE>mode, operands[1]);
4269 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4270 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4271 DONE;
4272 }
4273)
4274
4275;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4276(define_insn "aarch64_uqdec<mode>"
4277 [(set (match_operand:GPI 0 "register_operand" "=r")
4278 (minus:GPI
4279 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4280 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4281 (match_dup 2)))]
4282 "TARGET_SVE"
4283 {
4284 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4285 }
4286)
4287
43e9d192
IB		 4288;; -------------------------------------------------------------------
4289;; Logical operations
4290;; -------------------------------------------------------------------
4291
43fd192f
MC		 4292
4293(define_insn_and_split "*aarch64_and<mode>_imm2"
4294 [(set (match_operand:GPI 0 "register_operand" "=rk")
4295 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4296 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4297 ""
4298 "#"
4299 "true"
4300 [(const_int 0)]
4301 {
4302 HOST_WIDE_INT val = INTVAL (operands[2]);
4303 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4304 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4305
4306 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4307 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4308 DONE;
4309 }
4310)
4311
43e9d192 4312(define_insn "<optab><mode>3"
fe82d1f2
AL		 4313 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4314 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4315 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
43e9d192 4316 ""
fe82d1f2
AL		 4317 "@
4318 <logical>\\t%<w>0, %<w>1, %<w>2
e2b691c4 4319 <logical>\\t%<w>0, %<w>1, %2
fe82d1f2
AL		 4320 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4321 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
488461d8 4322 (set_attr "arch" "*,*,simd")]
99e6af15 4323)
43e9d192 4324
d0b6bb1b
IB		 4325;; zero_extend version of above
4326(define_insn "*<optab>si3_uxtw"
4327 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4328 (zero_extend:DI
4329 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4330 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4331 ""
954224d7
WD		 4332 "@
4333 <logical>\\t%w0, %w1, %w2
4334 <logical>\\t%w0, %w1, %2"
99e6af15
JG		 4335 [(set_attr "type" "logic_reg,logic_imm")]
4336)
d0b6bb1b 4337
2a1d1991
N		 4338(define_insn "*and<mode>3_compare0"
4339 [(set (reg:CC_NZ CC_REGNUM)
4340 (compare:CC_NZ
4341 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4342 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4343 (const_int 0)))
4344 (set (match_operand:GPI 0 "register_operand" "=r,r")
4345 (and:GPI (match_dup 1) (match_dup 2)))]
4346 ""
954224d7
WD		 4347 "@
4348 ands\\t%<w>0, %<w>1, %<w>2
4349 ands\\t%<w>0, %<w>1, %2"
99e6af15 4350 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4351)
4352
4353;; zero_extend version of above
4354(define_insn "*andsi3_compare0_uxtw"
4355 [(set (reg:CC_NZ CC_REGNUM)
4356 (compare:CC_NZ
4357 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4358 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4359 (const_int 0)))
4360 (set (match_operand:DI 0 "register_operand" "=r,r")
4361 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4362 ""
954224d7
WD		 4363 "@
4364 ands\\t%w0, %w1, %w2
4365 ands\\t%w0, %w1, %2"
99e6af15 4366 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4367)
4368
4369(define_insn "*and_<SHIFT:optab><mode>3_compare0"
4370 [(set (reg:CC_NZ CC_REGNUM)
4371 (compare:CC_NZ
4372 (and:GPI (SHIFT:GPI
4373 (match_operand:GPI 1 "register_operand" "r")
4374 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4375 (match_operand:GPI 3 "register_operand" "r"))
4376 (const_int 0)))
4377 (set (match_operand:GPI 0 "register_operand" "=r")
4378 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4379 ""
4380 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15 4381 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4382)
4383
4384;; zero_extend version of above
4385(define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4386 [(set (reg:CC_NZ CC_REGNUM)
4387 (compare:CC_NZ
4388 (and:SI (SHIFT:SI
4389 (match_operand:SI 1 "register_operand" "r")
4390 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4391 (match_operand:SI 3 "register_operand" "r"))
4392 (const_int 0)))
4393 (set (match_operand:DI 0 "register_operand" "=r")
4394 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4395 (match_dup 3))))]
4396 ""
4397 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15 4398 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4399)
4400
43e9d192
IB		 4401(define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4402 [(set (match_operand:GPI 0 "register_operand" "=r")
4403 (LOGICAL:GPI (SHIFT:GPI
4404 (match_operand:GPI 1 "register_operand" "r")
4405 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4406 (match_operand:GPI 3 "register_operand" "r")))]
4407 ""
4408 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4409 [(set_attr "type" "logic_shift_imm")]
4410)
43e9d192 4411
57b77d46
RE		 4412(define_insn "*<optab>_rol<mode>3"
4413 [(set (match_operand:GPI 0 "register_operand" "=r")
4414 (LOGICAL:GPI (rotate:GPI
4415 (match_operand:GPI 1 "register_operand" "r")
4416 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4417 (match_operand:GPI 3 "register_operand" "r")))]
4418 ""
4419 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4420 [(set_attr "type" "logic_shift_imm")]
4421)
4422
4423;; zero_extend versions of above
d0b6bb1b
IB		 4424(define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4425 [(set (match_operand:DI 0 "register_operand" "=r")
4426 (zero_extend:DI
4427 (LOGICAL:SI (SHIFT:SI
4428 (match_operand:SI 1 "register_operand" "r")
4429 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4430 (match_operand:SI 3 "register_operand" "r"))))]
4431 ""
4432 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4433 [(set_attr "type" "logic_shift_imm")]
4434)
d0b6bb1b 4435
57b77d46
RE		 4436(define_insn "*<optab>_rolsi3_uxtw"
4437 [(set (match_operand:DI 0 "register_operand" "=r")
4438 (zero_extend:DI
4439 (LOGICAL:SI (rotate:SI
4440 (match_operand:SI 1 "register_operand" "r")
4441 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4442 (match_operand:SI 3 "register_operand" "r"))))]
4443 ""
4444 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4445 [(set_attr "type" "logic_shift_imm")]
4446)
4447
43e9d192 4448(define_insn "one_cmpl<mode>2"
fe82d1f2
AL		 4449 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4450 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 4451 ""
fe82d1f2
AL		 4452 "@
4453 mvn\\t%<w>0, %<w>1
4454 mvn\\t%0.8b, %1.8b"
4455 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4456 (set_attr "arch" "*,simd")]
99e6af15 4457)
43e9d192 4458
1bd3a8af
AC		 4459(define_insn "*one_cmpl_zero_extend"
4460 [(set (match_operand:DI 0 "register_operand" "=r")
4461 (zero_extend:DI
4462 (not:SI (match_operand:SI 1 "register_operand" "r"))))]
4463 ""
4464 "mvn\\t%w0, %w1"
4465 [(set_attr "type" "logic_reg")]
4466)
4467
43e9d192
IB		 4468(define_insn "*one_cmpl_<optab><mode>2"
4469 [(set (match_operand:GPI 0 "register_operand" "=r")
4470 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4471 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4472 ""
4473 "mvn\\t%<w>0, %<w>1, <shift> %2"
99e6af15
JG		 4474 [(set_attr "type" "logic_shift_imm")]
4475)
43e9d192 4476
84be6032
AL		 4477;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4478
4479(define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4480 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4481 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4482 (match_operand:GPI 2 "register_operand" "r,w")))]
43e9d192 4483 ""
84be6032
AL		 4484 "@
4485 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4486 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4487 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4488 (set_attr "arch" "*,simd")]
84be6032
AL		 4489)
4490
8409e468
KT		 4491(define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4492 [(set (match_operand:DI 0 "register_operand" "=r")
4493 (zero_extend:DI
4494 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4495 (match_operand:SI 2 "register_operand" "r"))))]
4496 ""
4497 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4498 [(set_attr "type" "logic_reg")]
4499)
4500
4501(define_insn "*xor_one_cmplsidi3_ze"
4502 [(set (match_operand:DI 0 "register_operand" "=r")
4503 (zero_extend:DI
4504 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4505 (match_operand:SI 2 "register_operand" "r")))))]
4506 ""
4507 "eon\\t%w0, %w1, %w2"
4508 [(set_attr "type" "logic_reg")]
4509)
4510
84be6032
AL		 4511;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4512;; eon does not operate on SIMD registers so the vector variant must be split.
4513(define_insn_and_split "*xor_one_cmpl<mode>3"
4514 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4515 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4516 (match_operand:GPI 2 "register_operand" "r,w"))))]
4517 ""
eeaccc07
AL		 4518 "@
4519 eon\\t%<w>0, %<w>1, %<w>2
4520 #"
4521 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
84be6032
AL		 4522 [(set (match_operand:GPI 0 "register_operand" "=w")
4523 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4524 (match_operand:GPI 2 "register_operand" "w")))
4525 (set (match_dup 0) (not:GPI (match_dup 0)))]
4526 ""
4527 [(set_attr "type" "logic_reg,multiple")
488461d8 4528 (set_attr "arch" "*,simd")]
99e6af15 4529)
43e9d192 4530
add3c965
IB		 4531(define_insn "*and_one_cmpl<mode>3_compare0"
4532 [(set (reg:CC_NZ CC_REGNUM)
4533 (compare:CC_NZ
4534 (and:GPI (not:GPI
4535 (match_operand:GPI 1 "register_operand" "r"))
4536 (match_operand:GPI 2 "register_operand" "r"))
4537 (const_int 0)))
4538 (set (match_operand:GPI 0 "register_operand" "=r")
4539 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4540 ""
4541 "bics\\t%<w>0, %<w>2, %<w>1"
99e6af15
JG		 4542 [(set_attr "type" "logics_reg")]
4543)
add3c965
IB		 4544
4545;; zero_extend version of above
4546(define_insn "*and_one_cmplsi3_compare0_uxtw"
4547 [(set (reg:CC_NZ CC_REGNUM)
4548 (compare:CC_NZ
4549 (and:SI (not:SI
4550 (match_operand:SI 1 "register_operand" "r"))
4551 (match_operand:SI 2 "register_operand" "r"))
4552 (const_int 0)))
4553 (set (match_operand:DI 0 "register_operand" "=r")
4554 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4555 ""
4556 "bics\\t%w0, %w2, %w1"
99e6af15
JG		 4557 [(set_attr "type" "logics_reg")]
4558)
add3c965 4559
cd8fc5ee
AV		 4560(define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4561 [(set (reg:CC_NZ CC_REGNUM)
4562 (compare:CC_NZ
4563 (and:GPI (not:GPI
4564 (match_operand:GPI 0 "register_operand" "r"))
4565 (match_operand:GPI 1 "register_operand" "r"))
4566 (const_int 0)))]
4567 ""
4568 "bics\\t<w>zr, %<w>1, %<w>0"
4569 [(set_attr "type" "logics_reg")]
4570)
4571
cc05c2c3 4572(define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
43e9d192
IB		 4573 [(set (match_operand:GPI 0 "register_operand" "=r")
4574 (LOGICAL:GPI (not:GPI
4575 (SHIFT:GPI
4576 (match_operand:GPI 1 "register_operand" "r")
4577 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4578 (match_operand:GPI 3 "register_operand" "r")))]
4579 ""
4580 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
dba606bf 4581 [(set_attr "type" "logic_shift_imm")]
99e6af15 4582)
43e9d192 4583
6365da9e
KT		 4584(define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4585 [(set (match_operand:GPI 0 "register_operand" "=r")
4586 (not:GPI (xor:GPI
4587 (SHIFT:GPI
4588 (match_operand:GPI 1 "register_operand" "r")
4589 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4590 (match_operand:GPI 3 "register_operand" "r"))))]
4591 ""
4592 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4593 [(set_attr "type" "logic_shift_imm")]
4594)
4595
4596;; Zero-extend version of the above.
4597(define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4598 [(set (match_operand:DI 0 "register_operand" "=r")
4599 (zero_extend:DI
4600 (not:SI (xor:SI
4601 (SHIFT:SI
4602 (match_operand:SI 1 "register_operand" "r")
4603 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4604 (match_operand:SI 3 "register_operand" "r")))))]
4605 ""
4606 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4607 [(set_attr "type" "logic_shift_imm")]
4608)
4609
add3c965
IB		 4610(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4611 [(set (reg:CC_NZ CC_REGNUM)
4612 (compare:CC_NZ
4613 (and:GPI (not:GPI
4614 (SHIFT:GPI
4615 (match_operand:GPI 1 "register_operand" "r")
4616 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4617 (match_operand:GPI 3 "register_operand" "r"))
4618 (const_int 0)))
4619 (set (match_operand:GPI 0 "register_operand" "=r")
4620 (and:GPI (not:GPI
4621 (SHIFT:GPI
4622 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4623 ""
4624 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4625 [(set_attr "type" "logics_shift_imm")]
4626)
add3c965
IB		 4627
4628;; zero_extend version of above
4629(define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4630 [(set (reg:CC_NZ CC_REGNUM)
4631 (compare:CC_NZ
4632 (and:SI (not:SI
4633 (SHIFT:SI
4634 (match_operand:SI 1 "register_operand" "r")
4635 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4636 (match_operand:SI 3 "register_operand" "r"))
4637 (const_int 0)))
4638 (set (match_operand:DI 0 "register_operand" "=r")
4639 (zero_extend:DI (and:SI
4640 (not:SI
4641 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4642 ""
4643 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4644 [(set_attr "type" "logics_shift_imm")]
4645)
add3c965 4646
cd8fc5ee
AV		 4647(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4648 [(set (reg:CC_NZ CC_REGNUM)
4649 (compare:CC_NZ
4650 (and:GPI (not:GPI
4651 (SHIFT:GPI
4652 (match_operand:GPI 0 "register_operand" "r")
4653 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4654 (match_operand:GPI 2 "register_operand" "r"))
4655 (const_int 0)))]
4656 ""
4657 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4658 [(set_attr "type" "logics_shift_imm")]
4659)
4660
43e9d192
IB		 4661(define_insn "clz<mode>2"
4662 [(set (match_operand:GPI 0 "register_operand" "=r")
4663 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4664 ""
4665 "clz\\t%<w>0, %<w>1"
99e6af15
JG		 4666 [(set_attr "type" "clz")]
4667)
43e9d192
IB		 4668
4669(define_expand "ffs<mode>2"
4670 [(match_operand:GPI 0 "register_operand")
4671 (match_operand:GPI 1 "register_operand")]
4672 ""
4673 {
4674 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4675 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4676
4677 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4678 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
cd5660ab 4679 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
43e9d192
IB		 4680 DONE;
4681 }
4682)
4683
9d5019a1
AP		 4684;; Pop count be done via the "CNT" instruction in AdvSIMD.
4685;;
4686;; MOV v.1d, x0
4687;; CNT v1.8b, v.8b
4688;; ADDV b2, v1.8b
4689;; MOV w0, v2.b[0]
4690
4691(define_expand "popcount<mode>2"
4692 [(match_operand:GPI 0 "register_operand")
4693 (match_operand:GPI 1 "register_operand")]
4694 "TARGET_SIMD"
4695{
4696 rtx v = gen_reg_rtx (V8QImode);
4697 rtx v1 = gen_reg_rtx (V8QImode);
9d5019a1
AP		 4698 rtx in = operands[1];
4699 rtx out = operands[0];
4700 if(<MODE>mode == SImode)
4701 {
4702 rtx tmp;
4703 tmp = gen_reg_rtx (DImode);
4704 /* If we have SImode, zero extend to DImode, pop count does
4705 not change if we have extra zeros. */
4706 emit_insn (gen_zero_extendsidi2 (tmp, in));
4707 in = tmp;
4708 }
4709 emit_move_insn (v, gen_lowpart (V8QImode, in));
4710 emit_insn (gen_popcountv8qi2 (v1, v));
9921bbf9 4711 emit_insn (gen_aarch64_zero_extend<mode>_reduc_plus_v8qi (out, v1));
9d5019a1
AP		 4712 DONE;
4713})
4714
43e9d192
IB		 4715(define_insn "clrsb<mode>2"
4716 [(set (match_operand:GPI 0 "register_operand" "=r")
dfc98d99 4717 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192
IB		 4718 ""
4719 "cls\\t%<w>0, %<w>1"
99e6af15
JG		 4720 [(set_attr "type" "clz")]
4721)
43e9d192
IB		 4722
4723(define_insn "rbit<mode>2"
4724 [(set (match_operand:GPI 0 "register_operand" "=r")
4725 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4726 ""
4727 "rbit\\t%<w>0, %<w>1"
99e6af15
JG		 4728 [(set_attr "type" "rbit")]
4729)
43e9d192 4730
e8536e2b
KT		 4731;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4732;; it is unlikely to fold with any other operation, so keep this as a CTZ
4733;; expression and split after reload to enable scheduling them apart if
4734;; needed.
4735
4736(define_insn_and_split "ctz<mode>2"
4737 [(set (match_operand:GPI 0 "register_operand" "=r")
4738 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4739 ""
e8536e2b
KT		 4740 "#"
4741 "reload_completed"
4742 [(const_int 0)]
4743 "
4744 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4745 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4746 DONE;
4747")
43e9d192 4748
2b8568fe
KT		 4749(define_insn "*and<mode>_compare0"
4750 [(set (reg:CC_NZ CC_REGNUM)
4751 (compare:CC_NZ
4752 (match_operand:SHORT 0 "register_operand" "r")
4753 (const_int 0)))]
4754 ""
4755 "tst\\t%<w>0, <short_mask>"
4756 [(set_attr "type" "alus_imm")]
4757)
4758
e7ba492a 4759(define_insn "*ands<GPI:mode>_compare0"
b06335f9
KT		 4760 [(set (reg:CC_NZ CC_REGNUM)
4761 (compare:CC_NZ
4762 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4763 (const_int 0)))
4764 (set (match_operand:GPI 0 "register_operand" "=r")
4765 (zero_extend:GPI (match_dup 1)))]
4766 ""
4767 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4768 [(set_attr "type" "alus_imm")]
4769)
4770
43e9d192 4771(define_insn "*and<mode>3nr_compare0"
d803a491
IB		 4772 [(set (reg:CC_NZ CC_REGNUM)
4773 (compare:CC_NZ
43e9d192
IB		 4774 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4775 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4776 (const_int 0)))]
4777 ""
954224d7
WD		 4778 "@
4779 tst\\t%<w>0, %<w>1
4780 tst\\t%<w>0, %1"
1d70b920 4781 [(set_attr "type" "logics_reg,logics_imm")]
99e6af15 4782)
43e9d192 4783
2c2789d5
KT		 4784(define_split
4785 [(set (reg:CC_NZ CC_REGNUM)
4786 (compare:CC_NZ
4787 (and:GPI (match_operand:GPI 0 "register_operand")
4788 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4789 (const_int 0)))
4790 (clobber (match_operand:SI 2 "register_operand"))]
4791 ""
4792 [(set (match_dup 2) (match_dup 1))
4793 (set (reg:CC_NZ CC_REGNUM)
4794 (compare:CC_NZ
4795 (and:GPI (match_dup 0)
4796 (match_dup 2))
4797 (const_int 0)))]
4798)
4799
7325d85a
KT		 4800(define_insn "*and<mode>3nr_compare0_zextract"
4801 [(set (reg:CC_NZ CC_REGNUM)
4802 (compare:CC_NZ
4803 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4804 (match_operand:GPI 1 "const_int_operand" "n")
4805 (match_operand:GPI 2 "const_int_operand" "n"))
4806 (const_int 0)))]
4807 "INTVAL (operands[1]) > 0
4808 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4809 <= GET_MODE_BITSIZE (<MODE>mode))
4810 && aarch64_bitmask_imm (
4811 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4812 operands[2])),
4813 <MODE>mode)"
4814 {
4815 operands[1]
4816 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4817 return "tst\\t%<w>0, %1";
4818 }
4819 [(set_attr "type" "logics_shift_imm")]
4820)
4821
43e9d192 4822(define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
d803a491
IB		 4823 [(set (reg:CC_NZ CC_REGNUM)
4824 (compare:CC_NZ
43e9d192
IB		 4825 (and:GPI (SHIFT:GPI
4826 (match_operand:GPI 0 "register_operand" "r")
4827 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4828 (match_operand:GPI 2 "register_operand" "r"))
4829 (const_int 0)))]
4830 ""
4831 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
99e6af15
JG		 4832 [(set_attr "type" "logics_shift_imm")]
4833)
43e9d192 4834
2c2789d5
KT		 4835(define_split
4836 [(set (reg:CC_NZ CC_REGNUM)
4837 (compare:CC_NZ
4838 (and:GPI (SHIFT:GPI
4839 (match_operand:GPI 0 "register_operand")
4840 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4841 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4842 (const_int 0)))
4843 (clobber (match_operand:SI 3 "register_operand"))]
4844 ""
4845 [(set (match_dup 3) (match_dup 2))
4846 (set (reg:CC_NZ CC_REGNUM)
4847 (compare:CC_NZ
4848 (and:GPI (SHIFT:GPI
4849 (match_dup 0)
4850 (match_dup 1))
4851 (match_dup 3))
4852 (const_int 0)))]
4853)
4854
43e9d192
IB		 4855;; -------------------------------------------------------------------
4856;; Shifts
4857;; -------------------------------------------------------------------
4858
4859(define_expand "<optab><mode>3"
4860 [(set (match_operand:GPI 0 "register_operand")
4861 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 4862 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 4863 ""
4864 {
4865 if (CONST_INT_P (operands[2]))
4866 {
4867 operands[2] = GEN_INT (INTVAL (operands[2])
4868 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4869
4870 if (operands[2] == const0_rtx)
4871 {
4872 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4873 DONE;
4874 }
4875 }
4876 }
4877)
4878
4879(define_expand "ashl<mode>3"
4880 [(set (match_operand:SHORT 0 "register_operand")
4881 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4bc72f35 4882 (match_operand:QI 2 "const_int_operand")))]
43e9d192
IB		 4883 ""
4884 {
4bc72f35 4885 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
43e9d192 4886
4bc72f35
KT		 4887 if (operands[2] == const0_rtx)
4888 {
4889 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4890 DONE;
43e9d192
IB		 4891 }
4892 }
4893)
4894
4895(define_expand "rotr<mode>3"
4896 [(set (match_operand:GPI 0 "register_operand")
4897 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 4898 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 4899 ""
4900 {
4901 if (CONST_INT_P (operands[2]))
4902 {
4903 operands[2] = GEN_INT (INTVAL (operands[2])
4904 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4905
4906 if (operands[2] == const0_rtx)
4907 {
4908 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4909 DONE;
4910 }
4911 }
4912 }
4913)
4914
4915(define_expand "rotl<mode>3"
4916 [(set (match_operand:GPI 0 "register_operand")
4917 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 4918 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 4919 ""
4920 {
4921 /* (SZ - cnt) % SZ == -cnt % SZ */
4922 if (CONST_INT_P (operands[2]))
4923 {
4924 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4925 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4926 if (operands[2] == const0_rtx)
4927 {
4928 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4929 DONE;
4930 }
4931 }
4932 else
4933 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4934 NULL_RTX, 1);
4935 }
4936)
4937
7813b280
KT		 4938;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4939;; they truncate the shift/rotate amount by the size of the registers they
4940;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
4941;; such redundant masking instructions. GCC can do that automatically when
4942;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4943;; because some of the SISD shift alternatives don't perform this truncations.
4944;; So this pattern exists to catch such cases.
4945
4946(define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4947 [(set (match_operand:GPI 0 "register_operand" "=r")
4948 (SHIFT:GPI
4949 (match_operand:GPI 1 "register_operand" "r")
4950 (match_operator 4 "subreg_lowpart_operator"
4951 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4952 (match_operand 3 "const_int_operand" "n"))])))]
4953 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4954 "<shift>\t%<w>0, %<w>1, %<w>2"
4955 [(set_attr "type" "shift_reg")]
4956)
4957
cdeba3e0 4958(define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
7813b280
KT		 4959 [(set (match_operand:GPI 0 "register_operand" "=&r")
4960 (SHIFT:GPI
4961 (match_operand:GPI 1 "register_operand" "r")
4962 (match_operator 4 "subreg_lowpart_operator"
4963 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4964 (match_operand 3 "const_int_operand" "n")))])))]
4965 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4966 "#"
4967 "&& true"
4968 [(const_int 0)]
4969 {
4970 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
cdeba3e0 4971 : lowpart_subreg (SImode, operands[0], <MODE>mode));
7813b280
KT		 4972 emit_insn (gen_negsi2 (tmp, operands[2]));
4973
4974 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4975 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4976 SUBREG_BYTE (operands[4]));
4977 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4978 DONE;
4979 }
4980)
4981
cdeba3e0 4982(define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
7813b280
KT		 4983 [(set (match_operand:GPI 0 "register_operand" "=&r")
4984 (ashift:GPI
4985 (match_operand:GPI 1 "register_operand" "r")
4986 (minus:QI (match_operand 2 "const_int_operand" "n")
4987 (match_operator 5 "subreg_lowpart_operator"
4988 [(and:SI (match_operand:SI 3 "register_operand" "r")
4989 (match_operand 4 "const_int_operand" "n"))]))))]
4990 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4991 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4992 "#"
4993 "&& true"
4994 [(const_int 0)]
4995 {
4996 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4997 : operands[0]);
4998
4999 emit_insn (gen_negsi2 (tmp, operands[3]));
5000
5001 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5002 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5003 SUBREG_BYTE (operands[5]));
5004
5005 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5006 DONE;
5007 }
5008)
5009
5010(define_insn "*aarch64_<optab>_reg_di3_mask2"
5011 [(set (match_operand:DI 0 "register_operand" "=r")
5012 (SHIFT:DI
5013 (match_operand:DI 1 "register_operand" "r")
5014 (match_operator 4 "subreg_lowpart_operator"
5015 [(and:SI (match_operand:SI 2 "register_operand" "r")
cdeba3e0
JJ		 5016 (match_operand 3 "const_int_operand" "n"))])))]
5017 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
7813b280
KT		 5018{
5019 rtx xop[3];
5020 xop[0] = operands[0];
5021 xop[1] = operands[1];
5022 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5023 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5024 return "";
5025}
5026 [(set_attr "type" "shift_reg")]
5027)
5028
cdeba3e0 5029(define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
59abe903
MC		 5030 [(set (match_operand:GPI 0 "register_operand" "=&r")
5031 (ASHIFT:GPI
5032 (match_operand:GPI 1 "register_operand" "r")
5033 (minus:QI (match_operand 2 "const_int_operand" "n")
5034 (match_operand:QI 3 "register_operand" "r"))))]
5035 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5036 "#"
5037 "&& true"
5038 [(const_int 0)]
5039 {
5040 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5041
5042 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5043 : gen_lowpart (SImode, operands[0]));
5044
5045 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5046
5047 rtx and_op = gen_rtx_AND (SImode, tmp,
5048 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5049
5050 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5051
5052 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5053 DONE;
5054 }
5055 [(set_attr "length" "8")]
5056)
5057
2e100703
VP		 5058;; Logical left shift using SISD or Integer instruction
5059(define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3362d764
WD		 5060 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5061 (ashift:GPI
5062 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5063 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
2e100703
VP		 5064 ""
5065 "@
3362d764 5066 lsl\t%<w>0, %<w>1, %2
d99dacc9 5067 lsl\t%<w>0, %<w>1, %<w>2
2e100703 5068 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
d99dacc9 5069 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
488461d8
MM		 5070 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5071 (set_attr "arch" "*,*,simd,simd")]
2e100703
VP		 5072)
5073
5074;; Logical right shift using SISD or Integer instruction
5075(define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3362d764
WD		 5076 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5077 (lshiftrt:GPI
5078 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
0603375c
KT		 5079 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5080 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5081 ""
5082 "@
3362d764 5083 lsr\t%<w>0, %<w>1, %2
ad7b853a 5084 lsr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5085 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5086 #
ad7b853a 5087 #"
488461d8
MM		 5088 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5089 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5090)
5091
5092(define_split
5093 [(set (match_operand:DI 0 "aarch64_simd_register")
5094 (lshiftrt:DI
5095 (match_operand:DI 1 "aarch64_simd_register")
5096 (match_operand:QI 2 "aarch64_simd_register")))]
5097 "TARGET_SIMD && reload_completed"
498b13e2 5098 [(set (match_dup 3)
2e100703
VP		 5099 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5100 (set (match_dup 0)
498b13e2
KT		 5101 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5102 {
5103 operands[3] = gen_lowpart (QImode, operands[0]);
5104 }
2e100703
VP		 5105)
5106
5107(define_split
5108 [(set (match_operand:SI 0 "aarch64_simd_register")
5109 (lshiftrt:SI
5110 (match_operand:SI 1 "aarch64_simd_register")
5111 (match_operand:QI 2 "aarch64_simd_register")))]
5112 "TARGET_SIMD && reload_completed"
498b13e2 5113 [(set (match_dup 3)
2e100703
VP		 5114 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5115 (set (match_dup 0)
498b13e2
KT		 5116 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5117 {
5118 operands[3] = gen_lowpart (QImode, operands[0]);
5119 }
2e100703
VP		 5120)
5121
5122;; Arithmetic right shift using SISD or Integer instruction
5123(define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3362d764 5124 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
0603375c
KT		 5125 (ashiftrt:GPI
5126 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5127 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5128 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5129 ""
5130 "@
3362d764 5131 asr\t%<w>0, %<w>1, %2
ad7b853a 5132 asr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5133 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5134 #
ad7b853a 5135 #"
488461d8
MM		 5136 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5137 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5138)
5139
5140(define_split
5141 [(set (match_operand:DI 0 "aarch64_simd_register")
5142 (ashiftrt:DI
5143 (match_operand:DI 1 "aarch64_simd_register")
5144 (match_operand:QI 2 "aarch64_simd_register")))]
5145 "TARGET_SIMD && reload_completed"
21a7b5e1 5146 [(set (match_dup 3)
2e100703
VP		 5147 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5148 (set (match_dup 0)
21a7b5e1
MS		 5149 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5150{
5151 operands[3] = gen_lowpart (QImode, operands[0]);
5152}
2e100703
VP		 5153)
5154
5155(define_split
5156 [(set (match_operand:SI 0 "aarch64_simd_register")
5157 (ashiftrt:SI
5158 (match_operand:SI 1 "aarch64_simd_register")
5159 (match_operand:QI 2 "aarch64_simd_register")))]
5160 "TARGET_SIMD && reload_completed"
21a7b5e1 5161 [(set (match_dup 3)
2e100703
VP		 5162 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5163 (set (match_dup 0)
21a7b5e1
MS		 5164 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5165{
5166 operands[3] = gen_lowpart (QImode, operands[0]);
5167}
2e100703
VP		 5168)
5169
5170(define_insn "*aarch64_sisd_ushl"
5171 [(set (match_operand:DI 0 "register_operand" "=w")
5172 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5173 (match_operand:QI 2 "register_operand" "w")]
5174 UNSPEC_SISD_USHL))]
5175 "TARGET_SIMD"
5176 "ushl\t%d0, %d1, %d2"
488461d8 5177 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5178)
5179
5180(define_insn "*aarch64_ushl_2s"
5181 [(set (match_operand:SI 0 "register_operand" "=w")
5182 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5183 (match_operand:QI 2 "register_operand" "w")]
5184 UNSPEC_USHL_2S))]
5185 "TARGET_SIMD"
5186 "ushl\t%0.2s, %1.2s, %2.2s"
488461d8 5187 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5188)
5189
5190(define_insn "*aarch64_sisd_sshl"
5191 [(set (match_operand:DI 0 "register_operand" "=w")
5192 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5193 (match_operand:QI 2 "register_operand" "w")]
5194 UNSPEC_SISD_SSHL))]
5195 "TARGET_SIMD"
5196 "sshl\t%d0, %d1, %d2"
488461d8 5197 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5198)
5199
5200(define_insn "*aarch64_sshl_2s"
5201 [(set (match_operand:SI 0 "register_operand" "=w")
5202 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5203 (match_operand:QI 2 "register_operand" "w")]
5204 UNSPEC_SSHL_2S))]
5205 "TARGET_SIMD"
5206 "sshl\t%0.2s, %1.2s, %2.2s"
488461d8 5207 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5208)
5209
5210(define_insn "*aarch64_sisd_neg_qi"
5211 [(set (match_operand:QI 0 "register_operand" "=w")
5212 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5213 UNSPEC_SISD_NEG))]
5214 "TARGET_SIMD"
5215 "neg\t%d0, %d1"
488461d8 5216 [(set_attr "type" "neon_neg")]
2e100703
VP		 5217)
5218
5219;; Rotate right
5220(define_insn "*ror<mode>3_insn"
b81f1ee3
JG		 5221 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5222 (rotatert:GPI
5223 (match_operand:GPI 1 "register_operand" "r,r")
3362d764 5224 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
43e9d192 5225 ""
3362d764
WD		 5226 "@
5227 ror\\t%<w>0, %<w>1, %2
5228 ror\\t%<w>0, %<w>1, %<w>2"
5229 [(set_attr "type" "rotate_imm,shift_reg")]
43e9d192
IB		 5230)
5231
d0b6bb1b
IB		 5232;; zero_extend version of above
5233(define_insn "*<optab>si3_insn_uxtw"
3362d764 5234 [(set (match_operand:DI 0 "register_operand" "=r,r")
d0b6bb1b 5235 (zero_extend:DI (SHIFT:SI
3362d764
WD		 5236 (match_operand:SI 1 "register_operand" "r,r")
5237 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
d0b6bb1b 5238 ""
3362d764
WD		 5239 "@
5240 <shift>\\t%w0, %w1, %2
5241 <shift>\\t%w0, %w1, %w2"
94f7a25e 5242 [(set_attr "type" "bfx,shift_reg")]
d0b6bb1b
IB		 5243)
5244
43e9d192
IB		 5245(define_insn "*<optab><mode>3_insn"
5246 [(set (match_operand:SHORT 0 "register_operand" "=r")
5247 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5248 (match_operand 2 "const_int_operand" "n")))]
5249 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5250{
5251 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5252 return "<bfshift>\t%w0, %w1, %2, %3";
5253}
94f7a25e 5254 [(set_attr "type" "bfx")]
43e9d192
IB		 5255)
5256
bd83ff2c
IB		 5257(define_insn "*extr<mode>5_insn"
5258 [(set (match_operand:GPI 0 "register_operand" "=r")
5259 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5260 (match_operand 3 "const_int_operand" "n"))
5261 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5262 (match_operand 4 "const_int_operand" "n"))))]
5263 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5264 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5265 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5266 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5267)
5268
fb0cb7fa
KT		 5269;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5270;; so we have to match both orderings.
5271(define_insn "*extr<mode>5_insn_alt"
5272 [(set (match_operand:GPI 0 "register_operand" "=r")
5273 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5274 (match_operand 4 "const_int_operand" "n"))
5275 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5276 (match_operand 3 "const_int_operand" "n"))))]
5277 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5278 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5279 == GET_MODE_BITSIZE (<MODE>mode))"
5280 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5281 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5282)
5283
bd83ff2c
IB		 5284;; zero_extend version of the above
5285(define_insn "*extrsi5_insn_uxtw"
5286 [(set (match_operand:DI 0 "register_operand" "=r")
5287 (zero_extend:DI
5288 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5289 (match_operand 3 "const_int_operand" "n"))
5290 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5291 (match_operand 4 "const_int_operand" "n")))))]
5292 "UINTVAL (operands[3]) < 32 &&
5293 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5294 "extr\\t%w0, %w1, %w2, %4"
3362d764 5295 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5296)
5297
fb0cb7fa
KT		 5298(define_insn "*extrsi5_insn_uxtw_alt"
5299 [(set (match_operand:DI 0 "register_operand" "=r")
5300 (zero_extend:DI
5301 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5302 (match_operand 4 "const_int_operand" "n"))
5303 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5304 (match_operand 3 "const_int_operand" "n")))))]
5305 "UINTVAL (operands[3]) < 32 &&
5306 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5307 "extr\\t%w0, %w1, %w2, %4"
3362d764 5308 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5309)
5310
5977a10d
IB		 5311(define_insn "*ror<mode>3_insn"
5312 [(set (match_operand:GPI 0 "register_operand" "=r")
5313 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5314 (match_operand 2 "const_int_operand" "n")))]
5315 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5316{
5317 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5318 return "ror\\t%<w>0, %<w>1, %3";
5319}
b81f1ee3 5320 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5321)
5322
5323;; zero_extend version of the above
5324(define_insn "*rorsi3_insn_uxtw"
5325 [(set (match_operand:DI 0 "register_operand" "=r")
5326 (zero_extend:DI
5327 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5328 (match_operand 2 "const_int_operand" "n"))))]
5329 "UINTVAL (operands[2]) < 32"
5330{
5331 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5332 return "ror\\t%w0, %w1, %3";
5333}
b81f1ee3 5334 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5335)
5336
43e9d192
IB		 5337(define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5338 [(set (match_operand:GPI 0 "register_operand" "=r")
5339 (ANY_EXTEND:GPI
5340 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5341 (match_operand 2 "const_int_operand" "n"))))]
5342 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5343{
5344 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5345 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5346}
94f7a25e 5347 [(set_attr "type" "bfx")]
43e9d192
IB		 5348)
5349
5350(define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5351 [(set (match_operand:GPI 0 "register_operand" "=r")
5352 (zero_extend:GPI
5353 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5354 (match_operand 2 "const_int_operand" "n"))))]
5355 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5356{
5357 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5358 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5359}
94f7a25e 5360 [(set_attr "type" "bfx")]
43e9d192
IB		 5361)
5362
5363(define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5364 [(set (match_operand:GPI 0 "register_operand" "=r")
5365 (sign_extend:GPI
5366 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5367 (match_operand 2 "const_int_operand" "n"))))]
5368 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5369{
5370 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5371 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5372}
94f7a25e 5373 [(set_attr "type" "bfx")]
43e9d192
IB		 5374)
5375
5376;; -------------------------------------------------------------------
5377;; Bitfields
5378;; -------------------------------------------------------------------
5379
5380(define_expand "<optab>"
1bbffb87 5381 [(set (match_operand:DI 0 "register_operand")
e27bfda2
KT		 5382 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5383 (match_operand 2
5384 "aarch64_simd_shift_imm_offset_di")
5385 (match_operand 3 "aarch64_simd_shift_imm_di")))]
43e9d192 5386 ""
e27bfda2
KT		 5387 {
5388 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5389 1, GET_MODE_BITSIZE (DImode) - 1))
5390 FAIL;
5391 }
43e9d192
IB		 5392)
5393
e27bfda2 5394
43e9d192
IB		 5395(define_insn "*<optab><mode>"
5396 [(set (match_operand:GPI 0 "register_operand" "=r")
5397 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
e27bfda2
KT		 5398 (match_operand 2
5399 "aarch64_simd_shift_imm_offset_<mode>" "n")
5400 (match_operand 3
5401 "aarch64_simd_shift_imm_<mode>" "n")))]
5402 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5403 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
43e9d192 5404 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
94f7a25e 5405 [(set_attr "type" "bfx")]
65f9e789
IB		 5406)
5407
bcb036c5
KT		 5408;; When the bit position and width add up to 32 we can use a W-reg LSR
5409;; instruction taking advantage of the implicit zero-extension of the X-reg.
5410(define_split
5411 [(set (match_operand:DI 0 "register_operand")
5412 (zero_extract:DI (match_operand:DI 1 "register_operand")
5413 (match_operand 2
5414 "aarch64_simd_shift_imm_offset_di")
5415 (match_operand 3
5416 "aarch64_simd_shift_imm_di")))]
5417 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5418 GET_MODE_BITSIZE (DImode) - 1)
5419 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5420 == GET_MODE_BITSIZE (SImode)"
5421 [(set (match_dup 0)
5422 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5423 {
5424 operands[4] = gen_lowpart (SImode, operands[1]);
5425 }
5426)
5427
65f9e789
IB		 5428;; Bitfield Insert (insv)
5429(define_expand "insv<mode>"
5430 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5431 (match_operand 1 "const_int_operand")
5432 (match_operand 2 "const_int_operand"))
5433 (match_operand:GPI 3 "general_operand"))]
5434 ""
5435{
5436 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5437 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5438 rtx value = operands[3];
5439
5440 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5441 FAIL;
5442
5443 if (CONST_INT_P (value))
5444 {
5445 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5446
5447 /* Prefer AND/OR for inserting all zeros or all ones. */
5448 if ((UINTVAL (value) & mask) == 0
5449 || (UINTVAL (value) & mask) == mask)
5450 FAIL;
5451
5452 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5453 if (width == 16 && (pos % 16) == 0)
5454 DONE;
5455 }
5456 operands[3] = force_reg (<MODE>mode, value);
5457})
5458
5459(define_insn "*insv_reg<mode>"
5460 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5461 (match_operand 1 "const_int_operand" "n")
5462 (match_operand 2 "const_int_operand" "n"))
5463 (match_operand:GPI 3 "register_operand" "r"))]
5464 "!(UINTVAL (operands[1]) == 0
5465 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5466 > GET_MODE_BITSIZE (<MODE>mode)))"
5467 "bfi\\t%<w>0, %<w>3, %2, %1"
99e6af15 5468 [(set_attr "type" "bfm")]
2879bb2b
IB		 5469)
5470
e4f019d3
RL		 5471(define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5472 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5473 (match_operand 1 "const_int_operand" "n")
5474 (match_operand 2 "const_int_operand" "n"))
5475 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5476 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5477 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5478 [(set_attr "type" "bfm")]
5479)
5480
6a0d3939
SE		 5481;; Match a bfi instruction where the shift of OP3 means that we are
5482;; actually copying the least significant bits of OP3 into OP0 by way
5483;; of the AND masks and the IOR instruction. A similar instruction
5484;; with the two parts of the IOR swapped around was never triggered
5485;; in a bootstrap build and test of GCC so it was not included.
5486
5487(define_insn "*aarch64_bfi<GPI:mode>5_shift"
5488 [(set (match_operand:GPI 0 "register_operand" "=r")
5489 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5490 (match_operand:GPI 2 "const_int_operand" "n"))
5491 (and:GPI (ashift:GPI
5492 (match_operand:GPI 3 "register_operand" "r")
5493 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5494 (match_operand:GPI 5 "const_int_operand" "n"))))]
5495 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5496 UINTVAL (operands[4]),
5497 UINTVAL(operands[5]))"
5498 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5499 [(set_attr "type" "bfm")]
5500)
5501
5502(define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5503 [(set (match_operand:GPI 0 "register_operand" "=r")
5504 (ior:GPI (and:GPI (ashift:GPI
5505 (match_operand:GPI 1 "register_operand" "r")
5506 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5507 (match_operand:GPI 3 "const_int_operand" "n"))
5508 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5509 (match_operand:GPI 5 "const_int_operand" "n"))))]
5510 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5511 UINTVAL (operands[2]),
5512 UINTVAL(operands[3]))"
5513 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5514 [(set_attr "type" "bfm")]
5515)
5516
5517;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5518;; the shift is large enough to remove the need for an AND instruction.
5519
5520(define_insn "*aarch64_bfi<GPI:mode>4_noand"
5521 [(set (match_operand:GPI 0 "register_operand" "=r")
5522 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5523 (match_operand:GPI 2 "const_int_operand" "n"))
5524 (ashift:GPI
5525 (match_operand:GPI 3 "register_operand" "r")
5526 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5527 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5528 UINTVAL (operands[4]),
5529 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5530{
5531 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5532 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5533}
5534 [(set_attr "type" "bfm")]
5535)
5536
5537(define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5538 [(set (match_operand:GPI 0 "register_operand" "=r")
5539 (ior:GPI (ashift:GPI
5540 (match_operand:GPI 1 "register_operand" "r")
5541 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5542 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5543 (match_operand:GPI 4 "const_int_operand" "n"))))]
5544 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5545 UINTVAL (operands[2]),
5546 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5547{
5548 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5549 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5550}
5551 [(set_attr "type" "bfm")]
5552)
5553
5554;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
f0a8f6bc
SE		 5555;; copying the least significant bits of OP3 to OP0. We need two versions
5556;; of the instruction to handle different checks on the constant values.
6a0d3939
SE		 5557
5558(define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5559 [(set (match_operand:GPI 0 "register_operand" "=r")
5560 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5561 (match_operand:GPI 2 "const_int_operand" "n"))
5562 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5563 (match_operand:GPI 4 "const_int_operand" "n"))))]
5564 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5565 UINTVAL (operands[4]))"
5566 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5567 [(set_attr "type" "bfm")]
5568)
5569
f0a8f6bc
SE		 5570(define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5571 [(set (match_operand:GPI 0 "register_operand" "=r")
5572 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5573 (match_operand:GPI 4 "const_int_operand" "n"))
5574 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5575 (match_operand:GPI 2 "const_int_operand" "n"))))]
5576 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5577 UINTVAL (operands[4]))"
5578 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5579 [(set_attr "type" "bfm")]
5580)
5581
2879bb2b
IB		 5582(define_insn "*extr_insv_lower_reg<mode>"
5583 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5584 (match_operand 1 "const_int_operand" "n")
5585 (const_int 0))
98fa5ef3 5586 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
2879bb2b
IB		 5587 (match_dup 1)
5588 (match_operand 3 "const_int_operand" "n")))]
5589 "!(UINTVAL (operands[1]) == 0
5590 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5591 > GET_MODE_BITSIZE (<MODE>mode)))"
5592 "bfxil\\t%<w>0, %<w>2, %3, %1"
99e6af15 5593 [(set_attr "type" "bfm")]
43e9d192
IB		 5594)
5595
5596(define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5597 [(set (match_operand:GPI 0 "register_operand" "=r")
5598 (ashift:GPI (ANY_EXTEND:GPI
5599 (match_operand:ALLX 1 "register_operand" "r"))
5600 (match_operand 2 "const_int_operand" "n")))]
5601 "UINTVAL (operands[2]) < <GPI:sizen>"
5602{
5603 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5604 ? GEN_INT (<ALLX:sizen>)
5605 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5606 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5607}
94f7a25e 5608 [(set_attr "type" "bfx")]
43e9d192
IB		 5609)
5610
5611;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5612
5613(define_insn "*andim_ashift<mode>_bfiz"
5614 [(set (match_operand:GPI 0 "register_operand" "=r")
5615 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5616 (match_operand 2 "const_int_operand" "n"))
5617 (match_operand 3 "const_int_operand" "n")))]
8c83f71d 5618 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
43e9d192 5619 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
94f7a25e 5620 [(set_attr "type" "bfx")]
43e9d192
IB		 5621)
5622
58c2ad42
LM		 5623;; Match sbfiz pattern in a shift left + shift right operation.
5624
5625(define_insn "*ashift<mode>_extv_bfiz"
5626 [(set (match_operand:GPI 0 "register_operand" "=r")
5627 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5628 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5629 (const_int 0))
5630 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5631 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5632 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5633 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5634 [(set_attr "type" "bfx")]
5635)
5636
b65a1eb3
RS		 5637(define_insn "*ashiftsi_extvdi_bfiz"
5638 [(set (match_operand:SI 0 "register_operand" "=r")
5639 (ashift:SI
5640 (match_operator:SI 4 "subreg_lowpart_operator"
5641 [(sign_extract:DI
5642 (match_operand:DI 1 "register_operand" "r")
5643 (match_operand 2 "aarch64_simd_shift_imm_offset_si")
5644 (const_int 0))])
5645 (match_operand 3 "aarch64_simd_shift_imm_si")))]
5646 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5647 1, GET_MODE_BITSIZE (SImode) - 1)"
5648 "sbfiz\\t%w0, %w1, %3, %2"
5649 [(set_attr "type" "bfx")]
5650)
5651
680153bd
KT		 5652;; When the bit position and width of the equivalent extraction add up to 32
5653;; we can use a W-reg LSL instruction taking advantage of the implicit
5654;; zero-extension of the X-reg.
5655(define_split
5656 [(set (match_operand:DI 0 "register_operand")
5657 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5658 (match_operand 2 "const_int_operand"))
5659 (match_operand 3 "const_int_operand")))]
5660 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5661 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5662 == GET_MODE_BITSIZE (SImode)"
5663 [(set (match_dup 0)
5664 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5665 {
5666 operands[4] = gen_lowpart (SImode, operands[1]);
5667 }
5668)
5669
43e9d192
IB		 5670(define_insn "bswap<mode>2"
5671 [(set (match_operand:GPI 0 "register_operand" "=r")
5672 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5673 ""
5674 "rev\\t%<w>0, %<w>1"
99e6af15 5675 [(set_attr "type" "rev")]
43e9d192
IB		 5676)
5677
c16c63bb
IB		 5678(define_insn "bswaphi2"
5679 [(set (match_operand:HI 0 "register_operand" "=r")
5680 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5681 ""
5682 "rev16\\t%w0, %w1"
99e6af15 5683 [(set_attr "type" "rev")]
c16c63bb
IB		 5684)
5685
231c52ae
ST		 5686(define_insn "*aarch64_bfxil<mode>"
5687 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5688 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5689 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5690 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5691 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5692 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5693 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5694 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5695 {
5696 switch (which_alternative)
5697 {
5698 case 0:
5699 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5700 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5701 case 1:
5702 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5703 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5704 default:
5705 gcc_unreachable ();
5706 }
5707 }
5708 [(set_attr "type" "bfm")]
5709)
5710
2dc09f66
ST		 5711; Zero-extended version of above (aarch64_bfxil)
5712(define_insn "*aarch64_bfxilsi_uxtw"
5713 [(set (match_operand:DI 0 "register_operand" "=r,r")
5714 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5715 "r,0")
5716 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5717 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5718 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5719 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5720 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5721 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5722 {
5723 switch (which_alternative)
5724 {
5725 case 0:
5726 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5727 return "bfxil\\t%0, %1, 0, %3";
5728 case 1:
5729 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5730 return "bfxil\\t%0, %2, 0, %3";
5731 default:
5732 gcc_unreachable ();
5733 }
5734 }
5735 [(set_attr "type" "bfm")]
5736)
5737
f7d5cf8d
KT		 5738;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5739;; operations within an IOR/AND RTX, therefore we have two patterns matching
5740;; each valid permutation.
5741
5742(define_insn "rev16<mode>2"
5743 [(set (match_operand:GPI 0 "register_operand" "=r")
5744 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5745 (const_int 8))
5746 (match_operand:GPI 3 "const_int_operand" "n"))
5747 (and:GPI (lshiftrt:GPI (match_dup 1)
5748 (const_int 8))
5749 (match_operand:GPI 2 "const_int_operand" "n"))))]
5750 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5751 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5752 "rev16\\t%<w>0, %<w>1"
5753 [(set_attr "type" "rev")]
5754)
5755
5756(define_insn "rev16<mode>2_alt"
5757 [(set (match_operand:GPI 0 "register_operand" "=r")
5758 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5759 (const_int 8))
5760 (match_operand:GPI 2 "const_int_operand" "n"))
5761 (and:GPI (ashift:GPI (match_dup 1)
5762 (const_int 8))
5763 (match_operand:GPI 3 "const_int_operand" "n"))))]
5764 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5765 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5766 "rev16\\t%<w>0, %<w>1"
5767 [(set_attr "type" "rev")]
5768)
5769
d0b6bb1b
IB		 5770;; zero_extend version of above
5771(define_insn "*bswapsi2_uxtw"
5772 [(set (match_operand:DI 0 "register_operand" "=r")
5773 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5774 ""
5775 "rev\\t%w0, %w1"
99e6af15 5776 [(set_attr "type" "rev")]
d0b6bb1b
IB		 5777)
5778
43e9d192
IB		 5779;; -------------------------------------------------------------------
5780;; Floating-point intrinsics
5781;; -------------------------------------------------------------------
5782
42fc9a7f 5783;; frint floating-point round to integral standard patterns.
74dc11ed 5784;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
43e9d192 5785
42fc9a7f 5786(define_insn "<frint_pattern><mode>2"
d7f33f07
JW		 5787 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5788 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
42fc9a7f 5789 FRINT))]
43e9d192 5790 "TARGET_FLOAT"
42fc9a7f 5791 "frint<frint_suffix>\\t%<s>0, %<s>1"
d7f33f07 5792 [(set_attr "type" "f_rint<stype>")]
43e9d192
IB		 5793)
5794
42fc9a7f
JG		 5795;; frcvt floating-point round to integer and convert standard patterns.
5796;; Expands to lbtrunc, lceil, lfloor, lround.
d7f33f07 5797(define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
43e9d192 5798 [(set (match_operand:GPI 0 "register_operand" "=r")
d7f33f07
JW		 5799 (FIXUORS:GPI
5800 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5801 FCVT)))]
43e9d192 5802 "TARGET_FLOAT"
d7f33f07 5803 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
99e6af15 5804 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 5805)
5806
39252973
KT		 5807(define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5808 [(set (match_operand:GPI 0 "register_operand" "=r")
5809 (FIXUORS:GPI
5810 (mult:GPF
5811 (match_operand:GPF 1 "register_operand" "w")
5812 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5813 "TARGET_FLOAT
5814 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5815 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5816 {
5817 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5818 char buf[64];
5819 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5820 output_asm_insn (buf, operands);
5821 return "";
5822 }
5823 [(set_attr "type" "f_cvtf2i")]
5824)
5825
d6e6e8b6
WD		 5826;; fma - expand fma into patterns with the accumulator operand first since
5827;; reusing the accumulator results in better register allocation.
5828;; The register allocator considers copy preferences in operand order,
5829;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5830
5831(define_expand "fma<mode>4"
5832 [(set (match_operand:GPF_F16 0 "register_operand")
5833 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5834 (match_operand:GPF_F16 2 "register_operand")
5835 (match_operand:GPF_F16 3 "register_operand")))]
5836 "TARGET_FLOAT"
5837)
43e9d192 5838
d6e6e8b6 5839(define_insn "*aarch64_fma<mode>4"
9a594ad6 5840 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
d6e6e8b6
WD		 5841 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5842 (match_operand:GPF_F16 3 "register_operand" "w")
5843 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 5844 "TARGET_FLOAT"
d6e6e8b6 5845 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 5846 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 5847)
5848
d6e6e8b6
WD		 5849(define_expand "fnma<mode>4"
5850 [(set (match_operand:GPF_F16 0 "register_operand")
5851 (fma:GPF_F16
5852 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5853 (match_operand:GPF_F16 2 "register_operand")
5854 (match_operand:GPF_F16 3 "register_operand")))]
5855 "TARGET_FLOAT"
5856)
5857
5858(define_insn "*aarch64_fnma<mode>4"
9a594ad6
JW		 5859 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5860 (fma:GPF_F16
d6e6e8b6
WD		 5861 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5862 (match_operand:GPF_F16 3 "register_operand" "w")
5863 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 5864 "TARGET_FLOAT"
d6e6e8b6 5865 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 5866 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 5867)
5868
d6e6e8b6
WD		 5869
5870(define_expand "fms<mode>4"
5871 [(set (match_operand:GPF 0 "register_operand")
5872 (fma:GPF (match_operand:GPF 1 "register_operand")
5873 (match_operand:GPF 2 "register_operand")
5874 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5875 "TARGET_FLOAT"
5876)
5877
5878(define_insn "*aarch64_fms<mode>4"
43e9d192 5879 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5880 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5881 (match_operand:GPF 3 "register_operand" "w")
5882 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5883 "TARGET_FLOAT"
d6e6e8b6 5884 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5885 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5886)
5887
d6e6e8b6
WD		 5888(define_expand "fnms<mode>4"
5889 [(set (match_operand:GPF 0 "register_operand")
5890 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5891 (match_operand:GPF 2 "register_operand")
5892 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5893 "TARGET_FLOAT"
5894)
5895
5896(define_insn "*aarch64_fnms<mode>4"
43e9d192 5897 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5898 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5899 (match_operand:GPF 3 "register_operand" "w")
5900 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5901 "TARGET_FLOAT"
d6e6e8b6 5902 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5903 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5904)
5905
5906;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
d6e6e8b6 5907(define_insn "*aarch64_fnmadd<mode>4"
43e9d192 5908 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5909 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5910 (match_operand:GPF 3 "register_operand" "w")
5911 (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5912 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
d6e6e8b6 5913 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5914 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5915)
5916
5917;; -------------------------------------------------------------------
5918;; Floating-point conversions
5919;; -------------------------------------------------------------------
5920
5921(define_insn "extendsfdf2"
5922 [(set (match_operand:DF 0 "register_operand" "=w")
5923 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5924 "TARGET_FLOAT"
5925 "fcvt\\t%d0, %s1"
99e6af15 5926 [(set_attr "type" "f_cvt")]
43e9d192
IB		 5927)
5928
c2ec330c
AL		 5929(define_insn "extendhfsf2"
5930 [(set (match_operand:SF 0 "register_operand" "=w")
5931 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5932 "TARGET_FLOAT"
5933 "fcvt\\t%s0, %h1"
5934 [(set_attr "type" "f_cvt")]
5935)
5936
5937(define_insn "extendhfdf2"
5938 [(set (match_operand:DF 0 "register_operand" "=w")
5939 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5940 "TARGET_FLOAT"
5941 "fcvt\\t%d0, %h1"
5942 [(set_attr "type" "f_cvt")]
5943)
5944
43e9d192
IB		 5945(define_insn "truncdfsf2"
5946 [(set (match_operand:SF 0 "register_operand" "=w")
5947 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5948 "TARGET_FLOAT"
5949 "fcvt\\t%s0, %d1"
99e6af15 5950 [(set_attr "type" "f_cvt")]
43e9d192
IB		 5951)
5952
c2ec330c
AL		 5953(define_insn "truncsfhf2"
5954 [(set (match_operand:HF 0 "register_operand" "=w")
5955 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5956 "TARGET_FLOAT"
5957 "fcvt\\t%h0, %s1"
5958 [(set_attr "type" "f_cvt")]
5959)
5960
5961(define_insn "truncdfhf2"
5962 [(set (match_operand:HF 0 "register_operand" "=w")
5963 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5964 "TARGET_FLOAT"
5965 "fcvt\\t%h0, %d1"
5966 [(set_attr "type" "f_cvt")]
5967)
5968
22be0d08
MC		 5969;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5970;; and making r = w more expensive
5971
5972(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
43d0a8ee 5973 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
22be0d08
MC		 5974 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5975 "TARGET_FLOAT"
5976 "@
43d0a8ee
WD		 5977 fcvtz<su>\t%<s>0, %<s>1
5978 fcvtz<su>\t%<w>0, %<s>1"
5979 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
22be0d08
MC		 5980)
5981
5982;; Convert HF -> SI or DI
5983
5984(define_insn "<optab>_trunchf<GPI:mode>2"
5985 [(set (match_operand:GPI 0 "register_operand" "=r")
5986 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5987 "TARGET_FP_F16INST"
5988 "fcvtz<su>\t%<w>0, %h1"
5989 [(set_attr "type" "f_cvtf2i")]
5990)
5991
5992;; Convert DF -> SI or SF -> DI which can only be accomplished with
5993;; input in a fp register and output in a integer register
5994
5995(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
43e9d192 5996 [(set (match_operand:GPI 0 "register_operand" "=r")
22be0d08 5997 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
43e9d192 5998 "TARGET_FLOAT"
22be0d08 5999 "fcvtz<su>\t%<w>0, %<fpw>1"
99e6af15 6000 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 6001)
6002
eeb59c16
MC		 6003(define_insn "*fix_to_zero_extend<mode>di2"
6004 [(set (match_operand:DI 0 "register_operand" "=r")
6005 (zero_extend:DI
6006 (unsigned_fix:SI
6007 (match_operand:GPF 1 "register_operand" "w"))))]
6008 "TARGET_FLOAT"
6009 "fcvtzu\t%w0, %<s>1"
6010 [(set_attr "type" "f_cvtf2i")]
6011)
6012
188d0079
JH		 6013;; Equal width integer to fp and multiply combine.
6014(define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6015 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6016 (mult:GPF (FLOATUORS:GPF
6017 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6018 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6019 "TARGET_FLOAT"
6020 {
6021 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6022 switch (which_alternative)
6023 {
6024 case 0:
6025 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6026 case 1:
6027 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6028 default:
6029 gcc_unreachable ();
6030 }
6031 }
6032 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6033 (set_attr "arch" "simd,fp")]
6034)
6035
6036;; Unequal width integer to fp and multiply combine.
6037(define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6038 [(set (match_operand:GPF 0 "register_operand" "=w")
6039 (mult:GPF (FLOATUORS:GPF
6040 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6041 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6042 "TARGET_FLOAT"
6043 {
6044 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6045 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6046 }
6047 [(set_attr "type" "f_cvti2f")]
6048)
6049
6050;; Equal width integer to fp conversion.
0d35c5c2
VP		 6051(define_insn "<optab><fcvt_target><GPF:mode>2"
6052 [(set (match_operand:GPF 0 "register_operand" "=w,w")
43d0a8ee 6053 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
31ca7cba 6054 "TARGET_FLOAT"
0d35c5c2
VP		 6055 "@
6056 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6057 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
488461d8
MM		 6058 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6059 (set_attr "arch" "simd,fp")]
43e9d192
IB		 6060)
6061
188d0079 6062;; Unequal width integer to fp conversions.
0d35c5c2 6063(define_insn "<optab><fcvt_iesize><GPF:mode>2"
43e9d192 6064 [(set (match_operand:GPF 0 "register_operand" "=w")
0d35c5c2 6065 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
43e9d192 6066 "TARGET_FLOAT"
0d35c5c2
VP		 6067 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6068 [(set_attr "type" "f_cvti2f")]
43e9d192
IB		 6069)
6070
bf1e3646
JG		 6071;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6072;; midend will arrange for an SImode conversion to HFmode to first go
6073;; through DFmode, then to HFmode. But first it will try converting
6074;; to DImode then down, which would match our DImode pattern below and
6075;; give very poor code-generation. So, we must provide our own emulation
6076;; of the mid-end logic.
6077
6078(define_insn "aarch64_fp16_<optab><mode>hf2"
d7f33f07
JW		 6079 [(set (match_operand:HF 0 "register_operand" "=w")
6080 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6081 "TARGET_FP_F16INST"
6082 "<su_optab>cvtf\t%h0, %<w>1"
6083 [(set_attr "type" "f_cvti2f")]
6084)
6085
bf1e3646
JG		 6086(define_expand "<optab>sihf2"
6087 [(set (match_operand:HF 0 "register_operand")
6088 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6089 "TARGET_FLOAT"
6090{
6091 if (TARGET_FP_F16INST)
6092 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6093 else
6094 {
6095 rtx convert_target = gen_reg_rtx (DFmode);
6096 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6097 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6098 }
6099 DONE;
6100}
6101)
6102
6103;; For DImode there is no wide enough floating-point mode that we
6104;; can convert through natively (TFmode would work, but requires a library
6105;; call). However, we know that any value >= 65504 will be rounded
6106;; to infinity on conversion. This is well within the range of SImode, so
6107;; we can:
6108;; Saturate to SImode.
6109;; Convert from that to DFmode
6110;; Convert from that to HFmode (phew!).
6111;; Note that the saturation to SImode requires the SIMD extensions. If
6112;; we ever need to provide this pattern where the SIMD extensions are not
6113;; available, we would need a different approach.
6114
6115(define_expand "<optab>dihf2"
6116 [(set (match_operand:HF 0 "register_operand")
6117 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6118 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6119{
6120 if (TARGET_FP_F16INST)
6121 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6122 else
6123 {
6124 rtx sat_target = gen_reg_rtx (SImode);
6125 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6126 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6127 }
6128
6129 DONE;
6130}
6131)
6132
3f598afe
JW		 6133;; Convert between fixed-point and floating-point (scalar modes)
6134
6135(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6136 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6137 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6138 (match_operand:SI 2 "immediate_operand" "i, i")]
6139 FCVT_F2FIXED))]
6140 ""
6141 "@
1f0e9e34
JG		 6142 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6143 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
3f598afe 6144 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
488461d8 6145 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6146)
6147
6148(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6149 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6150 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6151 (match_operand:SI 2 "immediate_operand" "i, i")]
6152 FCVT_FIXED2F))]
6153 ""
6154 "@
1f0e9e34
JG		 6155 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6156 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
3f598afe 6157 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
488461d8 6158 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6159)
6160
68ad28c3
JW		 6161(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6162 [(set (match_operand:GPI 0 "register_operand" "=r")
6163 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6164 (match_operand:SI 2 "immediate_operand" "i")]
6165 FCVT_F2FIXED))]
6166 "TARGET_FP_F16INST"
6167 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6168 [(set_attr "type" "f_cvtf2i")]
6169)
6170
6171(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6172 [(set (match_operand:HF 0 "register_operand" "=w")
6173 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6174 (match_operand:SI 2 "immediate_operand" "i")]
6175 FCVT_FIXED2F))]
6176 "TARGET_FP_F16INST"
6177 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6178 [(set_attr "type" "f_cvti2f")]
6179)
6180
6181(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6182 [(set (match_operand:HI 0 "register_operand" "=w")
6183 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6184 (match_operand:SI 2 "immediate_operand" "i")]
6185 FCVT_F2FIXED))]
6186 "TARGET_SIMD"
6187 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6188 [(set_attr "type" "neon_fp_to_int_s")]
6189)
6190
6191(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6192 [(set (match_operand:HF 0 "register_operand" "=w")
6193 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6194 (match_operand:SI 2 "immediate_operand" "i")]
6195 FCVT_FIXED2F))]
6196 "TARGET_SIMD"
6197 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6198 [(set_attr "type" "neon_int_to_fp_s")]
6199)
6200
43e9d192
IB		 6201;; -------------------------------------------------------------------
6202;; Floating-point arithmetic
6203;; -------------------------------------------------------------------
6204
6205(define_insn "add<mode>3"
68ad28c3
JW		 6206 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6207 (plus:GPF_F16
6208 (match_operand:GPF_F16 1 "register_operand" "w")
6209 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6210 "TARGET_FLOAT"
6211 "fadd\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6212 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6213)
6214
6215(define_insn "sub<mode>3"
68ad28c3
JW		 6216 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6217 (minus:GPF_F16
6218 (match_operand:GPF_F16 1 "register_operand" "w")
6219 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6220 "TARGET_FLOAT"
6221 "fsub\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6222 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6223)
6224
6225(define_insn "mul<mode>3"
68ad28c3
JW		 6226 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6227 (mult:GPF_F16
6228 (match_operand:GPF_F16 1 "register_operand" "w")
6229 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6230 "TARGET_FLOAT"
6231 "fmul\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6232 [(set_attr "type" "fmul<stype>")]
43e9d192
IB		 6233)
6234
6235(define_insn "*fnmul<mode>3"
6236 [(set (match_operand:GPF 0 "register_operand" "=w")
6237 (mult:GPF
6238 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6239 (match_operand:GPF 2 "register_operand" "w")))]
2b3fe2e9
SN		 6240 "TARGET_FLOAT && !flag_rounding_math"
6241 "fnmul\\t%<s>0, %<s>1, %<s>2"
6242 [(set_attr "type" "fmul<s>")]
6243)
6244
6245(define_insn "*fnmul<mode>3"
6246 [(set (match_operand:GPF 0 "register_operand" "=w")
6247 (neg:GPF (mult:GPF
6248 (match_operand:GPF 1 "register_operand" "w")
6249 (match_operand:GPF 2 "register_operand" "w"))))]
43e9d192
IB		 6250 "TARGET_FLOAT"
6251 "fnmul\\t%<s>0, %<s>1, %<s>2"
99e6af15 6252 [(set_attr "type" "fmul<s>")]
43e9d192
IB		 6253)
6254
79a2bc2d 6255(define_expand "div<mode>3"
68ad28c3
JW		 6256 [(set (match_operand:GPF_F16 0 "register_operand")
6257 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6258 (match_operand:GPF_F16 2 "register_operand")))]
1be49a38 6259 "TARGET_FLOAT"
79a2bc2d
EM		 6260{
6261 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6262 DONE;
6263
6264 operands[1] = force_reg (<MODE>mode, operands[1]);
6265})
6266
6267(define_insn "*div<mode>3"
68ad28c3
JW		 6268 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6269 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6270 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6271 "TARGET_FLOAT"
6272 "fdiv\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6273 [(set_attr "type" "fdiv<stype>")]
43e9d192
IB		 6274)
6275
6276(define_insn "neg<mode>2"
d7f33f07
JW		 6277 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6278 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6279 "TARGET_FLOAT"
6280 "fneg\\t%<s>0, %<s>1"
d7f33f07 6281 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6282)
6283
98daafa0 6284(define_expand "sqrt<mode>2"
1bbffb87
DZ		 6285 [(set (match_operand:GPF_F16 0 "register_operand")
6286 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
98daafa0
EM		 6287 "TARGET_FLOAT"
6288{
6289 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6290 DONE;
6291})
6292
6293(define_insn "*sqrt<mode>2"
d7f33f07
JW		 6294 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6295 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6296 "TARGET_FLOAT"
6297 "fsqrt\\t%<s>0, %<s>1"
d7f33f07 6298 [(set_attr "type" "fsqrt<stype>")]
43e9d192
IB		 6299)
6300
6301(define_insn "abs<mode>2"
d7f33f07
JW		 6302 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6303 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6304 "TARGET_FLOAT"
6305 "fabs\\t%<s>0, %<s>1"
d7f33f07 6306 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6307)
6308
6309;; Given that smax/smin do not specify the result when either input is NaN,
6310;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6311;; for smin.
6312
6313(define_insn "smax<mode>3"
6314 [(set (match_operand:GPF 0 "register_operand" "=w")
6315 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6316 (match_operand:GPF 2 "register_operand" "w")))]
6317 "TARGET_FLOAT"
6318 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6319 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6320)
6321
6322(define_insn "smin<mode>3"
6323 [(set (match_operand:GPF 0 "register_operand" "=w")
6324 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6325 (match_operand:GPF 2 "register_operand" "w")))]
6326 "TARGET_FLOAT"
6327 "fminnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6328 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6329)
6330
1efafef3
TC		 6331;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6332;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6333;; which implement the IEEE fmax ()/fmin () functions.
6334(define_insn "<maxmin_uns><mode>3"
68ad28c3
JW		 6335 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6336 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6337 (match_operand:GPF_F16 2 "register_operand" "w")]
1efafef3 6338 FMAXMIN_UNS))]
202d0c11 6339 "TARGET_FLOAT"
1efafef3 6340 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6341 [(set_attr "type" "f_minmax<stype>")]
202d0c11
DS		 6342)
6343
f66e6423
TC		 6344(define_expand "lrint<GPF:mode><GPI:mode>2"
6345 [(match_operand:GPI 0 "register_operand")
6346 (match_operand:GPF 1 "register_operand")]
1013465f 6347 "TARGET_FLOAT
fb802d91 6348 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
1013465f 6349 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
f66e6423
TC		 6350{
6351 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6352 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6353 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6354 DONE;
6355}
6356)
6357
fb0f04fe
JG		 6358;; For copysign (x, y), we want to generate:
6359;;
6360;; LDR d2, #(1 << 63)
6361;; BSL v2.8b, [y], [x]
6362;;
315fdae8
RE		 6363;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6364;; we expect these operations to nearly always operate on
6365;; floating-point values, we do not want the operation to be
6366;; simplified into a bit-field insert operation that operates on the
6367;; integer side, since typically that would involve three inter-bank
6368;; register copies. As we do not expect copysign to be followed by
6369;; other logical operations on the result, it seems preferable to keep
6370;; this as an unspec operation, rather than exposing the underlying
6371;; logic to the compiler.
6372
6373(define_expand "copysign<GPF:mode>3"
6374 [(match_operand:GPF 0 "register_operand")
6375 (match_operand:GPF 1 "register_operand")
6376 (match_operand:GPF 2 "register_operand")]
fb0f04fe
JG		 6377 "TARGET_FLOAT && TARGET_SIMD"
6378{
315fdae8
RE		 6379 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6380 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6381 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6382 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6383 bitmask));
fb0f04fe
JG		 6384 DONE;
6385}
6386)
6387
315fdae8
RE		 6388(define_insn "copysign<GPF:mode>3_insn"
6389 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6390 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6391 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6392 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6393 UNSPEC_COPYSIGN))]
fb0f04fe 6394 "TARGET_FLOAT && TARGET_SIMD"
315fdae8
RE		 6395 "@
6396 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6397 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6398 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6399 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6400 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
fb0f04fe
JG		 6401)
6402
315fdae8 6403
4261463d
TC		 6404;; For xorsign (x, y), we want to generate:
6405;;
6406;; LDR d2, #1<<63
6407;; AND v3.8B, v1.8B, v2.8B
6408;; EOR v0.8B, v0.8B, v3.8B
6409;;
6410
6411(define_expand "xorsign<mode>3"
6412 [(match_operand:GPF 0 "register_operand")
6413 (match_operand:GPF 1 "register_operand")
6414 (match_operand:GPF 2 "register_operand")]
6415 "TARGET_FLOAT && TARGET_SIMD"
6416{
6417
5f565314 6418 machine_mode imode = <V_INT_EQUIV>mode;
4261463d
TC		 6419 rtx mask = gen_reg_rtx (imode);
6420 rtx op1x = gen_reg_rtx (imode);
6421 rtx op2x = gen_reg_rtx (imode);
6422
6423 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6424 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6425 imode)));
6426
5f565314
RS		 6427 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6428 lowpart_subreg (imode, operands[2],
6429 <MODE>mode)));
6430 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6431 lowpart_subreg (imode, operands[1],
6432 <MODE>mode),
6433 op2x));
4261463d
TC		 6434 emit_move_insn (operands[0],
6435 lowpart_subreg (<MODE>mode, op1x, imode));
6436 DONE;
6437}
6438)
6439
43e9d192
IB		 6440;; -------------------------------------------------------------------
6441;; Reload support
6442;; -------------------------------------------------------------------
b4f50fd4
RR		 6443;; Reload Scalar Floating point modes from constant pool.
6444;; The AArch64 port doesn't have __int128 constant move support.
1bbffb87 6445;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
0016d8d9 6446(define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
b4f50fd4
RR		 6447 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6448 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6449 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6450 "TARGET_FLOAT"
b4f50fd4
RR		 6451 {
6452 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6453 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6454 DONE;
6455 }
6456)
6457
6458;; Reload Vector modes from constant pool.
0016d8d9 6459(define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
b4f50fd4
RR		 6460 [(set (match_operand:VALL 0 "register_operand" "=w")
6461 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6462 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6463 "TARGET_FLOAT"
b4f50fd4
RR		 6464 {
6465 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6466 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6467 DONE;
6468 }
6469)
43e9d192 6470
0016d8d9 6471(define_expand "@aarch64_reload_mov<mode>"
43e9d192
IB		 6472 [(set (match_operand:TX 0 "register_operand" "=w")
6473 (match_operand:TX 1 "register_operand" "w"))
6474 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6475 ]
a636bb40 6476 "TARGET_FLOAT"
43e9d192
IB		 6477 {
6478 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6479 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6480 gen_aarch64_movtilow_tilow (op0, op1);
6481 gen_aarch64_movdi_tihigh (operands[2], op1);
6482 gen_aarch64_movtihigh_di (op0, operands[2]);
6483 DONE;
6484 }
6485)
6486
6487;; The following secondary reload helpers patterns are invoked
6488;; after or during reload as we don't want these patterns to start
6489;; kicking in during the combiner.
2a9704d0 6490
0016d8d9 6491(define_insn "@aarch64_movdi_<mode>low"
43e9d192 6492 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6493 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6494 (const_int 64) (const_int 0)))]
a636bb40 6495 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6496 "fmov\\t%x0, %d1"
c1dfcec7 6497 [(set_attr "type" "f_mrc")
43e9d192
IB		 6498 (set_attr "length" "4")
6499 ])
6500
0016d8d9 6501(define_insn "@aarch64_movdi_<mode>high"
43e9d192 6502 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6503 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6504 (const_int 64) (const_int 64)))]
a636bb40 6505 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6506 "fmov\\t%x0, %1.d[1]"
c1dfcec7 6507 [(set_attr "type" "f_mrc")
43e9d192
IB		 6508 (set_attr "length" "4")
6509 ])
6510
0016d8d9 6511(define_insn "@aarch64_mov<mode>high_di"
12dc6974 6512 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
43e9d192 6513 (const_int 64) (const_int 64))
12dc6974 6514 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6515 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6516 "fmov\\t%0.d[1], %x1"
c1dfcec7 6517 [(set_attr "type" "f_mcr")
43e9d192
IB		 6518 (set_attr "length" "4")
6519 ])
6520
0016d8d9 6521(define_insn "@aarch64_mov<mode>low_di"
12dc6974
SN		 6522 [(set (match_operand:TX 0 "register_operand" "=w")
6523 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6524 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6525 "fmov\\t%d0, %x1"
c1dfcec7 6526 [(set_attr "type" "f_mcr")
43e9d192
IB		 6527 (set_attr "length" "4")
6528 ])
6529
6530(define_insn "aarch64_movtilow_tilow"
6531 [(set (match_operand:TI 0 "register_operand" "=w")
a636bb40 6532 (zero_extend:TI
43e9d192 6533 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
a636bb40 6534 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6535 "fmov\\t%d0, %d1"
d7256bb8 6536 [(set_attr "type" "fmov")
43e9d192
IB		 6537 (set_attr "length" "4")
6538 ])
6539
6540;; There is a deliberate reason why the parameters of high and lo_sum's
6541;; don't have modes for ADRP and ADD instructions. This is to allow high
6542;; and lo_sum's to be used with the labels defining the jump tables in
6543;; rodata section.
6544
28514dda 6545(define_expand "add_losym"
1bbffb87
DZ		 6546 [(set (match_operand 0 "register_operand")
6547 (lo_sum (match_operand 1 "register_operand")
6548 (match_operand 2 "aarch64_valid_symref")))]
43e9d192 6549 ""
28514dda 6550{
ef4bddc2 6551 machine_mode mode = GET_MODE (operands[0]);
28514dda
YZ		 6552
6553 emit_insn ((mode == DImode
6554 ? gen_add_losym_di
6555 : gen_add_losym_si) (operands[0],
6556 operands[1],
6557 operands[2]));
6558 DONE;
6559})
6560
6561(define_insn "add_losym_<mode>"
6562 [(set (match_operand:P 0 "register_operand" "=r")
6563 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6564 (match_operand 2 "aarch64_valid_symref" "S")))]
6565 ""
d6591257 6566 "add\\t%<w>0, %<w>1, :lo12:%c2"
d8e11262 6567 [(set_attr "type" "alu_imm")]
28514dda 6568)
43e9d192 6569
28514dda
YZ		 6570(define_insn "ldr_got_small_<mode>"
6571 [(set (match_operand:PTR 0 "register_operand" "=r")
6572 (unspec:PTR [(mem:PTR (lo_sum:PTR
6573 (match_operand:PTR 1 "register_operand" "r")
6574 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6575 UNSPEC_GOTSMALLPIC))]
6576 ""
d6591257 6577 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
db46a2e6 6578 [(set_attr "type" "load_<ldst_sz>")]
43e9d192
IB		 6579)
6580
28514dda 6581(define_insn "ldr_got_small_sidi"
43e9d192 6582 [(set (match_operand:DI 0 "register_operand" "=r")
28514dda
YZ		 6583 (zero_extend:DI
6584 (unspec:SI [(mem:SI (lo_sum:DI
6585 (match_operand:DI 1 "register_operand" "r")
6586 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6587 UNSPEC_GOTSMALLPIC)))]
6588 "TARGET_ILP32"
d6591257 6589 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
89b2133e 6590 [(set_attr "type" "load_4")]
43e9d192
IB		 6591)
6592
1b1e81f8
JW		 6593(define_insn "ldr_got_small_28k_<mode>"
6594 [(set (match_operand:PTR 0 "register_operand" "=r")
6595 (unspec:PTR [(mem:PTR (lo_sum:PTR
6596 (match_operand:PTR 1 "register_operand" "r")
6597 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6598 UNSPEC_GOTSMALLPIC28K))]
6599 ""
d6591257 6600 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
db46a2e6 6601 [(set_attr "type" "load_<ldst_sz>")]
1b1e81f8
JW		 6602)
6603
6604(define_insn "ldr_got_small_28k_sidi"
6605 [(set (match_operand:DI 0 "register_operand" "=r")
6606 (zero_extend:DI
6607 (unspec:SI [(mem:SI (lo_sum:DI
6608 (match_operand:DI 1 "register_operand" "r")
6609 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6610 UNSPEC_GOTSMALLPIC28K)))]
6611 "TARGET_ILP32"
d6591257 6612 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
89b2133e 6613 [(set_attr "type" "load_4")]
1b1e81f8
JW		 6614)
6615
d91480de
D		 6616(define_insn "@ldr_got_tiny_<mode>"
6617 [(set (match_operand:PTR 0 "register_operand" "=r")
6618 (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")]
6619 UNSPEC_GOTTINYPIC))]
87dd8ab0 6620 ""
d91480de
D		 6621 "ldr\t%<w>0, %L1"
6622 [(set_attr "type" "load_<ldst_sz>")]
6623)
6624
6625(define_insn "ldr_got_tiny_sidi"
6626 [(set (match_operand:DI 0 "register_operand" "=r")
6627 (zero_extend:DI
6628 (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6629 UNSPEC_GOTTINYPIC)))]
6630 "TARGET_ILP32"
6631 "ldr\t%w0, %L1"
6632 [(set_attr "type" "load_4")]
87dd8ab0
MS		 6633)
6634
43e9d192
IB		 6635(define_insn "aarch64_load_tp_hard"
6636 [(set (match_operand:DI 0 "register_operand" "=r")
6637 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6638 ""
6639 "mrs\\t%0, tpidr_el0"
99e6af15 6640 [(set_attr "type" "mrs")]
43e9d192
IB		 6641)
6642
6643;; The TLS ABI specifically requires that the compiler does not schedule
6644;; instructions in the TLS stubs, in order to enable linker relaxation.
6645;; Therefore we treat the stubs as an atomic sequence.
23b88fda 6646(define_expand "tlsgd_small_<mode>"
87ca615a 6647 [(parallel [(set (match_operand:PTR 0 "register_operand")
43e9d192 6648 (call (mem:DI (match_dup 2)) (const_int 1)))
08cc4d92 6649 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
87ca615a 6650 (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6651 (clobber (reg:DI LR_REGNUM))])]
6652 ""
6653{
6654 operands[2] = aarch64_tls_get_addr ();
6655})
6656
23b88fda 6657(define_insn "*tlsgd_small_<mode>"
87ca615a 6658 [(set (match_operand:PTR 0 "register_operand" "")
43e9d192 6659 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
08cc4d92 6660 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
87ca615a 6661 (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6662 (clobber (reg:DI LR_REGNUM))
6663 ]
6664 ""
6665 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
c1dfcec7 6666 [(set_attr "type" "call")
43e9d192
IB		 6667 (set_attr "length" "16")])
6668
621ad2de
AP		 6669(define_insn "tlsie_small_<mode>"
6670 [(set (match_operand:PTR 0 "register_operand" "=r")
6671 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
43e9d192
IB		 6672 UNSPEC_GOTSMALLTLS))]
6673 ""
621ad2de 6674 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
89b2133e 6675 [(set_attr "type" "load_4")
43e9d192
IB		 6676 (set_attr "length" "8")]
6677)
6678
621ad2de 6679(define_insn "tlsie_small_sidi"
43e9d192 6680 [(set (match_operand:DI 0 "register_operand" "=r")
621ad2de
AP		 6681 (zero_extend:DI
6682 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6683 UNSPEC_GOTSMALLTLS)))]
6684 ""
6685 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
89b2133e 6686 [(set_attr "type" "load_4")
621ad2de
AP		 6687 (set_attr "length" "8")]
6688)
6689
5ae7caad
JW		 6690(define_insn "tlsie_tiny_<mode>"
6691 [(set (match_operand:PTR 0 "register_operand" "=&r")
6692 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6693 (match_operand:PTR 2 "register_operand" "r")]
6694 UNSPEC_GOTTINYTLS))]
6695 ""
6696 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6697 [(set_attr "type" "multiple")
6698 (set_attr "length" "8")]
6699)
6700
6701(define_insn "tlsie_tiny_sidi"
6702 [(set (match_operand:DI 0 "register_operand" "=&r")
6703 (zero_extend:DI
6704 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6705 (match_operand:DI 2 "register_operand" "r")
6706 ]
6707 UNSPEC_GOTTINYTLS)))]
6708 ""
5b295a81 6709 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5ae7caad
JW		 6710 [(set_attr "type" "multiple")
6711 (set_attr "length" "8")]
6712)
6713
cbf5629e
JW		 6714(define_insn "tlsle12_<mode>"
6715 [(set (match_operand:P 0 "register_operand" "=r")
6716 (unspec:P [(match_operand:P 1 "register_operand" "r")
6717 (match_operand 2 "aarch64_tls_le_symref" "S")]
6718 UNSPEC_TLSLE12))]
621ad2de 6719 ""
cbf5629e
JW		 6720 "add\\t%<w>0, %<w>1, #%L2";
6721 [(set_attr "type" "alu_sreg")
6722 (set_attr "length" "4")]
6723)
621ad2de 6724
cbf5629e 6725(define_insn "tlsle24_<mode>"
621ad2de 6726 [(set (match_operand:P 0 "register_operand" "=r")
cbf5629e
JW		 6727 (unspec:P [(match_operand:P 1 "register_operand" "r")
6728 (match_operand 2 "aarch64_tls_le_symref" "S")]
6729 UNSPEC_TLSLE24))]
43e9d192 6730 ""
0699caae 6731 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
cbf5629e 6732 [(set_attr "type" "multiple")
43e9d192
IB		 6733 (set_attr "length" "8")]
6734)
6735
cbf5629e
JW		 6736(define_insn "tlsle32_<mode>"
6737 [(set (match_operand:P 0 "register_operand" "=r")
6738 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6739 UNSPEC_TLSLE32))]
6740 ""
6741 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6742 [(set_attr "type" "multiple")
6743 (set_attr "length" "8")]
6744)
6745
6746(define_insn "tlsle48_<mode>"
6747 [(set (match_operand:P 0 "register_operand" "=r")
6748 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6749 UNSPEC_TLSLE48))]
6750 ""
6751 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6752 [(set_attr "type" "multiple")
6753 (set_attr "length" "12")]
6754)
6755
80c13ac5
RS		 6756(define_expand "tlsdesc_small_<mode>"
6757 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6758 "TARGET_TLS_DESC"
6759 {
6760 if (TARGET_SVE)
bb6ce448
RS		 6761 {
6762 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6763 rtx_insn *call
6764 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6765 RTL_CONST_CALL_P (call) = 1;
6766 }
80c13ac5
RS		 6767 else
6768 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6769 DONE;
6770 }
6771)
6772
6773;; tlsdesc calls preserve all core and Advanced SIMD registers except
6774;; R0 and LR.
6775(define_insn "tlsdesc_small_advsimd_<mode>"
621ad2de
AP		 6776 [(set (reg:PTR R0_REGNUM)
6777 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
80c13ac5 6778 UNSPEC_TLSDESC))
43e9d192 6779 (clobber (reg:DI LR_REGNUM))
0bc11714 6780 (clobber (reg:CC CC_REGNUM))
0e510d18
ST		 6781 (clobber (match_scratch:DI 1 "=r"))
6782 (use (reg:DI FP_REGNUM))]
80c13ac5
RS		 6783 "TARGET_TLS_DESC && !TARGET_SVE"
6784 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6785 [(set_attr "type" "call")
6786 (set_attr "length" "16")])
6787
bb6ce448
RS		 6788;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6789;; describing the extra call-preserved guarantees. This would work
6790;; for non-SVE too, but avoiding a call is probably better if we can.
80c13ac5
RS		 6791(define_insn "tlsdesc_small_sve_<mode>"
6792 [(set (reg:PTR R0_REGNUM)
bb6ce448
RS		 6793 (call (mem:DI (unspec:PTR
6794 [(match_operand 0 "aarch64_valid_symref")]
6795 UNSPEC_TLSDESC))
6796 (const_int 0)))
6797 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
80c13ac5 6798 (clobber (reg:DI LR_REGNUM))
bb6ce448 6799 (clobber (match_scratch:DI 2 "=r"))]
80c13ac5 6800 "TARGET_TLS_DESC && TARGET_SVE"
bb6ce448 6801 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
c1dfcec7 6802 [(set_attr "type" "call")
43e9d192
IB		 6803 (set_attr "length" "16")])
6804
6805(define_insn "stack_tie"
6806 [(set (mem:BLK (scratch))
6807 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6808 (match_operand:DI 1 "register_operand" "rk")]
6809 UNSPEC_PRLG_STK))]
6810 ""
6811 ""
6812 [(set_attr "length" "0")]
6813)
6814
e1d5d19e
KT		 6815(define_insn "aarch64_fjcvtzs"
6816 [(set (match_operand:SI 0 "register_operand" "=r")
6817 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
d2b86e14
AC		 6818 UNSPEC_FJCVTZS))
6819 (clobber (reg:CC CC_REGNUM))]
e1d5d19e
KT		 6820 "TARGET_JSCVT"
6821 "fjcvtzs\\t%w0, %d1"
6822 [(set_attr "type" "f_cvtf2i")]
6823)
6824
db58fd89
JW		 6825;; Pointer authentication patterns are always provided. In architecture
6826;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6827;; This lets the user write portable software which authenticates pointers
6828;; when run on something which implements ARMv8.3-A, and which runs
6829;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6830;; implemented.
6831
6832;; Signing/Authenticating R30 using SP as the salt.
6833
6834(define_insn "<pauth_mnem_prefix>sp"
6835 [(set (reg:DI R30_REGNUM)
6836 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6837 ""
8fc16d72 6838 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
db58fd89
JW		 6839)
6840
6841;; Signing/Authenticating X17 using X16 as the salt.
6842
6843(define_insn "<pauth_mnem_prefix>1716"
6844 [(set (reg:DI R17_REGNUM)
6845 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6846 ""
8fc16d72 6847 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
db58fd89
JW		 6848)
6849
6850;; Stripping the signature in R30.
6851
6852(define_insn "xpaclri"
6853 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6854 ""
6855 "hint\t7 // xpaclri"
6856)
6857
a3eb8a52
EB		 6858;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6859;; all of memory. This blocks insns from being moved across this point.
6860
6861(define_insn "blockage"
6862 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6863 ""
6864 ""
6865 [(set_attr "length" "0")
6866 (set_attr "type" "block")]
6867)
6868
5f5c5e0f 6869(define_insn "probe_stack_range"
cd1bef27 6870 [(set (match_operand:DI 0 "register_operand" "=rk")
5f5c5e0f
EB		 6871 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6872 (match_operand:DI 2 "register_operand" "r")]
6873 UNSPECV_PROBE_STACK_RANGE))]
a3eb8a52
EB		 6874 ""
6875{
6876 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6877}
6878 [(set_attr "length" "32")]
6879)
6880
eb471ba3
TC		 6881;; This instruction is used to generate the stack clash stack adjustment and
6882;; probing loop. We can't change the control flow during prologue and epilogue
6883;; code generation. So we must emit a volatile unspec and expand it later on.
6884
6885(define_insn "@probe_sve_stack_clash_<mode>"
6886 [(set (match_operand:P 0 "register_operand" "=rk")
6887 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6888 (match_operand:P 2 "register_operand" "r")
6889 (match_operand:P 3 "const_int_operand" "n")
6890 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6891 UNSPECV_PROBE_STACK_RANGE))]
6892 "TARGET_SVE"
6893{
6894 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6895 operands[3], operands[4]);
6896}
6897 [(set_attr "length" "28")]
6898)
6899
a5bc806c
AP		 6900;; Named pattern for expanding thread pointer reference.
6901(define_expand "get_thread_pointerdi"
1bbffb87 6902 [(match_operand:DI 0 "register_operand")]
a5bc806c
AP		 6903 ""
6904{
6905 rtx tmp = aarch64_load_tp (operands[0]);
6906 if (tmp != operands[0])
6907 emit_move_insn (operands[0], tmp);
6908 DONE;
6909})
6910
74b27d8e
RS		 6911;; Defined for -mstack-protector-guard=sysreg, which goes through this
6912;; pattern rather than stack_protect_combined_set. Our implementation
6913;; of the latter can handle both.
36e17020
VK		 6914(define_expand "stack_protect_set"
6915 [(match_operand 0 "memory_operand")
74b27d8e 6916 (match_operand 1 "")]
36e17020
VK		 6917 ""
6918{
74b27d8e
RS		 6919 emit_insn (gen_stack_protect_combined_set (operands[0], operands[1]));
6920 DONE;
6921})
36e17020 6922
74b27d8e
RS		 6923(define_expand "stack_protect_combined_set"
6924 [(match_operand 0 "memory_operand")
6925 (match_operand 1 "")]
6926 ""
6927{
6928 machine_mode mode = GET_MODE (operands[0]);
6929 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
6930 AARCH64_SALT_SSP_SET);
36e17020
VK		 6931 emit_insn ((mode == DImode
6932 ? gen_stack_protect_set_di
6933 : gen_stack_protect_set_si) (operands[0], operands[1]));
6934 DONE;
6935})
6936
74b27d8e 6937;; Operand 1 is either AARCH64_SALT_SSP_SET or AARCH64_SALT_SSP_TEST.
cd0b2d36
RR		 6938(define_insn "reg_stack_protect_address_<mode>"
6939 [(set (match_operand:PTR 0 "register_operand" "=r")
74b27d8e
RS		 6940 (unspec:PTR [(match_operand 1 "const_int_operand")]
6941 UNSPEC_SSP_SYSREG))]
cd0b2d36
RR		 6942 "aarch64_stack_protector_guard != SSP_GLOBAL"
6943 {
6944 char buf[150];
6945 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
6946 aarch64_stack_protector_guard_reg_str);
6947 output_asm_insn (buf, operands);
6948 return "";
6949 }
6950 [(set_attr "type" "mrs")])
6951
a7e73b41
RE		 6952;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
6953;; canary value does not live beyond the life of this sequence.
36e17020
VK		 6954(define_insn "stack_protect_set_<mode>"
6955 [(set (match_operand:PTR 0 "memory_operand" "=m")
6956 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6957 UNSPEC_SP_SET))
6958 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6959 ""
a7e73b41 6960 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
36e17020
VK		 6961 [(set_attr "length" "12")
6962 (set_attr "type" "multiple")])
6963
74b27d8e
RS		 6964;; Defined for -mstack-protector-guard=sysreg, which goes through this
6965;; pattern rather than stack_protect_combined_test. Our implementation
6966;; of the latter can handle both.
36e17020
VK		 6967(define_expand "stack_protect_test"
6968 [(match_operand 0 "memory_operand")
74b27d8e 6969 (match_operand 1 "")
36e17020
VK		 6970 (match_operand 2)]
6971 ""
6972{
74b27d8e
RS		 6973 emit_insn (gen_stack_protect_combined_test (operands[0], operands[1],
6974 operands[2]));
6975 DONE;
6976})
9c6ab05f 6977
74b27d8e
RS		 6978(define_expand "stack_protect_combined_test"
6979 [(match_operand 0 "memory_operand")
6980 (match_operand 1 "")
6981 (match_operand 2)]
6982 ""
6983{
6984 machine_mode mode = GET_MODE (operands[0]);
6985 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
6986 AARCH64_SALT_SSP_TEST);
36e17020 6987 emit_insn ((mode == DImode
fe1a2642
RS		 6988 ? gen_stack_protect_test_di
6989 : gen_stack_protect_test_si) (operands[0], operands[1]));
6990
6991 rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
6992 emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx),
6993 cc_reg, operands[2]));
36e17020
VK		 6994 DONE;
6995})
6996
fe1a2642
RS		 6997;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
6998;; canary value does not live beyond the end of this sequence.
36e17020 6999(define_insn "stack_protect_test_<mode>"
fe1a2642
RS		 7000 [(set (reg:CC CC_REGNUM)
7001 (unspec:CC [(match_operand:PTR 0 "memory_operand" "m")
7002 (match_operand:PTR 1 "memory_operand" "m")]
7003 UNSPEC_SP_TEST))
7004 (clobber (match_scratch:PTR 2 "=&r"))
b4e9251d 7005 (clobber (match_scratch:PTR 3 "=&r"))]
36e17020 7006 ""
fe1a2642
RS		 7007 "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0"
7008 [(set_attr "length" "16")
36e17020
VK		 7009 (set_attr "type" "multiple")])
7010
0d7e5fa6
AC		 7011;; Write into the Floating-point Status or Control Register.
7012(define_insn "@aarch64_set_<fpscr_name><GPI:mode>"
7013 [(unspec_volatile [(match_operand:GPI 0 "register_operand" "r")] SET_FPSCR)]
aa87aced 7014 ""
0d7e5fa6 7015 "msr\\t<fpscr_name>, %0"
aa87aced
KV		 7016 [(set_attr "type" "mrs")])
7017
0d7e5fa6
AC		 7018;; Read into the Floating-point Status or Control Register.
7019(define_insn "@aarch64_get_<fpscr_name><GPI:mode>"
7020 [(set (match_operand:GPI 0 "register_operand" "=r")
7021 (unspec_volatile:GPI [(const_int 0)] GET_FPSCR))]
aa87aced 7022 ""
0d7e5fa6 7023 "mrs\\t%0, <fpscr_name>"
aa87aced
KV		 7024 [(set_attr "type" "mrs")])
7025
594bdd53
FY		 7026;; Define the subtract-one-and-jump insns so loop.c
7027;; knows what to generate.
7028(define_expand "doloop_end"
7029 [(use (match_operand 0 "" "")) ; loop pseudo
7030 (use (match_operand 1 "" ""))] ; label
7031 "optimize > 0 && flag_modulo_sched"
7032{
7033 rtx s0;
7034 rtx bcomp;
7035 rtx loc_ref;
7036 rtx cc_reg;
7037 rtx insn;
7038 rtx cmp;
7039
7040 /* Currently SMS relies on the do-loop pattern to recognize loops
7041 where (1) the control part consists of all insns defining and/or
7042 using a certain 'count' register and (2) the loop count can be
7043 adjusted by modifying this register prior to the loop.
7044 ??? The possible introduction of a new block to initialize the
7045 new IV can potentially affect branch optimizations. */
7046
7047 if (GET_MODE (operands[0]) != DImode)
7048 FAIL;
7049
7050 s0 = operands [0];
7051 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7052
7053 cmp = XVECEXP (PATTERN (insn), 0, 0);
7054 cc_reg = SET_DEST (cmp);
7055 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7056 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
f7df4a84 7057 emit_jump_insn (gen_rtx_SET (pc_rtx,
594bdd53
FY		 7058 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7059 loc_ref, pc_rtx)));
7060 DONE;
7061})
7062
3751345d
RE		 7063;; Track speculation through conditional branches. We assume that
7064;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7065(define_insn "speculation_tracker"
7066 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
996bea89 7067 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
3751345d
RE		 7068 UNSPEC_SPECULATION_TRACKER))]
7069 ""
7070 {
7071 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7072 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7073 return "";
7074 }
7075 [(set_attr "type" "csel")]
7076)
7077
2214053f
RS		 7078;; Like speculation_tracker, but track the inverse condition.
7079(define_insn "speculation_tracker_rev"
7080 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7081 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7082 UNSPEC_SPECULATION_TRACKER_REV))]
7083 ""
7084 {
7085 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7086 output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands);
7087 return "";
7088 }
7089 [(set_attr "type" "csel")]
7090)
7091
b5f794b4
SD		 7092;; BTI <target> instructions
7093(define_insn "bti_noarg"
7094 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7095 ""
7096 "hint\t32 // bti"
7097 [(set_attr "type" "no_insn")]
7098)
7099
7100(define_insn "bti_c"
7101 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7102 ""
7103 "hint\t34 // bti c"
7104 [(set_attr "type" "no_insn")]
7105)
7106
7107(define_insn "bti_j"
7108 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7109 ""
7110 "hint\t36 // bti j"
7111 [(set_attr "type" "no_insn")]
7112)
7113
7114(define_insn "bti_jc"
7115 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7116 ""
7117 "hint\t38 // bti jc"
7118 [(set_attr "type" "no_insn")]
7119)
7120
def745b2
RE		 7121;; Hard speculation barrier.
7122(define_insn "speculation_barrier"
7123 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7124 ""
7125 "isb\;dsb\\tsy"
7126 [(set_attr "length" "8")
3751345d
RE		 7127 (set_attr "type" "block")
7128 (set_attr "speculation_barrier" "true")]
def745b2
RE		 7129)
7130
c0111dc4
RE		 7131;; Support for __builtin_speculation_safe_value when we have speculation
7132;; tracking enabled. Use the speculation tracker to decide whether to
7133;; copy operand 1 to the target, or to copy the fail value (operand 2).
21cebf90 7134(define_expand "@despeculate_copy<ALLI_TI:mode>"
1bbffb87 7135 [(set (match_operand:ALLI_TI 0 "register_operand")
c0111dc4 7136 (unspec_volatile:ALLI_TI
1bbffb87
DZ		 7137 [(match_operand:ALLI_TI 1 "register_operand")
7138 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
c0111dc4
RE		 7139 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7140 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7141 ""
7142 "
7143 {
7144 if (operands[2] == const0_rtx)
7145 {
7146 rtx tracker;
7147 if (<MODE>mode == TImode)
7148 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7149 else
7150 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7151
7152 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7153 tracker));
7154 DONE;
7155 }
7156 }
7157 "
7158)
7159
7160;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7161;; encoding for CSDB, but will work in older versions of the assembler.
7162(define_insn "*despeculate_copy<ALLI:mode>_insn"
7163 [(set (match_operand:ALLI 0 "register_operand" "=r")
7164 (unspec_volatile:ALLI
7165 [(match_operand:ALLI 1 "register_operand" "r")
7166 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7167 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7168 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7169 ""
7170 {
7171 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7172 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7173 operands);
7174 return "";
7175 }
7176 [(set_attr "length" "12")
7177 (set_attr "type" "block")
7178 (set_attr "speculation_barrier" "true")]
7179)
7180
7181;; Pattern to match despeculate_copyti
7182(define_insn "*despeculate_copyti_insn"
7183 [(set (match_operand:TI 0 "register_operand" "=r")
7184 (unspec_volatile:TI
7185 [(match_operand:TI 1 "register_operand" "r")
7186 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7187 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7188 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7189 ""
7190 {
7191 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7192 output_asm_insn
7193 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7194 operands);
7195 return "";
7196 }
7197 [(set_attr "length" "16")
7198 (set_attr "type" "block")
7199 (set_attr "speculation_barrier" "true")]
7200)
7201
7202(define_insn "despeculate_simple<ALLI:mode>"
7203 [(set (match_operand:ALLI 0 "register_operand" "=r")
7204 (unspec_volatile:ALLI
7205 [(match_operand:ALLI 1 "register_operand" "r")
7206 (use (match_operand:ALLI 2 "register_operand" ""))]
7207 UNSPECV_SPECULATION_BARRIER))]
7208 ""
7209 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7210 [(set_attr "type" "block")
7211 (set_attr "length" "8")
7212 (set_attr "speculation_barrier" "true")]
7213)
7214
7215(define_insn "despeculate_simpleti"
7216 [(set (match_operand:TI 0 "register_operand" "=r")
7217 (unspec_volatile:TI
7218 [(match_operand:TI 1 "register_operand" "r")
7219 (use (match_operand:DI 2 "register_operand" ""))]
7220 UNSPECV_SPECULATION_BARRIER))]
7221 ""
7222 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7223 [(set_attr "type" "block")
7224 (set_attr "length" "12")
7225 (set_attr "speculation_barrier" "true")]
7226)
7227
10bd1d96
KT		 7228(define_insn "aarch64_<frintnzs_op><mode>"
7229 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7230 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7231 FRINTNZX))]
7232 "TARGET_FRINT && TARGET_FLOAT
7233 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7234 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7235 [(set_attr "type" "f_rint<stype>")]
7236)
7237
89626179
SD		 7238;; Transactional Memory Extension (TME) instructions.
7239
7240(define_insn "tstart"
7241 [(set (match_operand:DI 0 "register_operand" "=r")
7242 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7243 (clobber (mem:BLK (scratch)))]
7244 "TARGET_TME"
7245 "tstart\\t%0"
7246 [(set_attr "type" "tme")]
7247)
7248
7249(define_insn "ttest"
7250 [(set (match_operand:DI 0 "register_operand" "=r")
7251 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7252 (clobber (mem:BLK (scratch)))]
7253 "TARGET_TME"
7254 "ttest\\t%0"
7255 [(set_attr "type" "tme")]
7256)
7257
7258(define_insn "tcommit"
7259 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7260 (clobber (mem:BLK (scratch)))]
7261 "TARGET_TME"
7262 "tcommit"
7263 [(set_attr "type" "tme")]
7264)
7265
7266(define_insn "tcancel"
7267 [(unspec_volatile:BLK
7268 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7269 (clobber (mem:BLK (scratch)))]
7270 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7271 "tcancel\\t#%0"
7272 [(set_attr "type" "tme")]
7273)
7274
c5dc215d
KT		 7275(define_insn "aarch64_rndr"
7276 [(set (match_operand:DI 0 "register_operand" "=r")
7277 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7278 (set (reg:CC_Z CC_REGNUM)
7279 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7280 "TARGET_RNG"
7281 "mrs\t%0, RNDR"
7282 [(set_attr "type" "mrs")]
7283)
7284
7285(define_insn "aarch64_rndrrs"
7286 [(set (match_operand:DI 0 "register_operand" "=r")
7287 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7288 (set (reg:CC_Z CC_REGNUM)
7289 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7290 "TARGET_RNG"
7291 "mrs\t%0, RNDRRS"
7292 [(set_attr "type" "mrs")]
7293)
7294
ef01e6bb
DZ		 7295;; Memory Tagging Extension (MTE) instructions.
7296
7297(define_insn "irg"
7298 [(set (match_operand:DI 0 "register_operand" "=rk")
7299 (ior:DI
7300 (and:DI (match_operand:DI 1 "register_operand" "rk")
7301 (const_int -1080863910568919041)) ;; 0xf0ff...
7302 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")]
7303 UNSPEC_GEN_TAG_RND)
7304 (const_int 56))))]
7305 "TARGET_MEMTAG"
7306 "irg\\t%0, %1, %2"
7307 [(set_attr "type" "memtag")]
7308)
7309
7310(define_insn "gmi"
7311 [(set (match_operand:DI 0 "register_operand" "=r")
7312 (ior:DI (ashift:DI
7313 (const_int 1)
7314 (and:QI (lshiftrt:DI
7315 (match_operand:DI 1 "register_operand" "rk")
7316 (const_int 56)) (const_int 15)))
7317 (match_operand:DI 2 "register_operand" "r")))]
7318 "TARGET_MEMTAG"
7319 "gmi\\t%0, %1, %2"
7320 [(set_attr "type" "memtag")]
7321)
7322
7323(define_insn "addg"
7324 [(set (match_operand:DI 0 "register_operand" "=rk")
7325 (ior:DI
7326 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7327 (match_operand:DI 2 "aarch64_granule16_uimm6" "i"))
7328 (const_int -1080863910568919041)) ;; 0xf0ff...
7329 (ashift:DI
7330 (unspec:QI
7331 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15))
7332 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")]
7333 UNSPEC_GEN_TAG)
7334 (const_int 56))))]
7335 "TARGET_MEMTAG"
7336 "addg\\t%0, %1, #%2, #%3"
7337 [(set_attr "type" "memtag")]
7338)
7339
7340(define_insn "subp"
7341 [(set (match_operand:DI 0 "register_operand" "=r")
7342 (minus:DI
7343 (and:DI (match_operand:DI 1 "register_operand" "rk")
7344 (const_int 72057594037927935)) ;; 0x00ff...
7345 (and:DI (match_operand:DI 2 "register_operand" "rk")
7346 (const_int 72057594037927935))))] ;; 0x00ff...
7347 "TARGET_MEMTAG"
7348 "subp\\t%0, %1, %2"
7349 [(set_attr "type" "memtag")]
7350)
7351
7352;; LDG will use the 16-byte aligned value of the address.
7353(define_insn "ldg"
7354 [(set (match_operand:DI 0 "register_operand" "+r")
7355 (ior:DI
7356 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff...
7357 (ashift:DI
7358 (mem:QI (unspec:DI
7359 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7360 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))
7361 (const_int -16))] UNSPEC_TAG_SPACE))
7362 (const_int 56))))]
7363 "TARGET_MEMTAG"
7364 "ldg\\t%0, [%1, #%2]"
7365 [(set_attr "type" "memtag")]
7366)
7367
7368;; STG doesn't align the address but aborts with alignment fault
7369;; when the address is not 16-byte aligned.
7370(define_insn "stg"
7371 [(set (mem:QI (unspec:DI
7372 [(plus:DI (match_operand:DI 1 "register_operand" "rk")
7373 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))]
7374 UNSPEC_TAG_SPACE))
7375 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk")
7376 (const_int 56)) (const_int 15)))]
7377 "TARGET_MEMTAG"
7378 "stg\\t%0, [%1, #%2]"
7379 [(set_attr "type" "memtag")]
7380)
7381
43e9d192
IB		 7382;; AdvSIMD Stuff
7383(include "aarch64-simd.md")
7384
0462169c
SN		 7385;; Atomic Operations
7386(include "atomics.md")
350013bc
BC		 7387
7388;; ldp/stp peephole patterns
7389(include "aarch64-ldpstp.md")
43cacb12
RS		 7390
7391;; SVE.
7392(include "aarch64-sve.md")
0617e23c
AM		 7393
7394;; SVE2.
7395(include "aarch64-sve2.md")
Mirror of https://gcc.gnu.org/git/gcc.git