]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/aarch64/aarch64.md
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update copyright years.
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64.md
CommitLineData
43e9d192 1;; Machine description for AArch64 architecture.
7adcbafe 2;; Copyright (C) 2009-2022 Free Software Foundation, Inc.
43e9d192
IB		 3;; Contributed by ARM Ltd.
4;;
5;; This file is part of GCC.
6;;
7;; GCC is free software; you can redistribute it and/or modify it
8;; under the terms of the GNU General Public License as published by
9;; the Free Software Foundation; either version 3, or (at your option)
10;; any later version.
11;;
12;; GCC is distributed in the hope that it will be useful, but
13;; WITHOUT ANY WARRANTY; without even the implied warranty of
14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15;; General Public License for more details.
16;;
17;; You should have received a copy of the GNU General Public License
18;; along with GCC; see the file COPYING3. If not see
19;; <http://www.gnu.org/licenses/>.
20
21;; Register numbers
22(define_constants
23 [
24 (R0_REGNUM 0)
25 (R1_REGNUM 1)
26 (R2_REGNUM 2)
27 (R3_REGNUM 3)
28 (R4_REGNUM 4)
29 (R5_REGNUM 5)
30 (R6_REGNUM 6)
31 (R7_REGNUM 7)
32 (R8_REGNUM 8)
33 (R9_REGNUM 9)
34 (R10_REGNUM 10)
35 (R11_REGNUM 11)
36 (R12_REGNUM 12)
37 (R13_REGNUM 13)
38 (R14_REGNUM 14)
39 (R15_REGNUM 15)
40 (R16_REGNUM 16)
43e9d192 41 (R17_REGNUM 17)
43e9d192
IB		 42 (R18_REGNUM 18)
43 (R19_REGNUM 19)
44 (R20_REGNUM 20)
45 (R21_REGNUM 21)
46 (R22_REGNUM 22)
47 (R23_REGNUM 23)
48 (R24_REGNUM 24)
49 (R25_REGNUM 25)
50 (R26_REGNUM 26)
51 (R27_REGNUM 27)
52 (R28_REGNUM 28)
53 (R29_REGNUM 29)
54 (R30_REGNUM 30)
43e9d192
IB		 55 (SP_REGNUM 31)
56 (V0_REGNUM 32)
234c9000
AH		 57 (V1_REGNUM 33)
58 (V2_REGNUM 34)
59 (V3_REGNUM 35)
80c13ac5 60 (V4_REGNUM 36)
234c9000
AH		 61 (V5_REGNUM 37)
62 (V6_REGNUM 38)
63 (V7_REGNUM 39)
80c13ac5 64 (V8_REGNUM 40)
234c9000
AH		 65 (V9_REGNUM 41)
66 (V10_REGNUM 42)
67 (V11_REGNUM 43)
80c13ac5 68 (V12_REGNUM 44)
234c9000
AH		 69 (V13_REGNUM 45)
70 (V14_REGNUM 46)
43e9d192 71 (V15_REGNUM 47)
80c13ac5 72 (V16_REGNUM 48)
234c9000
AH		 73 (V17_REGNUM 49)
74 (V18_REGNUM 50)
75 (V19_REGNUM 51)
80c13ac5 76 (V20_REGNUM 52)
234c9000
AH		 77 (V21_REGNUM 53)
78 (V22_REGNUM 54)
79 (V23_REGNUM 55)
80c13ac5 80 (V24_REGNUM 56)
234c9000
AH		 81 (V25_REGNUM 57)
82 (V26_REGNUM 58)
83 (V27_REGNUM 59)
80c13ac5 84 (V28_REGNUM 60)
234c9000
AH		 85 (V29_REGNUM 61)
86 (V30_REGNUM 62)
43e9d192
IB		 87 (V31_REGNUM 63)
88 (SFP_REGNUM 64)
89 (AP_REGNUM 65)
90 (CC_REGNUM 66)
43cacb12
RS		 91 ;; Defined only to make the DWARF description simpler.
92 (VG_REGNUM 67)
93 (P0_REGNUM 68)
80c13ac5
RS		 94 (P1_REGNUM 69)
95 (P2_REGNUM 70)
96 (P3_REGNUM 71)
97 (P4_REGNUM 72)
98 (P5_REGNUM 73)
99 (P6_REGNUM 74)
43cacb12 100 (P7_REGNUM 75)
80c13ac5
RS		 101 (P8_REGNUM 76)
102 (P9_REGNUM 77)
103 (P10_REGNUM 78)
104 (P11_REGNUM 79)
105 (P12_REGNUM 80)
106 (P13_REGNUM 81)
107 (P14_REGNUM 82)
43cacb12 108 (P15_REGNUM 83)
c600df9a 109 (LAST_SAVED_REGNUM 83)
183bfdaf
RS		 110 (FFR_REGNUM 84)
111 ;; "FFR token": a fake register used for representing the scheduling
112 ;; restrictions on FFR-related operations.
113 (FFRT_REGNUM 85)
5773855c
OH		 114 ;; The pair of scratch registers used for stack probing with -fstack-check.
115 ;; Leave R9 alone as a possible choice for the static chain.
116 ;; Note that the use of these registers is mutually exclusive with the use
117 ;; of STACK_CLASH_SVE_CFA_REGNUM, which is for -fstack-clash-protection
118 ;; rather than -fstack-check.
119 (PROBE_STACK_FIRST_REGNUM 10)
120 (PROBE_STACK_SECOND_REGNUM 11)
143d3b15
TC		 121 ;; Scratch register used by stack clash protection to calculate
122 ;; SVE CFA offsets during probing.
123 (STACK_CLASH_SVE_CFA_REGNUM 11)
124 ;; Scratch registers for prologue/epilogue use.
125 (EP0_REGNUM 12)
126 (EP1_REGNUM 13)
3751345d
RE		 127 ;; A couple of call-clobbered registers that we need to reserve when
128 ;; tracking speculation this is not ABI, so is subject to change.
3751345d 129 (SPECULATION_SCRATCH_REGNUM 14)
143d3b15
TC		 130 (SPECULATION_TRACKER_REGNUM 15)
131 ;; Scratch registers used in frame layout.
132 (IP0_REGNUM 16)
133 (IP1_REGNUM 17)
0e510d18 134 (FP_REGNUM 29)
143d3b15 135 (LR_REGNUM 30)
43e9d192
IB		 136 ]
137)
138
139(define_c_enum "unspec" [
8fc16d72
ST		 140 UNSPEC_AUTIA1716
141 UNSPEC_AUTIB1716
142 UNSPEC_AUTIASP
143 UNSPEC_AUTIBSP
08cc4d92 144 UNSPEC_CALLEE_ABI
43e9d192 145 UNSPEC_CASESI
0caf592d 146 UNSPEC_CPYMEM
5d357f26
KT		 147 UNSPEC_CRC32B
148 UNSPEC_CRC32CB
149 UNSPEC_CRC32CH
150 UNSPEC_CRC32CW
151 UNSPEC_CRC32CX
152 UNSPEC_CRC32H
153 UNSPEC_CRC32W
154 UNSPEC_CRC32X
3f598afe
JW		 155 UNSPEC_FCVTZS
156 UNSPEC_FCVTZU
e1d5d19e 157 UNSPEC_FJCVTZS
10bd1d96
KT		 158 UNSPEC_FRINT32Z
159 UNSPEC_FRINT32X
160 UNSPEC_FRINT64Z
161 UNSPEC_FRINT64X
58a3bd25 162 UNSPEC_URECPE
0050faf8
JG		 163 UNSPEC_FRECPE
164 UNSPEC_FRECPS
165 UNSPEC_FRECPX
43e9d192
IB		 166 UNSPEC_FRINTA
167 UNSPEC_FRINTI
168 UNSPEC_FRINTM
0659ce6f 169 UNSPEC_FRINTN
43e9d192
IB		 170 UNSPEC_FRINTP
171 UNSPEC_FRINTX
172 UNSPEC_FRINTZ
173 UNSPEC_GOTSMALLPIC
1b1e81f8 174 UNSPEC_GOTSMALLPIC28K
43e9d192 175 UNSPEC_GOTSMALLTLS
87dd8ab0 176 UNSPEC_GOTTINYPIC
5ae7caad 177 UNSPEC_GOTTINYTLS
89b4515c 178 UNSPEC_LD1
43e9d192 179 UNSPEC_LD2
fca7d0a4 180 UNSPEC_LD2_DREG
77efea31 181 UNSPEC_LD2_DUP
43e9d192 182 UNSPEC_LD3
fca7d0a4 183 UNSPEC_LD3_DREG
77efea31 184 UNSPEC_LD3_DUP
43e9d192 185 UNSPEC_LD4
fca7d0a4 186 UNSPEC_LD4_DREG
77efea31 187 UNSPEC_LD4_DUP
3ec1be97
CB		 188 UNSPEC_LD2_LANE
189 UNSPEC_LD3_LANE
190 UNSPEC_LD4_LANE
fdcddba8
PW		 191 UNSPEC_LD64B
192 UNSPEC_ST64B
193 UNSPEC_ST64BV
194 UNSPEC_ST64BV_RET
195 UNSPEC_ST64BV0
196 UNSPEC_ST64BV0_RET
43e9d192 197 UNSPEC_MB
bb768f8b 198 UNSPEC_MOVMEM
43e9d192 199 UNSPEC_NOP
8fc16d72
ST		 200 UNSPEC_PACIA1716
201 UNSPEC_PACIB1716
202 UNSPEC_PACIASP
203 UNSPEC_PACIBSP
43e9d192 204 UNSPEC_PRLG_STK
43cacb12 205 UNSPEC_REV
43e9d192 206 UNSPEC_RBIT
75add2d0 207 UNSPEC_SABAL
9f499a86 208 UNSPEC_SABAL2
6b2034c4 209 UNSPEC_SABDL
75add2d0
KT		 210 UNSPEC_SABDL2
211 UNSPEC_SADALP
3f598afe 212 UNSPEC_SCVTF
d3bd985e 213 UNSPEC_SETMEM
2e100703
VP		 214 UNSPEC_SISD_NEG
215 UNSPEC_SISD_SSHL
216 UNSPEC_SISD_USHL
217 UNSPEC_SSHL_2S
89b4515c 218 UNSPEC_ST1
43e9d192
IB		 219 UNSPEC_ST2
220 UNSPEC_ST3
221 UNSPEC_ST4
ba081b77
JG		 222 UNSPEC_ST2_LANE
223 UNSPEC_ST3_LANE
224 UNSPEC_ST4_LANE
43e9d192
IB		 225 UNSPEC_TLS
226 UNSPEC_TLSDESC
cbf5629e
JW		 227 UNSPEC_TLSLE12
228 UNSPEC_TLSLE24
229 UNSPEC_TLSLE32
230 UNSPEC_TLSLE48
75add2d0 231 UNSPEC_UABAL
9f499a86 232 UNSPEC_UABAL2
6b2034c4 233 UNSPEC_UABDL
75add2d0
KT		 234 UNSPEC_UABDL2
235 UNSPEC_UADALP
3f598afe 236 UNSPEC_UCVTF
2e100703 237 UNSPEC_USHL_2S
43e9d192 238 UNSPEC_VSTRUCTDUMMY
cd0b2d36 239 UNSPEC_SSP_SYSREG
36e17020
VK		 240 UNSPEC_SP_SET
241 UNSPEC_SP_TEST
850e5878 242 UNSPEC_RSHRN
a6fc00da
BH		 243 UNSPEC_RSQRT
244 UNSPEC_RSQRTE
245 UNSPEC_RSQRTS
d9c981c8 246 UNSPEC_NZCV
db58fd89 247 UNSPEC_XPACLRI
43cacb12
RS		 248 UNSPEC_LD1_SVE
249 UNSPEC_ST1_SVE
624d0f07
RS		 250 UNSPEC_LDNT1_SVE
251 UNSPEC_STNT1_SVE
43cacb12 252 UNSPEC_LD1RQ
bfaa08b7 253 UNSPEC_LD1_GATHER
624d0f07 254 UNSPEC_LDFF1_GATHER
0a09a948 255 UNSPEC_LDNT1_GATHER
f307441a 256 UNSPEC_ST1_SCATTER
0a09a948 257 UNSPEC_STNT1_SCATTER
06308276 258 UNSPEC_PRED_X
00fa90d9 259 UNSPEC_PRED_Z
34467289 260 UNSPEC_PTEST
624d0f07 261 UNSPEC_PTRUE
43cacb12
RS		 262 UNSPEC_UNPACKSHI
263 UNSPEC_UNPACKUHI
264 UNSPEC_UNPACKSLO
265 UNSPEC_UNPACKULO
266 UNSPEC_PACK
0a09a948
RS		 267 UNSPEC_WHILEGE
268 UNSPEC_WHILEGT
269 UNSPEC_WHILEHI
270 UNSPEC_WHILEHS
6ad9571b
RS		 271 UNSPEC_WHILELE
272 UNSPEC_WHILELO
273 UNSPEC_WHILELS
274 UNSPEC_WHILELT
58c036c8
RS		 275 UNSPEC_WHILERW
276 UNSPEC_WHILEWR
9f4cbab8
RS		 277 UNSPEC_LDN
278 UNSPEC_STN
f1739b48 279 UNSPEC_INSR
624d0f07 280 UNSPEC_CLASTA
bb6c2b68 281 UNSPEC_CLASTB
b781a135 282 UNSPEC_FADDA
002092be 283 UNSPEC_REV_SUBREG
4aeb1ba7 284 UNSPEC_REINTERPRET
3751345d 285 UNSPEC_SPECULATION_TRACKER
2214053f 286 UNSPEC_SPECULATION_TRACKER_REV
315fdae8 287 UNSPEC_COPYSIGN
89626179 288 UNSPEC_TTEST ; Represent transaction test.
624d0f07
RS		 289 UNSPEC_UPDATE_FFR
290 UNSPEC_UPDATE_FFRT
291 UNSPEC_RDFFR
292 UNSPEC_WRFFR
293 ;; Represents an SVE-style lane index, in which the indexing applies
294 ;; within the containing 128-bit block.
295 UNSPEC_SVE_LANE_SELECT
296 UNSPEC_SVE_CNT_PAT
297 UNSPEC_SVE_PREFETCH
298 UNSPEC_SVE_PREFETCH_GATHER
299 UNSPEC_SVE_COMPACT
300 UNSPEC_SVE_SPLICE
ef01e6bb
DZ		 301 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag.
302 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag.
303 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space.
9ceec73f 304 UNSPEC_LD1RO
74b27d8e 305 UNSPEC_SALT_ADDR
43e9d192
IB		 306])
307
308(define_c_enum "unspecv" [
309 UNSPECV_EH_RETURN ; Represent EH_RETURN
aa87aced
KV		 310 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
311 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
312 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
313 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
a3eb8a52 314 UNSPECV_BLOCKAGE ; Represent a blockage
f58101cf 315 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
def745b2 316 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
b5f794b4
SD		 317 UNSPECV_BTI_NOARG ; Represent BTI.
318 UNSPECV_BTI_C ; Represent BTI c.
319 UNSPECV_BTI_J ; Represent BTI j.
320 UNSPECV_BTI_JC ; Represent BTI jc.
89626179
SD		 321 UNSPECV_TSTART ; Represent transaction start.
322 UNSPECV_TCOMMIT ; Represent transaction commit.
323 UNSPECV_TCANCEL ; Represent transaction cancel.
c5dc215d
KT		 324 UNSPEC_RNDR ; Represent RNDR
325 UNSPEC_RNDRRS ; Represent RNDRRS
43e9d192
IB		 326 ]
327)
328
34467289
RS		 329;; These constants are used as a const_int in various SVE unspecs
330;; to indicate whether the governing predicate is known to be a PTRUE.
331(define_constants
332 [; Indicates that the predicate might not be a PTRUE.
333 (SVE_MAYBE_NOT_PTRUE 0)
334
335 ; Indicates that the predicate is known to be a PTRUE.
336 (SVE_KNOWN_PTRUE 1)])
337
c9c5a809
RS		 338;; These constants are used as a const_int in predicated SVE FP arithmetic
339;; to indicate whether the operation is allowed to make additional lanes
340;; active without worrying about the effect on faulting behavior.
341(define_constants
342 [; Indicates either that all lanes are active or that the instruction may
343 ; operate on inactive inputs even if doing so could induce a fault.
344 (SVE_RELAXED_GP 0)
345
346 ; Indicates that some lanes might be inactive and that the instruction
347 ; must not operate on inactive inputs if doing so could induce a fault.
348 (SVE_STRICT_GP 1)])
349
43e9d192
IB		 350;; If further include files are added the defintion of MD_INCLUDES
351;; must be updated.
352
353(include "constraints.md")
354(include "predicates.md")
355(include "iterators.md")
356
43e9d192
IB		 357;; -------------------------------------------------------------------
358;; Instruction types and attributes
359;; -------------------------------------------------------------------
360
026c3cfd 361; The "type" attribute is included here from AArch32 backend to be able
34b512f5
SN		 362; to share pipeline descriptions.
363(include "../arm/types.md")
43e9d192 364
a636bb40
KT		 365;; It is important to set the fp or simd attributes to yes when a pattern
366;; alternative uses the FP or SIMD register files, usually signified by use of
367;; the 'w' constraint. This will ensure that the alternative will be
368;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
369;; architecture extensions. If all the alternatives in a pattern use the
370;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
371;; or TARGET_SIMD.
372
488461d8
MM		 373;; Attributes of the architecture required to support the instruction (or
374;; alternative). This attribute is used to compute attribute "enabled", use type
375;; "any" to enable an alternative in all cases.
43e9d192 376
3c5af608 377(define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
90e6443f 378
488461d8 379(define_enum_attr "arch" "arches" (const_string "any"))
43e9d192 380
488461d8
MM		 381;; [For compatibility with Arm in pipeline models]
382;; Attribute that specifies whether or not the instruction touches fp
383;; registers.
384;; Note that this attribute is not used anywhere in either the arm or aarch64
385;; backends except in the scheduling description for xgene1. In that
386;; scheduling description this attribute is used to subclass the load_4 and
387;; load_8 types.
388(define_attr "fp" "no,yes"
389 (if_then_else
390 (eq_attr "arch" "fp")
391 (const_string "yes")
392 (const_string "no")))
393
394(define_attr "arch_enabled" "no,yes"
395 (if_then_else
396 (ior
397 (eq_attr "arch" "any")
398
3c5af608
MM		 399 (and (eq_attr "arch" "rcpc8_4")
400 (match_test "AARCH64_ISA_RCPC8_4"))
401
488461d8
MM		 402 (and (eq_attr "arch" "fp")
403 (match_test "TARGET_FLOAT"))
404
405 (and (eq_attr "arch" "simd")
406 (match_test "TARGET_SIMD"))
407
408 (and (eq_attr "arch" "fp16")
409 (match_test "TARGET_FP_F16INST"))
410
411 (and (eq_attr "arch" "sve")
412 (match_test "TARGET_SVE")))
413 (const_string "yes")
414 (const_string "no")))
43cacb12 415
43e9d192
IB		 416;; Attribute that controls whether an alternative is enabled or not.
417;; Currently it is only used to disable alternatives which touch fp or simd
488461d8
MM		 418;; registers when -mgeneral-regs-only is specified or to require a special
419;; architecture support.
420(define_attr "enabled" "no,yes" (attr "arch_enabled"))
43e9d192 421
973d2e01
TP		 422;; Attribute that specifies whether we are dealing with a branch to a
423;; label that is far away, i.e. further away than the maximum/minimum
424;; representable in a signed 21-bits number.
425;; 0 :=: no
426;; 1 :=: yes
427(define_attr "far_branch" "" (const_int 0))
428
a08acce8
RH		 429;; Attribute that specifies whether the alternative uses MOVPRFX.
430(define_attr "movprfx" "no,yes" (const_string "no"))
431
be178ecd
MM		 432;; Attribute to specify that an alternative has the length of a single
433;; instruction plus a speculation barrier.
434(define_attr "sls_length" "none,retbr,casesi" (const_string "none"))
435
a08acce8
RH		 436(define_attr "length" ""
437 (cond [(eq_attr "movprfx" "yes")
438 (const_int 8)
be178ecd
MM		 439
440 (eq_attr "sls_length" "retbr")
441 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4)
442 (match_test "TARGET_SB") (const_int 8)]
443 (const_int 12))
444
445 (eq_attr "sls_length" "casesi")
446 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16)
447 (match_test "TARGET_SB") (const_int 20)]
448 (const_int 24))
449 ]
450 (const_int 4)))
a08acce8 451
4adcec71
EM		 452;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
453;; no predicated insns.
454(define_attr "predicated" "yes,no" (const_string "no"))
455
3751345d
RE		 456;; Set to true on an insn that requires the speculation tracking state to be
457;; in the tracking register before the insn issues. Otherwise the compiler
458;; may chose to hold the tracking state encoded in SP.
459(define_attr "speculation_barrier" "true,false" (const_string "false"))
460
43e9d192
IB		 461;; -------------------------------------------------------------------
462;; Pipeline descriptions and scheduling
463;; -------------------------------------------------------------------
464
465;; Processor types.
466(include "aarch64-tune.md")
467
468;; Scheduling
c451f4d6 469(include "../arm/cortex-a53.md")
e0ae0288 470(include "../arm/cortex-a57.md")
b2ca46df 471(include "../arm/exynos-m1.md")
825ada35 472(include "falkor.md")
2e9d500e 473(include "saphira.md")
2d41ed58 474(include "thunderx.md")
a0f06fc9 475(include "../arm/xgene1.md")
f192030d 476(include "thunderx2t99.md")
8108dfde 477(include "tsv110.md")
fa477e45 478(include "thunderx3t110.md")
43e9d192
IB		 479
480;; -------------------------------------------------------------------
481;; Jumps and other miscellaneous insns
482;; -------------------------------------------------------------------
483
484(define_insn "indirect_jump"
485 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
486 ""
be178ecd
MM		 487 {
488 output_asm_insn ("br\\t%0", operands);
489 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
490 }
491 [(set_attr "type" "branch")
492 (set_attr "sls_length" "retbr")]
43e9d192
IB		 493)
494
495(define_insn "jump"
496 [(set (pc) (label_ref (match_operand 0 "" "")))]
497 ""
498 "b\\t%l0"
c1dfcec7 499 [(set_attr "type" "branch")]
43e9d192
IB		 500)
501
502(define_expand "cbranch<mode>4"
503 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 504 [(match_operand:GPI 1 "register_operand")
505 (match_operand:GPI 2 "aarch64_plus_operand")])
43e9d192
IB		 506 (label_ref (match_operand 3 "" ""))
507 (pc)))]
508 ""
509 "
510 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
511 operands[2]);
512 operands[2] = const0_rtx;
513 "
514)
515
516(define_expand "cbranch<mode>4"
517 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 518 [(match_operand:GPF 1 "register_operand")
519 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
43e9d192
IB		 520 (label_ref (match_operand 3 "" ""))
521 (pc)))]
522 ""
523 "
524 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
525 operands[2]);
526 operands[2] = const0_rtx;
527 "
528)
529
cf670503
ZC		 530(define_expand "cbranchcc4"
531 [(set (pc) (if_then_else
532 (match_operator 0 "aarch64_comparison_operator"
1bbffb87 533 [(match_operand 1 "cc_register")
cf670503
ZC		 534 (match_operand 2 "const0_operand")])
535 (label_ref (match_operand 3 "" ""))
536 (pc)))]
537 ""
538 "")
539
865257c4
RS		 540(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>"
541 [(set (match_operand:CC_ONLY 1 "cc_register" "")
542 (if_then_else:CC_ONLY
cf670503 543 (match_operator 4 "aarch64_comparison_operator"
c8012fbc 544 [(match_operand 0 "cc_register" "")
cf670503 545 (const_int 0)])
865257c4 546 (compare:CC_ONLY
c8012fbc
WD		 547 (match_operand:GPI 2 "register_operand" "r,r,r")
548 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
865257c4
RS		 549 (unspec:CC_ONLY
550 [(match_operand 5 "immediate_operand")]
551 UNSPEC_NZCV)))]
c8012fbc 552 ""
cf670503
ZC		 553 "@
554 ccmp\\t%<w>2, %<w>3, %k5, %m4
e2b691c4 555 ccmp\\t%<w>2, %3, %k5, %m4
cf670503
ZC		 556 ccmn\\t%<w>2, #%n3, %k5, %m4"
557 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
558)
559
865257c4
RS		 560(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>"
561 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
562 (if_then_else:CCFP_CCFPE
786e3c06
WD		 563 (match_operator 4 "aarch64_comparison_operator"
564 [(match_operand 0 "cc_register" "")
565 (const_int 0)])
865257c4 566 (compare:CCFP_CCFPE
786e3c06
WD		 567 (match_operand:GPF 2 "register_operand" "w")
568 (match_operand:GPF 3 "register_operand" "w"))
865257c4
RS		 569 (unspec:CCFP_CCFPE
570 [(match_operand 5 "immediate_operand")]
571 UNSPEC_NZCV)))]
786e3c06 572 "TARGET_FLOAT"
865257c4 573 "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4"
c297d256 574 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 575)
576
865257c4
RS		 577(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev"
578 [(set (match_operand:CC_ONLY 1 "cc_register" "")
579 (if_then_else:CC_ONLY
786e3c06
WD		 580 (match_operator 4 "aarch64_comparison_operator"
581 [(match_operand 0 "cc_register" "")
865257c4
RS		 582 (const_int 0)])
583 (unspec:CC_ONLY
584 [(match_operand 5 "immediate_operand")]
585 UNSPEC_NZCV)
586 (compare:CC_ONLY
587 (match_operand:GPI 2 "register_operand" "r,r,r")
588 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))))]
589 ""
590 "@
591 ccmp\\t%<w>2, %<w>3, %k5, %M4
592 ccmp\\t%<w>2, %3, %k5, %M4
593 ccmn\\t%<w>2, #%n3, %k5, %M4"
594 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
595)
596
597(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev"
598 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
599 (if_then_else:CCFP_CCFPE
600 (match_operator 4 "aarch64_comparison_operator"
601 [(match_operand 0 "cc_register" "")
602 (const_int 0)])
603 (unspec:CCFP_CCFPE
604 [(match_operand 5 "immediate_operand")]
605 UNSPEC_NZCV)
606 (compare:CCFP_CCFPE
786e3c06 607 (match_operand:GPF 2 "register_operand" "w")
865257c4 608 (match_operand:GPF 3 "register_operand" "w"))))]
786e3c06 609 "TARGET_FLOAT"
865257c4 610 "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4"
c297d256 611 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 612)
613
4f58fe36
KT		 614;; Expansion of signed mod by a power of 2 using CSNEG.
615;; For x0 % n where n is a power of 2 produce:
616;; negs x1, x0
617;; and x0, x0, #(n - 1)
618;; and x1, x1, #(n - 1)
619;; csneg x0, x0, x1, mi
620
621(define_expand "mod<mode>3"
1bbffb87
DZ		 622 [(match_operand:GPI 0 "register_operand")
623 (match_operand:GPI 1 "register_operand")
624 (match_operand:GPI 2 "const_int_operand")]
4f58fe36
KT		 625 ""
626 {
627 HOST_WIDE_INT val = INTVAL (operands[2]);
628
629 if (val <= 0
630 || exact_log2 (val) <= 0
631 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
632 FAIL;
633
634 rtx mask = GEN_INT (val - 1);
635
636 /* In the special case of x0 % 2 we can do the even shorter:
637 cmp x0, xzr
638 and x0, x0, 1
639 cneg x0, x0, lt. */
640 if (val == 2)
641 {
642 rtx masked = gen_reg_rtx (<MODE>mode);
643 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
644 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
645 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
646 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
647 DONE;
648 }
649
650 rtx neg_op = gen_reg_rtx (<MODE>mode);
651 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
652
653 /* Extract the condition register and mode. */
654 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
655 rtx cc_reg = SET_DEST (cmp);
656 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
657
658 rtx masked_pos = gen_reg_rtx (<MODE>mode);
659 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
660
661 rtx masked_neg = gen_reg_rtx (<MODE>mode);
662 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
663
664 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
665 masked_neg, masked_pos));
666 DONE;
667 }
668)
669
714e1b3b 670(define_insn "condjump"
43e9d192
IB		 671 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
672 [(match_operand 1 "cc_register" "") (const_int 0)])
673 (label_ref (match_operand 2 "" ""))
674 (pc)))]
675 ""
973d2e01 676 {
57d6f4d0
RS		 677 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
678 but the "." is required for SVE conditions. */
679 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
973d2e01 680 if (get_attr_length (insn) == 8)
57d6f4d0
RS		 681 return aarch64_gen_far_branch (operands, 2, "Lbcond",
682 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
973d2e01 683 else
57d6f4d0 684 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
973d2e01
TP		 685 }
686 [(set_attr "type" "branch")
687 (set (attr "length")
688 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
689 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
690 (const_int 4)
691 (const_int 8)))
692 (set (attr "far_branch")
693 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
694 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
695 (const_int 0)
696 (const_int 1)))]
43e9d192
IB		 697)
698
714e1b3b
KT		 699;; For a 24-bit immediate CST we can optimize the compare for equality
700;; and branch sequence from:
701;; mov x0, #imm1
702;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
703;; cmp x1, x0
704;; b<ne,eq> .Label
705;; into the shorter:
706;; sub x0, x1, #(CST & 0xfff000)
707;; subs x0, x0, #(CST & 0x000fff)
708;; b<ne,eq> .Label
e7ba492a 709(define_insn_and_split "*compare_condjump<GPI:mode>"
714e1b3b
KT		 710 [(set (pc) (if_then_else (EQL
711 (match_operand:GPI 0 "register_operand" "r")
712 (match_operand:GPI 1 "aarch64_imm24" "n"))
713 (label_ref:P (match_operand 2 "" ""))
714 (pc)))]
e7ba492a
RS		 715 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
716 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
714e1b3b
KT		 717 && !reload_completed"
718 "#"
719 "&& true"
720 [(const_int 0)]
721 {
722 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
723 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
e7ba492a
RS		 724 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
725 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
726 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
714e1b3b 727 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
e7ba492a
RS		 728 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
729 cc_reg, const0_rtx);
714e1b3b
KT		 730 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
731 DONE;
732 }
733)
734
43e9d192 735(define_expand "casesi"
1bbffb87
DZ		 736 [(match_operand:SI 0 "register_operand") ; Index
737 (match_operand:SI 1 "const_int_operand") ; Lower bound
738 (match_operand:SI 2 "const_int_operand") ; Total range
43e9d192
IB		 739 (match_operand:DI 3 "" "") ; Table label
740 (match_operand:DI 4 "" "")] ; Out of range label
741 ""
742 {
743 if (operands[1] != const0_rtx)
744 {
745 rtx reg = gen_reg_rtx (SImode);
746
747 /* Canonical RTL says that if you have:
748
749 (minus (X) (CONST))
750
751 then this should be emitted as:
752
753 (plus (X) (-CONST))
754
755 The use of trunc_int_for_mode ensures that the resulting
756 constant can be represented in SImode, this is important
757 for the corner case where operand[1] is INT_MIN. */
758
618ae596
JJ		 759 operands[1]
760 = GEN_INT (trunc_int_for_mode (-UINTVAL (operands[1]), SImode));
43e9d192
IB		 761
762 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
763 (operands[1], SImode))
764 operands[1] = force_reg (SImode, operands[1]);
765 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
766 operands[0] = reg;
767 }
768
769 if (!aarch64_plus_operand (operands[2], SImode))
770 operands[2] = force_reg (SImode, operands[2]);
771 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
772 const0_rtx),
773 operands[0], operands[2], operands[4]));
774
4033e62c 775 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
ec302910
JJ		 776 operands[2]
777 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
778 UNSPEC_CASESI);
779 operands[2] = gen_rtx_MEM (DImode, operands[2]);
780 MEM_READONLY_P (operands[2]) = 1;
781 MEM_NOTRAP_P (operands[2]) = 1;
782 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
43e9d192
IB		 783 DONE;
784 }
785)
786
ec302910
JJ		 787(define_expand "casesi_dispatch"
788 [(parallel
789 [(set (pc) (match_operand:DI 0 ""))
790 (clobber (reg:CC CC_REGNUM))
791 (clobber (match_scratch:DI 2))
792 (clobber (match_scratch:DI 3))
793 (use (label_ref:DI (match_operand 1 "")))])]
794 "")
795
796(define_insn "*casesi_dispatch"
43e9d192
IB		 797 [(parallel
798 [(set (pc)
799 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
800 (match_operand:SI 1 "register_operand" "r")]
801 UNSPEC_CASESI)))
802 (clobber (reg:CC CC_REGNUM))
803 (clobber (match_scratch:DI 3 "=r"))
804 (clobber (match_scratch:DI 4 "=r"))
ec302910 805 (use (label_ref:DI (match_operand 2 "" "")))])]
43e9d192
IB		 806 ""
807 "*
808 return aarch64_output_casesi (operands);
809 "
be178ecd 810 [(set_attr "sls_length" "casesi")
34b512f5 811 (set_attr "type" "branch")]
43e9d192
IB		 812)
813
814(define_insn "nop"
815 [(unspec[(const_int 0)] UNSPEC_NOP)]
816 ""
817 "nop"
c1dfcec7 818 [(set_attr "type" "no_insn")]
43e9d192
IB		 819)
820
79c6bb5f 821(define_insn "prefetch"
9bf2f779 822 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
79c6bb5f
GG		 823 (match_operand:QI 1 "const_int_operand" "")
824 (match_operand:QI 2 "const_int_operand" ""))]
825 ""
826 {
9bf2f779 827 const char * pftype[2][4] =
79c6bb5f 828 {
9bf2f779
KT		 829 {"prfm\\tPLDL1STRM, %0",
830 "prfm\\tPLDL3KEEP, %0",
831 "prfm\\tPLDL2KEEP, %0",
832 "prfm\\tPLDL1KEEP, %0"},
833 {"prfm\\tPSTL1STRM, %0",
834 "prfm\\tPSTL3KEEP, %0",
835 "prfm\\tPSTL2KEEP, %0",
836 "prfm\\tPSTL1KEEP, %0"},
79c6bb5f
GG		 837 };
838
839 int locality = INTVAL (operands[2]);
840
841 gcc_assert (IN_RANGE (locality, 0, 3));
842
9bf2f779
KT		 843 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
844 the address into a DImode MEM so that aarch64_print_operand knows
845 how to print it. */
846 operands[0] = gen_rtx_MEM (DImode, operands[0]);
79c6bb5f
GG		 847 return pftype[INTVAL(operands[1])][locality];
848 }
89b2133e 849 [(set_attr "type" "load_4")]
79c6bb5f
GG		 850)
851
cd93c60e
AP		 852(define_insn "trap"
853 [(trap_if (const_int 1) (const_int 8))]
854 ""
855 "brk #1000"
856 [(set_attr "type" "trap")])
857
43e9d192
IB		 858(define_expand "prologue"
859 [(clobber (const_int 0))]
860 ""
861 "
862 aarch64_expand_prologue ();
863 DONE;
864 "
865)
866
867(define_expand "epilogue"
868 [(clobber (const_int 0))]
869 ""
870 "
871 aarch64_expand_epilogue (false);
872 DONE;
873 "
874)
875
876(define_expand "sibcall_epilogue"
877 [(clobber (const_int 0))]
878 ""
879 "
880 aarch64_expand_epilogue (true);
881 DONE;
882 "
883)
884
885(define_insn "*do_return"
886 [(return)]
887 ""
db58fd89 888 {
be178ecd 889 const char *ret = NULL;
db58fd89 890 if (aarch64_return_address_signing_enabled ()
14d31404 891 && (TARGET_PAUTH))
8fc16d72
ST		 892 {
893 if (aarch64_ra_sign_key == AARCH64_KEY_B)
be178ecd 894 ret = "retab";
8fc16d72 895 else
be178ecd 896 ret = "retaa";
8fc16d72 897 }
be178ecd
MM		 898 else
899 ret = "ret";
900 output_asm_insn (ret, operands);
901 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
db58fd89 902 }
be178ecd
MM		 903 [(set_attr "type" "branch")
904 (set_attr "sls_length" "retbr")]
43e9d192
IB		 905)
906
4f942779
RL		 907(define_expand "return"
908 [(simple_return)]
909 "aarch64_use_return_insn_p ()"
910 ""
911)
912
913(define_insn "simple_return"
914 [(simple_return)]
ce9d2a37 915 ""
be178ecd
MM		 916 {
917 output_asm_insn ("ret", operands);
918 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
919 }
920 [(set_attr "type" "branch")
921 (set_attr "sls_length" "retbr")]
4f942779
RL		 922)
923
43e9d192
IB		 924(define_insn "*cb<optab><mode>1"
925 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
926 (const_int 0))
927 (label_ref (match_operand 1 "" ""))
928 (pc)))]
6e1eaca9 929 "!aarch64_track_speculation"
973d2e01
TP		 930 {
931 if (get_attr_length (insn) == 8)
932 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
933 else
934 return "<cbz>\\t%<w>0, %l1";
935 }
936 [(set_attr "type" "branch")
937 (set (attr "length")
938 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
939 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
940 (const_int 4)
941 (const_int 8)))
942 (set (attr "far_branch")
943 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
944 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
945 (const_int 0)
946 (const_int 1)))]
43e9d192
IB		 947)
948
949(define_insn "*tb<optab><mode>1"
950 [(set (pc) (if_then_else
951 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
952 (const_int 1)
e27bfda2
KT		 953 (match_operand 1
954 "aarch64_simd_shift_imm_<mode>" "n"))
43e9d192
IB		 955 (const_int 0))
956 (label_ref (match_operand 2 "" ""))
957 (pc)))
096e8448 958 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 959 "!aarch64_track_speculation"
096e8448
JW		 960 {
961 if (get_attr_length (insn) == 8)
962 {
973d2e01
TP		 963 if (get_attr_far_branch (insn) == 1)
964 return aarch64_gen_far_branch (operands, 2, "Ltb",
965 "<inv_tb>\\t%<w>0, %1, ");
966 else
967 {
968 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
969 return "tst\t%<w>0, %1\;<bcond>\t%l2";
970 }
096e8448
JW		 971 }
972 else
973 return "<tbz>\t%<w>0, %1, %l2";
974 }
c1dfcec7 975 [(set_attr "type" "branch")
43e9d192
IB		 976 (set (attr "length")
977 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
978 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
979 (const_int 4)
973d2e01
TP		 980 (const_int 8)))
981 (set (attr "far_branch")
982 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
983 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
984 (const_int 0)
985 (const_int 1)))]
986
43e9d192
IB		 987)
988
989(define_insn "*cb<optab><mode>1"
990 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
991 (const_int 0))
992 (label_ref (match_operand 1 "" ""))
993 (pc)))
096e8448 994 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 995 "!aarch64_track_speculation"
096e8448
JW		 996 {
997 if (get_attr_length (insn) == 8)
998 {
973d2e01
TP		 999 if (get_attr_far_branch (insn) == 1)
1000 return aarch64_gen_far_branch (operands, 1, "Ltb",
1001 "<inv_tb>\\t%<w>0, <sizem1>, ");
1002 else
1003 {
1004 char buf[64];
1005 uint64_t val = ((uint64_t) 1)
1006 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
1007 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
1008 output_asm_insn (buf, operands);
1009 return "<bcond>\t%l1";
1010 }
096e8448
JW		 1011 }
1012 else
1013 return "<tbz>\t%<w>0, <sizem1>, %l1";
1014 }
c1dfcec7 1015 [(set_attr "type" "branch")
43e9d192
IB		 1016 (set (attr "length")
1017 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
1018 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
1019 (const_int 4)
973d2e01
TP		 1020 (const_int 8)))
1021 (set (attr "far_branch")
1022 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
1023 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
1024 (const_int 0)
1025 (const_int 1)))]
43e9d192
IB		 1026)
1027
1028;; -------------------------------------------------------------------
1029;; Subroutine calls and sibcalls
1030;; -------------------------------------------------------------------
1031
1032(define_expand "call"
08cc4d92
RS		 1033 [(parallel
1034 [(call (match_operand 0 "memory_operand")
1035 (match_operand 1 "general_operand"))
1036 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1037 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 1038 ""
1039 "
1040 {
08cc4d92 1041 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
78607708 1042 DONE;
43e9d192
IB		 1043 }"
1044)
1045
47210a04 1046(define_insn "*call_insn"
96b7f495 1047 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucr, Usf"))
43e9d192 1048 (match_operand 1 "" ""))
08cc4d92 1049 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 1050 (clobber (reg:DI LR_REGNUM))]
1051 ""
47210a04 1052 "@
96b7f495 1053 * return aarch64_indirect_call_asm (operands[0]);
d6591257 1054 bl\\t%c0"
96b7f495 1055 [(set_attr "type" "call, call")])
43e9d192
IB		 1056
1057(define_expand "call_value"
08cc4d92
RS		 1058 [(parallel
1059 [(set (match_operand 0 "")
1060 (call (match_operand 1 "memory_operand")
1061 (match_operand 2 "general_operand")))
1062 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1063 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 1064 ""
1065 "
1066 {
08cc4d92 1067 aarch64_expand_call (operands[0], operands[1], operands[3], false);
78607708 1068 DONE;
43e9d192
IB		 1069 }"
1070)
1071
47210a04 1072(define_insn "*call_value_insn"
43e9d192 1073 [(set (match_operand 0 "" "")
96b7f495 1074 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "Ucr, Usf"))
43e9d192 1075 (match_operand 2 "" "")))
08cc4d92 1076 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 1077 (clobber (reg:DI LR_REGNUM))]
1078 ""
47210a04 1079 "@
96b7f495 1080 * return aarch64_indirect_call_asm (operands[1]);
d6591257 1081 bl\\t%c1"
47210a04 1082 [(set_attr "type" "call, call")]
43e9d192
IB		 1083)
1084
1085(define_expand "sibcall"
08cc4d92
RS		 1086 [(parallel
1087 [(call (match_operand 0 "memory_operand")
1088 (match_operand 1 "general_operand"))
1089 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1090 (return)])]
43e9d192
IB		 1091 ""
1092 {
08cc4d92 1093 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
78607708 1094 DONE;
43e9d192
IB		 1095 }
1096)
1097
1098(define_expand "sibcall_value"
08cc4d92
RS		 1099 [(parallel
1100 [(set (match_operand 0 "")
1101 (call (match_operand 1 "memory_operand")
1102 (match_operand 2 "general_operand")))
1103 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1104 (return)])]
43e9d192
IB		 1105 ""
1106 {
08cc4d92 1107 aarch64_expand_call (operands[0], operands[1], operands[3], true);
78607708 1108 DONE;
43e9d192
IB		 1109 }
1110)
1111
1112(define_insn "*sibcall_insn"
fee9ba42 1113 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1114 (match_operand 1 ""))
1115 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1116 (return)]
fee9ba42 1117 "SIBLING_CALL_P (insn)"
be178ecd
MM		 1118 {
1119 if (which_alternative == 0)
1120 {
1121 output_asm_insn ("br\\t%0", operands);
1122 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1123 }
1124 return "b\\t%c0";
1125 }
1126 [(set_attr "type" "branch, branch")
1127 (set_attr "sls_length" "retbr,none")]
43e9d192
IB		 1128)
1129
1130(define_insn "*sibcall_value_insn"
08cc4d92 1131 [(set (match_operand 0 "")
aaea00aa
JG		 1132 (call (mem:DI
1133 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1134 (match_operand 2 "")))
1135 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1136 (return)]
fee9ba42 1137 "SIBLING_CALL_P (insn)"
be178ecd
MM		 1138 {
1139 if (which_alternative == 0)
1140 {
1141 output_asm_insn ("br\\t%1", operands);
1142 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1143 }
1144 return "b\\t%c1";
1145 }
1146 [(set_attr "type" "branch, branch")
1147 (set_attr "sls_length" "retbr,none")]
43e9d192
IB		 1148)
1149
1150;; Call subroutine returning any type.
1151
1152(define_expand "untyped_call"
1153 [(parallel [(call (match_operand 0 "")
1154 (const_int 0))
1155 (match_operand 1 "")
1156 (match_operand 2 "")])]
1157 ""
1158{
1159 int i;
1160
08cc4d92
RS		 1161 /* Untyped calls always use the default ABI. It's only possible to use
1162 ABI variants if we know the type of the target function. */
1163 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
43e9d192
IB		 1164
1165 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1166 {
1167 rtx set = XVECEXP (operands[2], 0, i);
1168 emit_move_insn (SET_DEST (set), SET_SRC (set));
1169 }
1170
1171 /* The optimizer does not know that the call sets the function value
1172 registers we stored in the result block. We avoid problems by
1173 claiming that all hard registers are used and clobbered at this
1174 point. */
1175 emit_insn (gen_blockage ());
1176 DONE;
1177})
1178
1179;; -------------------------------------------------------------------
1180;; Moves
1181;; -------------------------------------------------------------------
1182
1183(define_expand "mov<mode>"
1bbffb87
DZ		 1184 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1185 (match_operand:SHORT 1 "general_operand"))]
43e9d192
IB		 1186 ""
1187 "
1188 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1189 operands[1] = force_reg (<MODE>mode, operands[1]);
43cacb12
RS		 1190
1191 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1192 {
1193 aarch64_expand_mov_immediate (operands[0], operands[1]);
1194 DONE;
1195 }
43e9d192
IB		 1196 "
1197)
1198
1199(define_insn "*mov<mode>_aarch64"
43d0a8ee
WD		 1200 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1201 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
43e9d192
IB		 1202 "(register_operand (operands[0], <MODE>mode)
1203 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
b7342d25
IB		 1204{
1205 switch (which_alternative)
1206 {
1207 case 0:
1208 return "mov\t%w0, %w1";
1209 case 1:
1210 return "mov\t%w0, %1";
1211 case 2:
1212 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1213 <MODE>mode);
1214 case 3:
43cacb12 1215 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
b7342d25 1216 case 4:
43cacb12 1217 return "ldr<size>\t%w0, %1";
b7342d25 1218 case 5:
43cacb12 1219 return "ldr\t%<size>0, %1";
b7342d25 1220 case 6:
43cacb12 1221 return "str<size>\t%w1, %0";
b7342d25 1222 case 7:
43cacb12 1223 return "str\t%<size>1, %0";
b7342d25 1224 case 8:
43cacb12 1225 return "umov\t%w0, %1.<v>[0]";
b7342d25 1226 case 9:
43cacb12
RS		 1227 return "dup\t%0.<Vallxd>, %w1";
1228 case 10:
21216998 1229 return "dup\t%<Vetype>0, %1.<v>[0]";
b7342d25
IB		 1230 default:
1231 gcc_unreachable ();
1232 }
1233}
43cacb12
RS		 1234 ;; The "mov_imm" type for CNT is just a placeholder.
1235 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1236 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
488461d8 1237 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
43e9d192
IB		 1238)
1239
1240(define_expand "mov<mode>"
1bbffb87
DZ		 1241 [(set (match_operand:GPI 0 "nonimmediate_operand")
1242 (match_operand:GPI 1 "general_operand"))]
43e9d192
IB		 1243 ""
1244 "
01b9402c
KT		 1245 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1246 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
141a3ccf
KT		 1247 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1248 DONE;
1249
43e9d192
IB		 1250 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1251 operands[1] = force_reg (<MODE>mode, operands[1]);
1252
48c79f05
RS		 1253 /* Lower moves of symbolic constants into individual instructions.
1254 Doing this now is sometimes necessary for correctness, since some
1255 sequences require temporary pseudo registers. Lowering now is also
1256 often better for optimization, since more RTL passes get the
1257 chance to optimize the individual instructions.
1258
1259 When called after RA, also split multi-instruction moves into
1260 smaller pieces now, since we can't be sure that sure that there
1261 will be a following split pass. */
1262 if (CONST_INT_P (operands[1])
1263 ? (reload_completed
1264 && !aarch64_mov_imm_operand (operands[1], <MODE>mode))
1265 : CONSTANT_P (operands[1]))
82614948
RR		 1266 {
1267 aarch64_expand_mov_immediate (operands[0], operands[1]);
1268 DONE;
1269 }
43e9d192
IB		 1270 "
1271)
1272
82614948 1273(define_insn_and_split "*movsi_aarch64"
a195c727
WD		 1274 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, r, w,r,w, w")
1275 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usw,Usa,Ush,rZ,w,w,Ds"))]
43e9d192
IB		 1276 "(register_operand (operands[0], SImode)
1277 || aarch64_reg_or_zero (operands[1], SImode))"
1278 "@
28514dda
YZ		 1279 mov\\t%w0, %w1
1280 mov\\t%w0, %w1
43e9d192
IB		 1281 mov\\t%w0, %w1
1282 mov\\t%w0, %1
82614948 1283 #
43cacb12 1284 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1285 ldr\\t%w0, %1
0b064172 1286 ldr\\t%s0, %1
43e9d192 1287 str\\t%w1, %0
0b064172 1288 str\\t%s1, %0
a195c727 1289 adrp\\t%x0, %A1\;ldr\\t%w0, [%x0, %L1]
d6591257 1290 adr\\t%x0, %c1
28514dda 1291 adrp\\t%x0, %A1
43e9d192
IB		 1292 fmov\\t%s0, %w1
1293 fmov\\t%w0, %s1
a2170965
TC		 1294 fmov\\t%s0, %s1
1295 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1296 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
7263fa9f 1297 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1298 [(const_int 0)]
1299 "{
1300 aarch64_expand_mov_immediate (operands[0], operands[1]);
1301 DONE;
1302 }"
43cacb12
RS		 1303 ;; The "mov_imm" type for CNT is just a placeholder.
1304 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
a195c727
WD		 1305 load_4,store_4,store_4,load_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1306 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,*,fp,fp,fp,simd")
1307 (set_attr "length" "4,4,4,4,*, 4,4, 4,4, 4,8,4,4, 4, 4, 4, 4")
1308]
43e9d192
IB		 1309)
1310
82614948 1311(define_insn_and_split "*movdi_aarch64"
a195c727
WD		 1312 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, r, w,r,w, w")
1313 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usw,Usa,Ush,rZ,w,w,Dd"))]
43e9d192
IB		 1314 "(register_operand (operands[0], DImode)
1315 || aarch64_reg_or_zero (operands[1], DImode))"
1316 "@
1317 mov\\t%x0, %x1
1318 mov\\t%0, %x1
1319 mov\\t%x0, %1
1320 mov\\t%x0, %1
9de00935 1321 mov\\t%w0, %1
82614948 1322 #
43cacb12 1323 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1324 ldr\\t%x0, %1
0b064172 1325 ldr\\t%d0, %1
43e9d192 1326 str\\t%x1, %0
0b064172 1327 str\\t%d1, %0
a195c727 1328 * return TARGET_ILP32 ? \"adrp\\t%0, %A1\;ldr\\t%w0, [%0, %L1]\" : \"adrp\\t%0, %A1\;ldr\\t%0, [%0, %L1]\";
d6591257 1329 adr\\t%x0, %c1
43e9d192
IB		 1330 adrp\\t%x0, %A1
1331 fmov\\t%d0, %x1
1332 fmov\\t%x0, %d1
1333 fmov\\t%d0, %d1
a2170965 1334 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
a195c727 1335 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode)
7263fa9f 1336 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1337 [(const_int 0)]
1338 "{
1339 aarch64_expand_mov_immediate (operands[0], operands[1]);
1340 DONE;
1341 }"
43cacb12
RS		 1342 ;; The "mov_imm" type for CNTD is just a placeholder.
1343 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
a195c727
WD		 1344 load_8,load_8,store_8,store_8,load_8,adr,adr,f_mcr,f_mrc,
1345 fmov,neon_move")
1346 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,*,fp,fp,fp,simd")
1347 (set_attr "length" "4,4,4,4,4,*, 4,4, 4,4, 4,8,4,4, 4, 4, 4, 4")]
43e9d192
IB		 1348)
1349
1350(define_insn "insv_imm<mode>"
1351 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1352 (const_int 16)
0b81f26c
JG		 1353 (match_operand:GPI 1 "const_int_operand" "n"))
1354 (match_operand:GPI 2 "const_int_operand" "n"))]
50d38551
IB		 1355 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1356 && UINTVAL (operands[1]) % 16 == 0"
a05c0ddf 1357 "movk\\t%<w>0, %X2, lsl %1"
99e6af15 1358 [(set_attr "type" "mov_imm")]
43e9d192
IB		 1359)
1360
bba0c624
RS		 1361;; Match MOVK as a normal AND and IOR operation.
1362(define_insn "aarch64_movk<mode>"
1363 [(set (match_operand:GPI 0 "register_operand" "=r")
1364 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
1365 (match_operand:GPI 2 "const_int_operand"))
1366 (match_operand:GPI 3 "const_int_operand")))]
1367 "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1368 rtx_mode_t (operands[3], <MODE>mode)) >= 0"
1369 {
1370 int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1371 rtx_mode_t (operands[3], <MODE>mode));
1372 operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode);
1373 operands[3] = gen_int_mode (shift, SImode);
1374 return "movk\\t%<w>0, #%X2, lsl %3";
1375 }
1ccdd460 1376 [(set_attr "type" "mov_imm")]
bba0c624
RS		 1377)
1378
43e9d192 1379(define_expand "movti"
1bbffb87
DZ		 1380 [(set (match_operand:TI 0 "nonimmediate_operand")
1381 (match_operand:TI 1 "general_operand"))]
43e9d192
IB		 1382 ""
1383 "
1384 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1385 operands[1] = force_reg (TImode, operands[1]);
43cacb12
RS		 1386
1387 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1388 {
1389 emit_move_insn (gen_lowpart (DImode, operands[0]),
1390 gen_lowpart (DImode, operands[1]));
1391 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1392 DONE;
1393 }
43e9d192
IB		 1394 "
1395)
1396
1397(define_insn "*movti_aarch64"
1398 [(set (match_operand:TI 0
d4f9e819 1399 "nonimmediate_operand" "= r,w,w, r,w,r,m,m,w,m")
43e9d192 1400 (match_operand:TI 1
d4f9e819 1401 "aarch64_movti_operand" " rUti,Z,r, w,w,m,r,Z,m,w"))]
43e9d192
IB		 1402 "(register_operand (operands[0], TImode)
1403 || aarch64_reg_or_zero (operands[1], TImode))"
1404 "@
1405 #
d4f9e819 1406 movi\\t%0.2d, #0
43e9d192
IB		 1407 #
1408 #
9fe9816b 1409 mov\\t%0.16b, %1.16b
43e9d192
IB		 1410 ldp\\t%0, %H0, %1
1411 stp\\t%1, %H1, %0
1412 stp\\txzr, xzr, %0
1413 ldr\\t%q0, %1
1414 str\\t%q1, %0"
d4f9e819 1415 [(set_attr "type" "multiple,neon_move,f_mcr,f_mrc,neon_logic_q, \
db46a2e6
JG		 1416 load_16,store_16,store_16,\
1417 load_16,store_16")
d4f9e819
RS		 1418 (set_attr "length" "8,4,8,8,4,4,4,4,4,4")
1419 (set_attr "arch" "*,simd,*,*,simd,*,*,*,fp,fp")]
78ec3036 1420)
43e9d192
IB		 1421
1422;; Split a TImode register-register or register-immediate move into
1423;; its component DImode pieces, taking care to handle overlapping
1424;; source and dest registers.
1425(define_split
1426 [(set (match_operand:TI 0 "register_operand" "")
1427 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1428 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1429 [(const_int 0)]
1430{
1431 aarch64_split_128bit_move (operands[0], operands[1]);
1432 DONE;
1433})
1434
1435(define_expand "mov<mode>"
abbe1ed2
SMW		 1436 [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand")
1437 (match_operand:GPF_TF_F16_MOV 1 "general_operand"))]
43e9d192 1438 ""
261fb553 1439 {
43e9d192 1440 if (!TARGET_FLOAT)
261fb553 1441 {
fc29dfc9 1442 aarch64_err_no_fpadvsimd (<MODE>mode);
248e357f
FY		 1443 machine_mode intmode
1444 = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require ();
1445 emit_move_insn (gen_lowpart (intmode, operands[0]),
1446 gen_lowpart (intmode, operands[1]));
1447 DONE;
261fb553 1448 }
43e9d192 1449
490948ca
JW		 1450 if (GET_CODE (operands[0]) == MEM
1451 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1452 && aarch64_float_const_zero_rtx_p (operands[1])))
43e9d192 1453 operands[1] = force_reg (<MODE>mode, operands[1]);
261fb553 1454 }
43e9d192
IB		 1455)
1456
abbe1ed2
SMW		 1457(define_insn "*mov<mode>_aarch64"
1458 [(set (match_operand:HFBF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1459 (match_operand:HFBF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1460 "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode)
1461 || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))"
c2ec330c 1462 "@
523d7207 1463 movi\\t%0.4h, #0
90e6443f 1464 fmov\\t%h0, %w1
d89fe63e 1465 dup\\t%w0.4h, %w1
c2ec330c
AL		 1466 umov\\t%w0, %1.h[0]
1467 mov\\t%0.h[0], %1.h[0]
90e6443f 1468 fmov\\t%h0, %1
980902fb 1469 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
c2ec330c
AL		 1470 ldr\\t%h0, %1
1471 str\\t%h1, %0
1472 ldrh\\t%w0, %1
1473 strh\\t%w1, %0
1474 mov\\t%w0, %w1"
d89fe63e 1475 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
89b2133e 1476 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
488461d8 1477 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
c2ec330c
AL		 1478)
1479
43e9d192 1480(define_insn "*movsf_aarch64"
90e6443f
TC		 1481 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1482 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
43e9d192 1483 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
701170a4 1484 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
43e9d192 1485 "@
523d7207 1486 movi\\t%0.2s, #0
43e9d192
IB		 1487 fmov\\t%s0, %w1
1488 fmov\\t%w0, %s1
1489 fmov\\t%s0, %s1
3520f7cc 1490 fmov\\t%s0, %1
90e6443f 1491 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
43e9d192
IB		 1492 ldr\\t%s0, %1
1493 str\\t%s1, %0
1494 ldr\\t%w0, %1
1495 str\\t%w1, %0
90e6443f
TC		 1496 mov\\t%w0, %w1
1497 mov\\t%w0, %1"
1498 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
89b2133e 1499 f_loads,f_stores,load_4,store_4,mov_reg,\
90e6443f 1500 fconsts")
488461d8 1501 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1502)
1503
1504(define_insn "*movdf_aarch64"
90e6443f
TC		 1505 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1506 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
43e9d192 1507 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
701170a4 1508 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
43e9d192 1509 "@
523d7207 1510 movi\\t%d0, #0
43e9d192
IB		 1511 fmov\\t%d0, %x1
1512 fmov\\t%x0, %d1
1513 fmov\\t%d0, %d1
3520f7cc 1514 fmov\\t%d0, %1
90e6443f 1515 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
43e9d192
IB		 1516 ldr\\t%d0, %1
1517 str\\t%d1, %0
1518 ldr\\t%x0, %1
1519 str\\t%x1, %0
90e6443f
TC		 1520 mov\\t%x0, %x1
1521 mov\\t%x0, %1"
1522 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
db46a2e6 1523 f_loadd,f_stored,load_8,store_8,mov_reg,\
90e6443f 1524 fconstd")
488461d8 1525 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
90e6443f
TC		 1526)
1527
1528(define_split
1529 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1530 (match_operand:GPF_HF 1 "general_operand"))]
1531 "can_create_pseudo_p ()
1532 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1533 && !aarch64_float_const_representable_p (operands[1])
1534 && aarch64_float_const_rtx_p (operands[1])"
1535 [(const_int 0)]
1536 {
1537 unsigned HOST_WIDE_INT ival;
1538 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1539 FAIL;
1540
1541 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1542 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1543 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1544 DONE;
1545 }
43e9d192
IB		 1546)
1547
43e9d192
IB		 1548(define_insn "*movtf_aarch64"
1549 [(set (match_operand:TF 0
d4f9e819 1550 "nonimmediate_operand" "=w,?r ,w ,?r,w,?w,w,m,?r,m ,m")
43e9d192 1551 (match_operand:TF 1
d4f9e819 1552 "general_operand" " w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))]
43e9d192 1553 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
6a0f8c01 1554 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
43e9d192 1555 "@
9fe9816b 1556 mov\\t%0.16b, %1.16b
12dc6974
SN		 1557 #
1558 #
1559 #
43e9d192
IB		 1560 movi\\t%0.2d, #0
1561 fmov\\t%s0, wzr
1562 ldr\\t%q0, %1
1563 str\\t%q1, %0
1564 ldp\\t%0, %H0, %1
6a0f8c01
JW		 1565 stp\\t%1, %H1, %0
1566 stp\\txzr, xzr, %0"
04e34811 1567 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
db46a2e6 1568 f_loadd,f_stored,load_16,store_16,store_16")
6a0f8c01 1569 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
488461d8 1570 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1571)
1572
12dc6974
SN		 1573(define_split
1574 [(set (match_operand:TF 0 "register_operand" "")
d4f9e819 1575 (match_operand:TF 1 "nonmemory_operand" ""))]
12dc6974
SN		 1576 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1577 [(const_int 0)]
1578 {
1579 aarch64_split_128bit_move (operands[0], operands[1]);
1580 DONE;
1581 }
1582)
1583
0caf592d
KT		 1584(define_insn "aarch64_cpymemdi"
1585 [(parallel [
1586 (set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
1587 (clobber (match_operand:DI 0 "register_operand" "+&r"))
1588 (clobber (match_operand:DI 1 "register_operand" "+&r"))
1589 (set (mem:BLK (match_dup 0))
1590 (unspec:BLK [(mem:BLK (match_dup 1)) (match_dup 2)] UNSPEC_CPYMEM))])]
1591 "TARGET_MOPS"
1592 "cpyfp\t[%x0]!, [%x1]!, %x2!\;cpyfm\t[%x0]!, [%x1]!, %x2!\;cpyfe\t[%x0]!, [%x1]!, %x2!"
1593 [(set_attr "length" "12")]
1594)
1595
e2c75eea
JG		 1596;; 0 is dst
1597;; 1 is src
76715c32 1598;; 2 is size of copy in bytes
e2c75eea
JG		 1599;; 3 is alignment
1600
76715c32 1601(define_expand "cpymemdi"
e2c75eea
JG		 1602 [(match_operand:BLK 0 "memory_operand")
1603 (match_operand:BLK 1 "memory_operand")
0caf592d 1604 (match_operand:DI 2 "general_operand")
e2c75eea 1605 (match_operand:DI 3 "immediate_operand")]
0caf592d 1606 "!STRICT_ALIGNMENT || TARGET_MOPS"
e2c75eea 1607{
76715c32 1608 if (aarch64_expand_cpymem (operands))
e2c75eea
JG		 1609 DONE;
1610 FAIL;
1611}
1612)
1613
bb768f8b
KT		 1614(define_insn "aarch64_movmemdi"
1615 [(parallel [
1616 (set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
1617 (clobber (match_operand:DI 0 "register_operand" "+&r"))
1618 (clobber (match_operand:DI 1 "register_operand" "+&r"))
1619 (set (mem:BLK (match_dup 0))
1620 (unspec:BLK [(mem:BLK (match_dup 1)) (match_dup 2)] UNSPEC_MOVMEM))])]
1621 "TARGET_MOPS"
1622 "cpyp\t[%x0]!, [%x1]!, %x2!\;cpym\t[%x0]!, [%x1]!, %x2!\;cpye\t[%x0]!, [%x1]!, %x2!"
1623 [(set_attr "length" "12")]
1624)
1625
1626;; 0 is dst
1627;; 1 is src
1628;; 2 is size of copy in bytes
1629;; 3 is alignment
1630
1631(define_expand "movmemdi"
1632 [(match_operand:BLK 0 "memory_operand")
1633 (match_operand:BLK 1 "memory_operand")
1634 (match_operand:DI 2 "general_operand")
1635 (match_operand:DI 3 "immediate_operand")]
1636 "TARGET_MOPS"
1637{
1638 rtx sz_reg = operands[2];
1639 /* For constant-sized memmoves check the threshold.
1640 FIXME: We should add a non-MOPS memmove expansion for smaller,
1641 constant-sized memmove to avoid going to a libcall. */
1642 if (CONST_INT_P (sz_reg)
1643 && INTVAL (sz_reg) < aarch64_mops_memmove_size_threshold)
1644 FAIL;
1645
1646 rtx addr_dst = XEXP (operands[0], 0);
1647 rtx addr_src = XEXP (operands[1], 0);
1648
1649 if (!REG_P (sz_reg))
1650 sz_reg = force_reg (DImode, sz_reg);
1651 if (!REG_P (addr_dst))
1652 addr_dst = force_reg (DImode, addr_dst);
1653 if (!REG_P (addr_src))
1654 addr_src = force_reg (DImode, addr_src);
1655 emit_insn (gen_aarch64_movmemdi (addr_dst, addr_src, sz_reg));
1656 DONE;
1657}
1658)
1659
d3bd985e
KT		 1660(define_insn "aarch64_setmemdi"
1661 [(parallel [
1662 (set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
1663 (clobber (match_operand:DI 0 "register_operand" "+&r"))
1664 (set (mem:BLK (match_dup 0))
1665 (unspec:BLK [(match_operand:QI 1 "aarch64_reg_or_zero" "rZ")
1666 (match_dup 2)] UNSPEC_SETMEM))])]
1667 "TARGET_MOPS"
1668 "setp\t[%x0]!, %x2!, %x1\;setm\t[%x0]!, %x2!, %x1\;sete\t[%x0]!, %x2!, %x1"
1669 [(set_attr "length" "12")]
1670)
1671
54bbde55
SD		 1672;; 0 is dst
1673;; 1 is val
1674;; 2 is size of copy in bytes
1675;; 3 is alignment
54bbde55
SD		 1676(define_expand "setmemdi"
1677 [(set (match_operand:BLK 0 "memory_operand") ;; Dest
1678 (match_operand:QI 2 "nonmemory_operand")) ;; Value
d3bd985e 1679 (use (match_operand:DI 1 "general_operand")) ;; Length
54bbde55 1680 (match_operand 3 "immediate_operand")] ;; Align
d3bd985e
KT		 1681 "TARGET_SIMD || TARGET_MOPS"
1682 {
54bbde55
SD		 1683 if (aarch64_expand_setmem (operands))
1684 DONE;
1685
1686 FAIL;
1687})
1688
43e9d192
IB		 1689;; Operands 1 and 3 are tied together by the final condition; so we allow
1690;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1691(define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1692 [(set (match_operand:SX 0 "register_operand" "=r,w")
1693 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1694 (set (match_operand:SX2 2 "register_operand" "=r,w")
1695 (match_operand:SX2 3 "memory_operand" "m,m"))]
1696 "rtx_equal_p (XEXP (operands[3], 0),
1697 plus_constant (Pmode,
1698 XEXP (operands[1], 0),
1699 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc 1700 "@
2c272091
RS		 1701 ldp\\t%w0, %w2, %z1
1702 ldp\\t%s0, %s2, %z1"
89b2133e 1703 [(set_attr "type" "load_8,neon_load1_2reg")
488461d8 1704 (set_attr "arch" "*,fp")]
43e9d192
IB		 1705)
1706
dfe1da23
JW		 1707;; Storing different modes that can still be merged
1708(define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1709 [(set (match_operand:DX 0 "register_operand" "=r,w")
1710 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1711 (set (match_operand:DX2 2 "register_operand" "=r,w")
1712 (match_operand:DX2 3 "memory_operand" "m,m"))]
1713 "rtx_equal_p (XEXP (operands[3], 0),
1714 plus_constant (Pmode,
1715 XEXP (operands[1], 0),
1716 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc 1717 "@
2c272091
RS		 1718 ldp\\t%x0, %x2, %z1
1719 ldp\\t%d0, %d2, %z1"
db46a2e6 1720 [(set_attr "type" "load_16,neon_load1_2reg")
488461d8 1721 (set_attr "arch" "*,fp")]
350013bc
BC		 1722)
1723
a0d0b980
SE		 1724(define_insn "load_pair_dw_tftf"
1725 [(set (match_operand:TF 0 "register_operand" "=w")
1726 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1727 (set (match_operand:TF 2 "register_operand" "=w")
1728 (match_operand:TF 3 "memory_operand" "m"))]
1729 "TARGET_SIMD
1730 && rtx_equal_p (XEXP (operands[3], 0),
1731 plus_constant (Pmode,
1732 XEXP (operands[1], 0),
1733 GET_MODE_SIZE (TFmode)))"
2c272091 1734 "ldp\\t%q0, %q2, %z1"
a0d0b980
SE		 1735 [(set_attr "type" "neon_ldp_q")
1736 (set_attr "fp" "yes")]
1737)
1738
43e9d192
IB		 1739;; Operands 0 and 2 are tied together by the final condition; so we allow
1740;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1741(define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1742 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1743 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1744 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1745 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1746 "rtx_equal_p (XEXP (operands[2], 0),
1747 plus_constant (Pmode,
1748 XEXP (operands[0], 0),
1749 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc 1750 "@
2c272091
RS		 1751 stp\\t%w1, %w3, %z0
1752 stp\\t%s1, %s3, %z0"
89b2133e 1753 [(set_attr "type" "store_8,neon_store1_2reg")
488461d8 1754 (set_attr "arch" "*,fp")]
350013bc
BC		 1755)
1756
dfe1da23
JW		 1757;; Storing different modes that can still be merged
1758(define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1759 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1760 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1761 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1762 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1763 "rtx_equal_p (XEXP (operands[2], 0),
1764 plus_constant (Pmode,
1765 XEXP (operands[0], 0),
1766 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc 1767 "@
2c272091
RS		 1768 stp\\t%x1, %x3, %z0
1769 stp\\t%d1, %d3, %z0"
db46a2e6 1770 [(set_attr "type" "store_16,neon_store1_2reg")
488461d8 1771 (set_attr "arch" "*,fp")]
43e9d192
IB		 1772)
1773
a0d0b980
SE		 1774(define_insn "store_pair_dw_tftf"
1775 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1776 (match_operand:TF 1 "register_operand" "w"))
1777 (set (match_operand:TF 2 "memory_operand" "=m")
1778 (match_operand:TF 3 "register_operand" "w"))]
1779 "TARGET_SIMD &&
1780 rtx_equal_p (XEXP (operands[2], 0),
1781 plus_constant (Pmode,
1782 XEXP (operands[0], 0),
1783 GET_MODE_SIZE (TFmode)))"
2c272091 1784 "stp\\t%q1, %q3, %z0"
a0d0b980
SE		 1785 [(set_attr "type" "neon_stp_q")
1786 (set_attr "fp" "yes")]
1787)
1788
3e322b3f
JW		 1789;; Load pair with post-index writeback. This is primarily used in function
1790;; epilogues.
28514dda 1791(define_insn "loadwb_pair<GPI:mode>_<P:mode>"
43e9d192 1792 [(parallel
28514dda
YZ		 1793 [(set (match_operand:P 0 "register_operand" "=k")
1794 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1795 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
43e9d192 1796 (set (match_operand:GPI 2 "register_operand" "=r")
3e322b3f 1797 (mem:GPI (match_dup 1)))
43e9d192 1798 (set (match_operand:GPI 3 "register_operand" "=r")
28514dda
YZ		 1799 (mem:GPI (plus:P (match_dup 1)
1800 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1801 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1802 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1803 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
43e9d192
IB		 1804)
1805
159313d9
JW		 1806(define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1807 [(parallel
1808 [(set (match_operand:P 0 "register_operand" "=k")
1809 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1810 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
159313d9 1811 (set (match_operand:GPF 2 "register_operand" "=w")
3e322b3f 1812 (mem:GPF (match_dup 1)))
159313d9
JW		 1813 (set (match_operand:GPF 3 "register_operand" "=w")
1814 (mem:GPF (plus:P (match_dup 1)
1815 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1816 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1817 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
159313d9
JW		 1818 [(set_attr "type" "neon_load1_2reg")]
1819)
1820
a0d0b980
SE		 1821(define_insn "loadwb_pair<TX:mode>_<P:mode>"
1822 [(parallel
1823 [(set (match_operand:P 0 "register_operand" "=k")
1824 (plus:P (match_operand:P 1 "register_operand" "0")
1825 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1826 (set (match_operand:TX 2 "register_operand" "=w")
1827 (mem:TX (match_dup 1)))
1828 (set (match_operand:TX 3 "register_operand" "=w")
1829 (mem:TX (plus:P (match_dup 1)
1830 (match_operand:P 5 "const_int_operand" "n"))))])]
1831 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1832 "ldp\\t%q2, %q3, [%1], %4"
1833 [(set_attr "type" "neon_ldp_q")]
1834)
1835
3e322b3f
JW		 1836;; Store pair with pre-index writeback. This is primarily used in function
1837;; prologues.
28514dda 1838(define_insn "storewb_pair<GPI:mode>_<P:mode>"
43e9d192 1839 [(parallel
28514dda
YZ		 1840 [(set (match_operand:P 0 "register_operand" "=&k")
1841 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1842 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
28514dda 1843 (set (mem:GPI (plus:P (match_dup 0)
43e9d192
IB		 1844 (match_dup 4)))
1845 (match_operand:GPI 2 "register_operand" "r"))
28514dda
YZ		 1846 (set (mem:GPI (plus:P (match_dup 0)
1847 (match_operand:P 5 "const_int_operand" "n")))
43e9d192
IB		 1848 (match_operand:GPI 3 "register_operand" "r"))])]
1849 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1850 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1851 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
80c11907
JW		 1852)
1853
1854(define_insn "storewb_pair<GPF:mode>_<P:mode>"
1855 [(parallel
1856 [(set (match_operand:P 0 "register_operand" "=&k")
1857 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1858 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
80c11907
JW		 1859 (set (mem:GPF (plus:P (match_dup 0)
1860 (match_dup 4)))
1861 (match_operand:GPF 2 "register_operand" "w"))
1862 (set (mem:GPF (plus:P (match_dup 0)
1863 (match_operand:P 5 "const_int_operand" "n")))
1864 (match_operand:GPF 3 "register_operand" "w"))])]
1865 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1866 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
80c11907 1867 [(set_attr "type" "neon_store1_2reg<q>")]
43e9d192
IB		 1868)
1869
a0d0b980
SE		 1870(define_insn "storewb_pair<TX:mode>_<P:mode>"
1871 [(parallel
1872 [(set (match_operand:P 0 "register_operand" "=&k")
1873 (plus:P (match_operand:P 1 "register_operand" "0")
1874 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1875 (set (mem:TX (plus:P (match_dup 0)
1876 (match_dup 4)))
1877 (match_operand:TX 2 "register_operand" "w"))
1878 (set (mem:TX (plus:P (match_dup 0)
1879 (match_operand:P 5 "const_int_operand" "n")))
1880 (match_operand:TX 3 "register_operand" "w"))])]
1881 "TARGET_SIMD
1882 && INTVAL (operands[5])
1883 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1884 "stp\\t%q2, %q3, [%0, %4]!"
1885 [(set_attr "type" "neon_stp_q")]
1886)
1887
43e9d192
IB		 1888;; -------------------------------------------------------------------
1889;; Sign/Zero extension
1890;; -------------------------------------------------------------------
1891
1892(define_expand "<optab>sidi2"
1893 [(set (match_operand:DI 0 "register_operand")
1894 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1895 ""
1896)
1897
1898(define_insn "*extendsidi2_aarch64"
1899 [(set (match_operand:DI 0 "register_operand" "=r,r")
1900 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1901 ""
1902 "@
1903 sxtw\t%0, %w1
1904 ldrsw\t%0, %1"
89b2133e 1905 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1906)
1907
350013bc
BC		 1908(define_insn "*load_pair_extendsidi2_aarch64"
1909 [(set (match_operand:DI 0 "register_operand" "=r")
1910 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1911 (set (match_operand:DI 2 "register_operand" "=r")
1912 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1913 "rtx_equal_p (XEXP (operands[3], 0),
1914 plus_constant (Pmode,
1915 XEXP (operands[1], 0),
1916 GET_MODE_SIZE (SImode)))"
2c272091 1917 "ldpsw\\t%0, %2, %z1"
89b2133e 1918 [(set_attr "type" "load_8")]
350013bc
BC		 1919)
1920
43e9d192 1921(define_insn "*zero_extendsidi2_aarch64"
0cfc095c
WD		 1922 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1923 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
43e9d192
IB		 1924 ""
1925 "@
1926 uxtw\t%0, %w1
0cfc095c
WD		 1927 ldr\t%w0, %1
1928 fmov\t%s0, %w1
1929 ldr\t%s0, %1
1930 fmov\t%w0, %s1
1931 fmov\t%s0, %s1"
e6b85350 1932 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
0cfc095c 1933 (set_attr "arch" "*,*,fp,fp,fp,fp")]
43e9d192
IB		 1934)
1935
350013bc 1936(define_insn "*load_pair_zero_extendsidi2_aarch64"
0cfc095c
WD		 1937 [(set (match_operand:DI 0 "register_operand" "=r,w")
1938 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1939 (set (match_operand:DI 2 "register_operand" "=r,w")
1940 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
350013bc
BC		 1941 "rtx_equal_p (XEXP (operands[3], 0),
1942 plus_constant (Pmode,
1943 XEXP (operands[1], 0),
1944 GET_MODE_SIZE (SImode)))"
0cfc095c 1945 "@
2c272091
RS		 1946 ldp\t%w0, %w2, %z1
1947 ldp\t%s0, %s2, %z1"
0cfc095c
WD		 1948 [(set_attr "type" "load_8,neon_load1_2reg")
1949 (set_attr "arch" "*,fp")]
350013bc
BC		 1950)
1951
43e9d192
IB		 1952(define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1953 [(set (match_operand:GPI 0 "register_operand")
1954 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1955 ""
1956)
1957
1958(define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
8b06ccb2
KT		 1959 [(set (match_operand:GPI 0 "register_operand" "=r,r,r")
1960 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,w")))]
43e9d192
IB		 1961 ""
1962 "@
1963 sxt<SHORT:size>\t%<GPI:w>0, %w1
8b06ccb2 1964 ldrs<SHORT:size>\t%<GPI:w>0, %1
a5e78ee6 1965 smov\t%<GPI:w>0, %1.<SHORT:size>[0]"
8b06ccb2
KT		 1966 [(set_attr "type" "extend,load_4,neon_to_gp")
1967 (set_attr "arch" "*,*,fp")]
43e9d192
IB		 1968)
1969
1970(define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
8695bf78
JW		 1971 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,r")
1972 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m,w")))]
43e9d192
IB		 1973 ""
1974 "@
63715e5e 1975 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
c902d3c8 1976 ldr<SHORT:size>\t%w0, %1
8695bf78
JW		 1977 ldr\t%<SHORT:size>0, %1
1978 umov\t%w0, %1.<SHORT:size>[0]"
1979 [(set_attr "type" "logic_imm,load_4,f_loads,neon_to_gp")
1980 (set_attr "arch" "*,*,fp,fp")]
43e9d192
IB		 1981)
1982
1983(define_expand "<optab>qihi2"
1984 [(set (match_operand:HI 0 "register_operand")
1985 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1986 ""
1987)
1988
63715e5e 1989(define_insn "*extendqihi2_aarch64"
43e9d192 1990 [(set (match_operand:HI 0 "register_operand" "=r,r")
63715e5e 1991 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
43e9d192
IB		 1992 ""
1993 "@
63715e5e
WD		 1994 sxtb\t%w0, %w1
1995 ldrsb\t%w0, %1"
89b2133e 1996 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1997)
1998
63715e5e
WD		 1999(define_insn "*zero_extendqihi2_aarch64"
2000 [(set (match_operand:HI 0 "register_operand" "=r,r")
2001 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
2002 ""
2003 "@
2004 and\t%w0, %w1, 255
2005 ldrb\t%w0, %1"
89b2133e 2006 [(set_attr "type" "logic_imm,load_4")]
63715e5e
WD		 2007)
2008
43e9d192
IB		 2009;; -------------------------------------------------------------------
2010;; Simple arithmetic
2011;; -------------------------------------------------------------------
2012
2013(define_expand "add<mode>3"
2014 [(set
1bbffb87
DZ		 2015 (match_operand:GPI 0 "register_operand")
2016 (plus:GPI (match_operand:GPI 1 "register_operand")
2017 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
43e9d192 2018 ""
89f0a62e 2019{
35323bd8
KT		 2020 /* If operands[1] is a subreg extract the inner RTX. */
2021 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
2022
45a6c1e3
WD		 2023 /* If the constant is too large for a single instruction and isn't frame
2024 based, split off the immediate so it is available for CSE. */
2025 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
0fdc30bc 2026 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
45a6c1e3 2027 && can_create_pseudo_p ()
35323bd8
KT		 2028 && (!REG_P (op1)
2029 || !REGNO_PTR_FRAME_P (REGNO (op1))))
45a6c1e3 2030 operands[2] = force_reg (<MODE>mode, operands[2]);
a65b9ad8
KT		 2031 /* Some tunings prefer to avoid VL-based operations.
2032 Split off the poly immediate here. The rtx costs hook will reject attempts
2033 to combine them back. */
2034 else if (GET_CODE (operands[2]) == CONST_POLY_INT
2035 && can_create_pseudo_p ()
2036 && (aarch64_tune_params.extra_tuning_flags
2037 & AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS))
2038 operands[2] = force_reg (<MODE>mode, operands[2]);
43cacb12
RS		 2039 /* Expand polynomial additions now if the destination is the stack
2040 pointer, since we don't want to use that as a temporary. */
2041 else if (operands[0] == stack_pointer_rtx
2042 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
2043 {
2044 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2045 operands[2], NULL_RTX, NULL_RTX);
2046 DONE;
2047 }
89f0a62e 2048})
43e9d192 2049
89f0a62e 2050(define_insn "*add<mode>3_aarch64"
43e9d192 2051 [(set
0fdc30bc 2052 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
89f0a62e 2053 (plus:GPI
0fdc30bc
RS		 2054 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
2055 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
43e9d192
IB		 2056 ""
2057 "@
89f0a62e
RH		 2058 add\\t%<w>0, %<w>1, %2
2059 add\\t%<w>0, %<w>1, %<w>2
2060 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
2061 sub\\t%<w>0, %<w>1, #%n2
43cacb12 2062 #
0fdc30bc
RS		 2063 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
2064 * return aarch64_output_sve_addvl_addpl (operands[2]);"
2065 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2066 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
2067 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
43e9d192
IB		 2068)
2069
3b585938
IB		 2070;; zero_extend version of above
2071(define_insn "*addsi3_aarch64_uxtw"
2072 [(set
89f0a62e 2073 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
3b585938 2074 (zero_extend:DI
89f0a62e 2075 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
ff1335e2 2076 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
3b585938
IB		 2077 ""
2078 "@
2079 add\\t%w0, %w1, %2
2080 add\\t%w0, %w1, %w2
89f0a62e
RH		 2081 sub\\t%w0, %w1, #%n2
2082 #"
2083 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
2084)
2085
2086;; If there's a free register, and we can load the constant with a
2087;; single instruction, do so. This has a chance to improve scheduling.
2088(define_peephole2
2089 [(match_scratch:GPI 3 "r")
2090 (set (match_operand:GPI 0 "register_operand")
2091 (plus:GPI
2092 (match_operand:GPI 1 "register_operand")
2093 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
2094 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
2095 [(set (match_dup 3) (match_dup 2))
2096 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
2097)
2098
2099(define_peephole2
2100 [(match_scratch:SI 3 "r")
2101 (set (match_operand:DI 0 "register_operand")
2102 (zero_extend:DI
2103 (plus:SI
2104 (match_operand:SI 1 "register_operand")
2105 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2106 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
2107 [(set (match_dup 3) (match_dup 2))
2108 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
2109)
2110
2111;; After peephole2 has had a chance to run, split any remaining long
2112;; additions into two add immediates.
2113(define_split
2114 [(set (match_operand:GPI 0 "register_operand")
2115 (plus:GPI
2116 (match_operand:GPI 1 "register_operand")
2117 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
2118 "epilogue_completed"
2119 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
2120 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
2121 {
2122 HOST_WIDE_INT i = INTVAL (operands[2]);
2123 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2124 operands[3] = GEN_INT (i - s);
2125 operands[4] = GEN_INT (s);
2126 }
3b585938
IB		 2127)
2128
43cacb12
RS		 2129;; Match addition of polynomial offsets that require one temporary, for which
2130;; we can use the early-clobbered destination register. This is a separate
2131;; pattern so that the early clobber doesn't affect register allocation
2132;; for other forms of addition. However, we still need to provide an
2133;; all-register alternative, in case the offset goes out of range after
2134;; elimination. For completeness we might as well provide all GPR-based
2135;; alternatives from the main pattern.
2136;;
2137;; We don't have a pattern for additions requiring two temporaries since at
2138;; present LRA doesn't allow new scratches to be added during elimination.
2139;; Such offsets should be rare anyway.
2140;;
2141;; ??? But if we added LRA support for new scratches, much of the ugliness
2142;; here would go away. We could just handle all polynomial constants in
2143;; this pattern.
2144(define_insn_and_split "*add<mode>3_poly_1"
2145 [(set
0fdc30bc 2146 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
43cacb12 2147 (plus:GPI
c001c194
JJ		 2148 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,0,rk,rk")
2149 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uai,Uav,Uat")))]
43cacb12
RS		 2150 "TARGET_SVE && operands[0] != stack_pointer_rtx"
2151 "@
2152 add\\t%<w>0, %<w>1, %2
2153 add\\t%<w>0, %<w>1, %<w>2
2154 sub\\t%<w>0, %<w>1, #%n2
2155 #
0fdc30bc
RS		 2156 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
2157 * return aarch64_output_sve_addvl_addpl (operands[2]);
43cacb12
RS		 2158 #"
2159 "&& epilogue_completed
2160 && !reg_overlap_mentioned_p (operands[0], operands[1])
2161 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
2162 [(const_int 0)]
2163 {
2164 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2165 operands[2], operands[0], NULL_RTX);
2166 DONE;
2167 }
0fdc30bc
RS		 2168 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2169 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
43cacb12
RS		 2170)
2171
89f0a62e
RH		 2172(define_split
2173 [(set (match_operand:DI 0 "register_operand")
2174 (zero_extend:DI
2175 (plus:SI
2176 (match_operand:SI 1 "register_operand")
2177 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2178 "epilogue_completed"
2179 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
2180 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
2181 {
2182 HOST_WIDE_INT i = INTVAL (operands[2]);
2183 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2184 operands[3] = GEN_INT (i - s);
2185 operands[4] = GEN_INT (s);
2186 operands[5] = gen_lowpart (SImode, operands[0]);
2187 }
43e9d192
IB		 2188)
2189
30c46053
MC		 2190(define_expand "addv<mode>4"
2191 [(match_operand:GPI 0 "register_operand")
2192 (match_operand:GPI 1 "register_operand")
a58fe3c5 2193 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 2194 (label_ref (match_operand 3 "" ""))]
2195 ""
2196{
a58fe3c5
RE		 2197 if (CONST_INT_P (operands[2]))
2198 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
2199 operands[2]));
2200 else
2201 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
30c46053
MC		 2202 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2203
2204 DONE;
2205})
2206
2207(define_expand "uaddv<mode>4"
2208 [(match_operand:GPI 0 "register_operand")
2209 (match_operand:GPI 1 "register_operand")
2210 (match_operand:GPI 2 "register_operand")
2211 (label_ref (match_operand 3 "" ""))]
2212 ""
2213{
2214 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
f7343f20 2215 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
30c46053
MC		 2216
2217 DONE;
2218})
2219
2195867f 2220(define_expand "addti3"
1bbffb87
DZ		 2221 [(set (match_operand:TI 0 "register_operand")
2222 (plus:TI (match_operand:TI 1 "register_operand")
2223 (match_operand:TI 2 "aarch64_reg_or_imm")))]
30c46053
MC		 2224 ""
2225{
2226 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2227
2228 aarch64_addti_scratch_regs (operands[1], operands[2],
2229 &low_dest, &op1_low, &op2_low,
2230 &high_dest, &op1_high, &op2_high);
2231
2232 if (op2_low == const0_rtx)
2233 {
2234 low_dest = op1_low;
2235 if (!aarch64_pluslong_operand (op2_high, DImode))
2236 op2_high = force_reg (DImode, op2_high);
2237 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2238 }
2239 else
2240 {
2241 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2242 force_reg (DImode, op2_low)));
2243 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2244 force_reg (DImode, op2_high)));
2245 }
2246
2247 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2248 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2249
2250 DONE;
2251})
2252
2253(define_expand "addvti4"
1bbffb87
DZ		 2254 [(match_operand:TI 0 "register_operand")
2255 (match_operand:TI 1 "register_operand")
2256 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053 2257 (label_ref (match_operand 3 "" ""))]
2195867f
RH		 2258 ""
2259{
30c46053 2260 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2195867f 2261
30c46053
MC		 2262 aarch64_addti_scratch_regs (operands[1], operands[2],
2263 &low_dest, &op1_low, &op2_low,
2264 &high_dest, &op1_high, &op2_high);
2195867f 2265
30c46053
MC		 2266 if (op2_low == const0_rtx)
2267 {
2268 low_dest = op1_low;
2269 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2270 force_reg (DImode, op2_high)));
2271 }
2272 else
2273 {
2274 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2275 force_reg (DImode, op2_low)));
2276 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2277 force_reg (DImode, op2_high)));
2278 }
2279
2280 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2281 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2282
2283 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2195867f
RH		 2284 DONE;
2285})
2286
30c46053 2287(define_expand "uaddvti4"
1bbffb87
DZ		 2288 [(match_operand:TI 0 "register_operand")
2289 (match_operand:TI 1 "register_operand")
2290 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053
MC		 2291 (label_ref (match_operand 3 "" ""))]
2292 ""
2293{
2294 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2295
2296 aarch64_addti_scratch_regs (operands[1], operands[2],
2297 &low_dest, &op1_low, &op2_low,
2298 &high_dest, &op1_high, &op2_high);
2299
2300 if (op2_low == const0_rtx)
2301 {
2302 low_dest = op1_low;
2303 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2304 force_reg (DImode, op2_high)));
2305 }
2306 else
2307 {
2308 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2309 force_reg (DImode, op2_low)));
2310 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2311 force_reg (DImode, op2_high)));
2312 }
2313
2314 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2315 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2316
f7343f20 2317 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
30c46053
MC		 2318 DONE;
2319 })
2320
2195867f 2321(define_insn "add<mode>3_compare0"
43e9d192
IB		 2322 [(set (reg:CC_NZ CC_REGNUM)
2323 (compare:CC_NZ
341fca1b 2324 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
34b512f5 2325 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
43e9d192 2326 (const_int 0)))
34b512f5 2327 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
43e9d192
IB		 2328 (plus:GPI (match_dup 1) (match_dup 2)))]
2329 ""
2330 "@
2331 adds\\t%<w>0, %<w>1, %<w>2
e2b691c4 2332 adds\\t%<w>0, %<w>1, %2
43e9d192 2333 subs\\t%<w>0, %<w>1, #%n2"
1d61feeb 2334 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2335)
2336
3b585938
IB		 2337;; zero_extend version of above
2338(define_insn "*addsi3_compare0_uxtw"
2339 [(set (reg:CC_NZ CC_REGNUM)
2340 (compare:CC_NZ
341fca1b 2341 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
34b512f5 2342 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
3b585938 2343 (const_int 0)))
34b512f5 2344 (set (match_operand:DI 0 "register_operand" "=r,r,r")
3b585938
IB		 2345 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2346 ""
2347 "@
2348 adds\\t%w0, %w1, %w2
e2b691c4 2349 adds\\t%w0, %w1, %2
3b585938 2350 subs\\t%w0, %w1, #%n2"
1d61feeb 2351 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3b585938
IB		 2352)
2353
ef22810a
RH		 2354(define_insn "*add<mode>3_compareC_cconly"
2355 [(set (reg:CC_C CC_REGNUM)
f7343f20
RE		 2356 (compare:CC_C
2357 (plus:GPI
2358 (match_operand:GPI 0 "register_operand" "r,r,r")
2359 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2360 (match_dup 0)))]
ef22810a 2361 ""
ef22810a 2362 "@
f7343f20
RE		 2363 cmn\\t%<w>0, %<w>1
2364 cmn\\t%<w>0, %1
2365 cmp\\t%<w>0, #%n1"
2366 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a 2367)
954224d7 2368
ef22810a
RH		 2369(define_insn "add<mode>3_compareC"
2370 [(set (reg:CC_C CC_REGNUM)
30c46053 2371 (compare:CC_C
f7343f20 2372 (plus:GPI
341fca1b 2373 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
f7343f20
RE		 2374 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2375 (match_dup 1)))
2376 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
ef22810a
RH		 2377 (plus:GPI (match_dup 1) (match_dup 2)))]
2378 ""
f7343f20
RE		 2379 "@
2380 adds\\t%<w>0, %<w>1, %<w>2
2381 adds\\t%<w>0, %<w>1, %2
2382 subs\\t%<w>0, %<w>1, #%n2"
2383 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a
RH		 2384)
2385
30c46053
MC		 2386(define_insn "*add<mode>3_compareV_cconly_imm"
2387 [(set (reg:CC_V CC_REGNUM)
2388 (compare:CC_V
2389 (plus:<DWI>
2390 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2391 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2392 (sign_extend:<DWI>
2393 (plus:GPI
2394 (match_dup 0)
2395 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2396 "INTVAL (operands[1]) == INTVAL (operands[2])"
2397 "@
2398 cmn\\t%<w>0, %<w>1
2399 cmp\\t%<w>0, #%n1"
2400 [(set_attr "type" "alus_imm")]
2401)
2402
2403(define_insn "*add<mode>3_compareV_cconly"
2404 [(set (reg:CC_V CC_REGNUM)
2405 (compare:CC_V
2406 (plus:<DWI>
2407 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2408 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2409 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2410 ""
2411 "cmn\\t%<w>0, %<w>1"
2412 [(set_attr "type" "alus_sreg")]
2413)
2414
a58fe3c5 2415(define_insn "add<mode>3_compareV_imm"
30c46053
MC		 2416 [(set (reg:CC_V CC_REGNUM)
2417 (compare:CC_V
2418 (plus:<DWI>
2419 (sign_extend:<DWI>
341fca1b 2420 (match_operand:GPI 1 "register_operand" "rk,rk"))
30c46053
MC		 2421 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2422 (sign_extend:<DWI>
2423 (plus:GPI (match_dup 1) (match_dup 2)))))
2424 (set (match_operand:GPI 0 "register_operand" "=r,r")
2425 (plus:GPI (match_dup 1) (match_dup 2)))]
2426 ""
2427 "@
2428 adds\\t%<w>0, %<w>1, %<w>2
2429 subs\\t%<w>0, %<w>1, #%n2"
2430 [(set_attr "type" "alus_imm,alus_imm")]
2431)
2432
2433(define_insn "add<mode>3_compareV"
2434 [(set (reg:CC_V CC_REGNUM)
2435 (compare:CC_V
2436 (plus:<DWI>
341fca1b 2437 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
30c46053
MC		 2438 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2439 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2440 (set (match_operand:GPI 0 "register_operand" "=r")
2441 (plus:GPI (match_dup 1) (match_dup 2)))]
2442 ""
2443 "adds\\t%<w>0, %<w>1, %<w>2"
2444 [(set_attr "type" "alus_sreg")]
2445)
2446
dd1bdce4
VK		 2447(define_insn "*adds_shift_imm_<mode>"
2448 [(set (reg:CC_NZ CC_REGNUM)
2449 (compare:CC_NZ
30c46053 2450 (plus:GPI (ASHIFT:GPI
dd1bdce4
VK		 2451 (match_operand:GPI 1 "register_operand" "r")
2452 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2453 (match_operand:GPI 3 "register_operand" "r"))
2454 (const_int 0)))
2455 (set (match_operand:GPI 0 "register_operand" "=r")
2456 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2457 (match_dup 3)))]
2458 ""
2459 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2460 [(set_attr "type" "alus_shift_imm")]
2461)
2462
2463(define_insn "*subs_shift_imm_<mode>"
2464 [(set (reg:CC_NZ CC_REGNUM)
2465 (compare:CC_NZ
2466 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2467 (ASHIFT:GPI
2468 (match_operand:GPI 2 "register_operand" "r")
2469 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2470 (const_int 0)))
2471 (set (match_operand:GPI 0 "register_operand" "=r")
2472 (minus:GPI (match_dup 1)
2473 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2474 ""
2475 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2476 [(set_attr "type" "alus_shift_imm")]
2477)
2478
f3d096b0
N		 2479(define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2480 [(set (reg:CC_NZ CC_REGNUM)
2481 (compare:CC_NZ
2482 (plus:GPI
2483 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
341fca1b 2484 (match_operand:GPI 2 "register_operand" "rk"))
f3d096b0
N		 2485 (const_int 0)))
2486 (set (match_operand:GPI 0 "register_operand" "=r")
2487 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2488 ""
d4febc75 2489 "adds\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
99e6af15 2490 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2491)
2492
2493(define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2494 [(set (reg:CC_NZ CC_REGNUM)
2495 (compare:CC_NZ
341fca1b 2496 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
f3d096b0
N		 2497 (ANY_EXTEND:GPI
2498 (match_operand:ALLX 2 "register_operand" "r")))
2499 (const_int 0)))
2500 (set (match_operand:GPI 0 "register_operand" "=r")
2501 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2502 ""
d4febc75 2503 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
99e6af15 2504 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2505)
2506
dd1bdce4
VK		 2507(define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2508 [(set (reg:CC_NZ CC_REGNUM)
2509 (compare:CC_NZ
2510 (plus:GPI (ashift:GPI
2511 (ANY_EXTEND:GPI
2512 (match_operand:ALLX 1 "register_operand" "r"))
2513 (match_operand 2 "aarch64_imm3" "Ui3"))
341fca1b 2514 (match_operand:GPI 3 "register_operand" "rk"))
dd1bdce4
VK		 2515 (const_int 0)))
2516 (set (match_operand:GPI 0 "register_operand" "=rk")
2517 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2518 (match_dup 2))
2519 (match_dup 3)))]
2520 ""
d4febc75 2521 "adds\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
dd1bdce4
VK		 2522 [(set_attr "type" "alus_ext")]
2523)
2524
2525(define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2526 [(set (reg:CC_NZ CC_REGNUM)
2527 (compare:CC_NZ
341fca1b 2528 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
dd1bdce4
VK		 2529 (ashift:GPI
2530 (ANY_EXTEND:GPI
2531 (match_operand:ALLX 2 "register_operand" "r"))
2532 (match_operand 3 "aarch64_imm3" "Ui3")))
2533 (const_int 0)))
2534 (set (match_operand:GPI 0 "register_operand" "=rk")
2535 (minus:GPI (match_dup 1)
2536 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2537 (match_dup 3))))]
2538 ""
d4febc75 2539 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
dd1bdce4
VK		 2540 [(set_attr "type" "alus_ext")]
2541)
2542
43e9d192
IB		 2543(define_insn "*add<mode>3nr_compare0"
2544 [(set (reg:CC_NZ CC_REGNUM)
2545 (compare:CC_NZ
34b512f5
SN		 2546 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2547 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
43e9d192
IB		 2548 (const_int 0)))]
2549 ""
2550 "@
2551 cmn\\t%<w>0, %<w>1
e2b691c4 2552 cmn\\t%<w>0, %1
43e9d192 2553 cmp\\t%<w>0, #%n1"
1d61feeb 2554 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2555)
2556
2f65ab2e
MC		 2557(define_insn "aarch64_sub<mode>_compare0"
2558 [(set (reg:CC_NZ CC_REGNUM)
2559 (compare:CC_NZ
2560 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2561 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2562 (const_int 0)))]
2563 ""
2564 "cmp\\t%<w>0, %<w>1"
2565 [(set_attr "type" "alus_sreg")]
2566)
2567
f5313c63 2568(define_insn "*compare_neg<mode>"
1c992d1e
RE		 2569 [(set (reg:CC_Z CC_REGNUM)
2570 (compare:CC_Z
274b2532
KT		 2571 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2572 (match_operand:GPI 1 "register_operand" "r")))]
f5313c63 2573 ""
274b2532 2574 "cmn\\t%<w>1, %<w>0"
1d61feeb 2575 [(set_attr "type" "alus_sreg")]
f5313c63
IB		 2576)
2577
43e9d192 2578(define_insn "*add_<shift>_<mode>"
24c56925 2579 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 2580 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2581 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2582 (match_operand:GPI 3 "register_operand" "r")))]
2583 ""
2584 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
ae27ce51 2585 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 2586)
2587
3b585938
IB		 2588;; zero_extend version of above
2589(define_insn "*add_<shift>_si_uxtw"
24c56925 2590 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 2591 (zero_extend:DI
2592 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2a9704d0
MS		 2593 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2594 (match_operand:SI 3 "register_operand" "r"))))]
3b585938
IB		 2595 ""
2596 "add\\t%w0, %w3, %w1, <shift> %2"
ae27ce51 2597 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 2598)
2599
43e9d192
IB		 2600(define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2601 [(set (match_operand:GPI 0 "register_operand" "=rk")
2602 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2603 (match_operand:GPI 2 "register_operand" "r")))]
2604 ""
d4febc75 2605 "add\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
99e6af15 2606 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2607)
2608
3b585938
IB		 2609;; zero_extend version of above
2610(define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2611 [(set (match_operand:DI 0 "register_operand" "=rk")
2612 (zero_extend:DI
2613 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2614 (match_operand:GPI 2 "register_operand" "r"))))]
2615 ""
2616 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
99e6af15 2617 [(set_attr "type" "alu_ext")]
3b585938
IB		 2618)
2619
43e9d192
IB		 2620(define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2621 [(set (match_operand:GPI 0 "register_operand" "=rk")
2622 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2623 (match_operand:ALLX 1 "register_operand" "r"))
2624 (match_operand 2 "aarch64_imm3" "Ui3"))
2625 (match_operand:GPI 3 "register_operand" "r")))]
2626 ""
d4febc75 2627 "add\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
99e6af15 2628 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2629)
2630
3b585938
IB		 2631;; zero_extend version of above
2632(define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2633 [(set (match_operand:DI 0 "register_operand" "=rk")
2634 (zero_extend:DI
2635 (plus:SI (ashift:SI (ANY_EXTEND:SI
2636 (match_operand:SHORT 1 "register_operand" "r"))
2637 (match_operand 2 "aarch64_imm3" "Ui3"))
2638 (match_operand:SI 3 "register_operand" "r"))))]
2639 ""
2640 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
99e6af15 2641 [(set_attr "type" "alu_ext")]
3b585938
IB		 2642)
2643
ef22810a
RH		 2644(define_expand "add<mode>3_carryin"
2645 [(set (match_operand:GPI 0 "register_operand")
2646 (plus:GPI
2647 (plus:GPI
f7343f20 2648 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
ef22810a
RH		 2649 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2650 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
3b585938 2651 ""
43e9d192 2652 ""
43e9d192
IB		 2653)
2654
ef22810a
RH		 2655;; Note that add with carry with two zero inputs is matched by cset,
2656;; and that add with carry with one zero input is matched by cinc.
3b585938 2657
ef22810a
RH		 2658(define_insn "*add<mode>3_carryin"
2659 [(set (match_operand:GPI 0 "register_operand" "=r")
2660 (plus:GPI
2661 (plus:GPI
2662 (match_operand:GPI 3 "aarch64_carry_operation" "")
2663 (match_operand:GPI 1 "register_operand" "r"))
2664 (match_operand:GPI 2 "register_operand" "r")))]
43e9d192
IB		 2665 ""
2666 "adc\\t%<w>0, %<w>1, %<w>2"
99e6af15 2667 [(set_attr "type" "adc_reg")]
43e9d192
IB		 2668)
2669
3b585938 2670;; zero_extend version of above
ef22810a
RH		 2671(define_insn "*addsi3_carryin_uxtw"
2672 [(set (match_operand:DI 0 "register_operand" "=r")
2673 (zero_extend:DI
2674 (plus:SI
2675 (plus:SI
2676 (match_operand:SI 3 "aarch64_carry_operation" "")
2677 (match_operand:SI 1 "register_operand" "r"))
2678 (match_operand:SI 2 "register_operand" "r"))))]
3b585938
IB		 2679 ""
2680 "adc\\t%w0, %w1, %w2"
99e6af15 2681 [(set_attr "type" "adc_reg")]
3b585938
IB		 2682)
2683
30c46053
MC		 2684(define_expand "add<mode>3_carryinC"
2685 [(parallel
2686 [(set (match_dup 3)
f7343f20 2687 (compare:CC_ADC
30c46053
MC		 2688 (plus:<DWI>
2689 (plus:<DWI>
2690 (match_dup 4)
2691 (zero_extend:<DWI>
1bbffb87 2692 (match_operand:GPI 1 "register_operand")))
30c46053 2693 (zero_extend:<DWI>
1bbffb87 2694 (match_operand:GPI 2 "register_operand")))
f7343f20 2695 (match_dup 6)))
30c46053
MC		 2696 (set (match_operand:GPI 0 "register_operand")
2697 (plus:GPI
2698 (plus:GPI (match_dup 5) (match_dup 1))
2699 (match_dup 2)))])]
2700 ""
2701{
f7343f20
RE		 2702 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2703 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2704 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2705 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2706 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2707 << GET_MODE_BITSIZE (<MODE>mode),
2708 TImode);
30c46053
MC		 2709})
2710
2711(define_insn "*add<mode>3_carryinC_zero"
f7343f20
RE		 2712 [(set (reg:CC_ADC CC_REGNUM)
2713 (compare:CC_ADC
30c46053
MC		 2714 (plus:<DWI>
2715 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2716 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
f7343f20 2717 (match_operand 4 "const_scalar_int_operand" "")))
16621f0d 2718 (set (match_operand:GPI 0 "register_operand" "=r")
f7343f20
RE		 2719 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2720 (match_dup 1)))]
2721 "rtx_mode_t (operands[4], <DWI>mode)
2722 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2723 "adcs\\t%<w>0, %<w>1, <w>zr"
2724 [(set_attr "type" "adc_reg")]
2725)
2726
2727(define_insn "*add<mode>3_carryinC"
f7343f20
RE		 2728 [(set (reg:CC_ADC CC_REGNUM)
2729 (compare:CC_ADC
30c46053
MC		 2730 (plus:<DWI>
2731 (plus:<DWI>
2732 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2733 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2734 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
f7343f20 2735 (match_operand 5 "const_scalar_int_operand" "")))
16621f0d 2736 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053 2737 (plus:GPI
f7343f20
RE		 2738 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2739 (match_dup 1))
30c46053 2740 (match_dup 2)))]
f7343f20
RE		 2741 "rtx_mode_t (operands[5], <DWI>mode)
2742 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2743 "adcs\\t%<w>0, %<w>1, %<w>2"
2744 [(set_attr "type" "adc_reg")]
2745)
2746
2747(define_expand "add<mode>3_carryinV"
2748 [(parallel
2749 [(set (reg:CC_V CC_REGNUM)
2750 (compare:CC_V
2751 (plus:<DWI>
2752 (plus:<DWI>
2753 (match_dup 3)
2754 (sign_extend:<DWI>
1bbffb87 2755 (match_operand:GPI 1 "register_operand")))
30c46053 2756 (sign_extend:<DWI>
1bbffb87 2757 (match_operand:GPI 2 "register_operand")))
30c46053
MC		 2758 (sign_extend:<DWI>
2759 (plus:GPI
2760 (plus:GPI (match_dup 4) (match_dup 1))
2761 (match_dup 2)))))
2762 (set (match_operand:GPI 0 "register_operand")
2763 (plus:GPI
2764 (plus:GPI (match_dup 4) (match_dup 1))
2765 (match_dup 2)))])]
2766 ""
2767{
2768 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
f7343f20
RE		 2769 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2770 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
30c46053
MC		 2771})
2772
2773(define_insn "*add<mode>3_carryinV_zero"
2774 [(set (reg:CC_V CC_REGNUM)
2775 (compare:CC_V
2776 (plus:<DWI>
2777 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2778 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2779 (sign_extend:<DWI>
2780 (plus:GPI
2781 (match_operand:GPI 3 "aarch64_carry_operation" "")
2782 (match_dup 1)))))
16621f0d 2783 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2784 (plus:GPI (match_dup 3) (match_dup 1)))]
2785 ""
2786 "adcs\\t%<w>0, %<w>1, <w>zr"
2787 [(set_attr "type" "adc_reg")]
2788)
2789
2790(define_insn "*add<mode>3_carryinV"
2791 [(set (reg:CC_V CC_REGNUM)
2792 (compare:CC_V
2793 (plus:<DWI>
2794 (plus:<DWI>
2795 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2796 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2797 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2798 (sign_extend:<DWI>
2799 (plus:GPI
2800 (plus:GPI
2801 (match_operand:GPI 4 "aarch64_carry_operation" "")
2802 (match_dup 1))
2803 (match_dup 2)))))
16621f0d 2804 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2805 (plus:GPI
2806 (plus:GPI (match_dup 4) (match_dup 1))
2807 (match_dup 2)))]
2808 ""
2809 "adcs\\t%<w>0, %<w>1, %<w>2"
2810 [(set_attr "type" "adc_reg")]
2811)
2812
dd1bdce4
VK		 2813(define_insn "*add_uxt<mode>_shift2"
2814 [(set (match_operand:GPI 0 "register_operand" "=rk")
2815 (plus:GPI (and:GPI
2816 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2817 (match_operand 2 "aarch64_imm3" "Ui3"))
2818 (match_operand 3 "const_int_operand" "n"))
2819 (match_operand:GPI 4 "register_operand" "r")))]
2820 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2821 "*
2822 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2823 INTVAL (operands[3])));
d4febc75 2824 return \"add\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
dd1bdce4
VK		 2825 [(set_attr "type" "alu_ext")]
2826)
2827
2828;; zero_extend version of above
2829(define_insn "*add_uxtsi_shift2_uxtw"
2830 [(set (match_operand:DI 0 "register_operand" "=rk")
2831 (zero_extend:DI
2832 (plus:SI (and:SI
2833 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2834 (match_operand 2 "aarch64_imm3" "Ui3"))
2835 (match_operand 3 "const_int_operand" "n"))
2836 (match_operand:SI 4 "register_operand" "r"))))]
2837 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2838 "*
2839 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2840 INTVAL (operands[3])));
2841 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2842 [(set_attr "type" "alu_ext")]
2843)
2844
43e9d192
IB		 2845(define_insn "subsi3"
2846 [(set (match_operand:SI 0 "register_operand" "=rk")
7eb8fb77
RS		 2847 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2848 (match_operand:SI 2 "register_operand" "r")))]
43e9d192
IB		 2849 ""
2850 "sub\\t%w0, %w1, %w2"
1d61feeb 2851 [(set_attr "type" "alu_sreg")]
43e9d192
IB		 2852)
2853
3b585938
IB		 2854;; zero_extend version of above
2855(define_insn "*subsi3_uxtw"
2856 [(set (match_operand:DI 0 "register_operand" "=rk")
2857 (zero_extend:DI
7eb8fb77 2858 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 2859 (match_operand:SI 2 "register_operand" "r"))))]
2860 ""
2861 "sub\\t%w0, %w1, %w2"
1d61feeb 2862 [(set_attr "type" "alu_sreg")]
3b585938
IB		 2863)
2864
43e9d192 2865(define_insn "subdi3"
4f2962fd 2866 [(set (match_operand:DI 0 "register_operand" "=rk,w")
7eb8fb77
RS		 2867 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2868 (match_operand:DI 2 "register_operand" "r,w")))]
43e9d192
IB		 2869 ""
2870 "@
2871 sub\\t%x0, %x1, %x2
2872 sub\\t%d0, %d1, %d2"
1d61feeb 2873 [(set_attr "type" "alu_sreg, neon_sub")
488461d8 2874 (set_attr "arch" "*,simd")]
43e9d192
IB		 2875)
2876
a58fe3c5 2877(define_expand "subv<GPI:mode>4"
30c46053 2878 [(match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 2879 (match_operand:GPI 1 "register_operand")
2880 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 2881 (label_ref (match_operand 3 "" ""))]
2882 ""
2883{
a58fe3c5
RE		 2884 if (CONST_INT_P (operands[2]))
2885 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2886 else
2887 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
30c46053
MC		 2888 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2889
2890 DONE;
2891})
2892
a58fe3c5
RE		 2893(define_insn "subv<GPI:mode>_insn"
2894 [(set (reg:CC_V CC_REGNUM)
2895 (compare:CC_V
2896 (sign_extend:<DWI>
2897 (minus:GPI
2898 (match_operand:GPI 1 "register_operand" "rk")
2899 (match_operand:GPI 2 "register_operand" "r")))
2900 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2901 (sign_extend:<DWI> (match_dup 2)))))
2902 (set (match_operand:GPI 0 "register_operand" "=r")
2903 (minus:GPI (match_dup 1) (match_dup 2)))]
2904 ""
2905 "subs\\t%<w>0, %<w>1, %<w>2"
2906 [(set_attr "type" "alus_sreg")]
2907)
2908
2909(define_insn "subv<GPI:mode>_imm"
2910 [(set (reg:CC_V CC_REGNUM)
2911 (compare:CC_V
2912 (sign_extend:<DWI>
2913 (minus:GPI
2914 (match_operand:GPI 1 "register_operand" "rk,rk")
2915 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2916 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2917 (match_dup 2))))
2918 (set (match_operand:GPI 0 "register_operand" "=r,r")
2919 (minus:GPI (match_dup 1) (match_dup 2)))]
2920 ""
2921 "@
2922 subs\\t%<w>0, %<w>1, %2
2923 adds\\t%<w>0, %<w>1, #%n2"
2924 [(set_attr "type" "alus_sreg")]
2925)
2926
2927(define_expand "negv<GPI:mode>3"
2928 [(match_operand:GPI 0 "register_operand")
2929 (match_operand:GPI 1 "register_operand")
2930 (label_ref (match_operand 2 "" ""))]
2931 ""
2932 {
2933 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2934 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2935
2936 DONE;
2937 }
2938)
2939
2940(define_insn "negv<GPI:mode>_insn"
2941 [(set (reg:CC_V CC_REGNUM)
2942 (compare:CC_V
2943 (sign_extend:<DWI>
2944 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2945 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2946 (set (match_operand:GPI 0 "register_operand" "=r")
2947 (neg:GPI (match_dup 1)))]
2948 ""
2949 "negs\\t%<w>0, %<w>1"
2950 [(set_attr "type" "alus_sreg")]
2951)
2952
2953(define_insn "negv<GPI:mode>_cmp_only"
2954 [(set (reg:CC_V CC_REGNUM)
2955 (compare:CC_V
2956 (sign_extend:<DWI>
2957 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2958 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2959 ""
2960 "negs\\t%<w>zr, %<w>0"
2961 [(set_attr "type" "alus_sreg")]
2962)
2963
2964(define_insn "*cmpv<GPI:mode>_insn"
2965 [(set (reg:CC_V CC_REGNUM)
2966 (compare:CC_V
2967 (sign_extend:<DWI>
2968 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2969 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2970 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2971 (sign_extend:<DWI> (match_dup 1)))))]
2972 ""
2973 "@
2974 cmp\\t%<w>0, %<w>1
2975 cmp\\t%<w>0, %1
2976 cmp\\t%<w>0, #%n1"
2977 [(set_attr "type" "alus_sreg")]
2978)
2979
30c46053
MC		 2980(define_expand "usubv<mode>4"
2981 [(match_operand:GPI 0 "register_operand")
2982 (match_operand:GPI 1 "aarch64_reg_or_zero")
2983 (match_operand:GPI 2 "aarch64_reg_or_zero")
2984 (label_ref (match_operand 3 "" ""))]
2985 ""
2986{
2987 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2988 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2989
2990 DONE;
2991})
2992
2195867f 2993(define_expand "subti3"
1bbffb87
DZ		 2994 [(set (match_operand:TI 0 "register_operand")
2995 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2996 (match_operand:TI 2 "register_operand")))]
2195867f
RH		 2997 ""
2998{
30c46053
MC		 2999 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3000
3001 aarch64_subvti_scratch_regs (operands[1], operands[2],
3002 &low_dest, &op1_low, &op2_low,
3003 &high_dest, &op1_high, &op2_high);
2195867f 3004
30c46053
MC		 3005 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
3006 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2195867f 3007
30c46053
MC		 3008 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
3009 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
3010 DONE;
3011})
3012
3013(define_expand "subvti4"
3014 [(match_operand:TI 0 "register_operand")
a58fe3c5 3015 (match_operand:TI 1 "register_operand")
30c46053
MC		 3016 (match_operand:TI 2 "aarch64_reg_or_imm")
3017 (label_ref (match_operand 3 "" ""))]
3018 ""
3019{
3020 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3021
3022 aarch64_subvti_scratch_regs (operands[1], operands[2],
3023 &low_dest, &op1_low, &op2_low,
3024 &high_dest, &op1_high, &op2_high);
3025 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 3026 high_dest, op1_high, op2_high, false);
30c46053
MC		 3027
3028 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
3029 DONE;
3030})
3031
3032(define_expand "usubvti4"
3033 [(match_operand:TI 0 "register_operand")
a58fe3c5 3034 (match_operand:TI 1 "register_operand")
30c46053
MC		 3035 (match_operand:TI 2 "aarch64_reg_or_imm")
3036 (label_ref (match_operand 3 "" ""))]
3037 ""
3038{
3039 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3040
3041 aarch64_subvti_scratch_regs (operands[1], operands[2],
3042 &low_dest, &op1_low, &op2_low,
3043 &high_dest, &op1_high, &op2_high);
3044 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 3045 high_dest, op1_high, op2_high, true);
30c46053
MC		 3046
3047 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2195867f
RH		 3048 DONE;
3049})
43e9d192 3050
a58fe3c5
RE		 3051(define_expand "negvti3"
3052 [(match_operand:TI 0 "register_operand")
3053 (match_operand:TI 1 "register_operand")
3054 (label_ref (match_operand 2 "" ""))]
3055 ""
3056 {
3057 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
3058 gen_lowpart (DImode, operands[1])));
3059 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
3060 gen_highpart (DImode, operands[1])));
3061 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
3062
3063 DONE;
3064 }
3065)
3066
3067(define_insn "negdi_carryout"
3068 [(set (reg:CC CC_REGNUM)
3069 (compare:CC
3070 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3071 (set (match_operand:DI 0 "register_operand" "=r")
3072 (neg:DI (match_dup 1)))]
3073 ""
3074 "negs\\t%0, %1"
3075 [(set_attr "type" "alus_sreg")]
3076)
3077
3078(define_insn "negvdi_carryinV"
3079 [(set (reg:CC_V CC_REGNUM)
3080 (compare:CC_V
3081 (neg:TI (plus:TI
3082 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3083 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3084 (sign_extend:TI
3085 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3086 (match_dup 1))))))
3087 (set (match_operand:DI 0 "register_operand" "=r")
3088 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3089 (match_dup 1))))]
3090 ""
3091 "ngcs\\t%0, %1"
3092 [(set_attr "type" "alus_sreg")]
3093)
3094
ef22810a 3095(define_insn "*sub<mode>3_compare0"
43e9d192 3096 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 3097 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3098 (match_operand:GPI 2 "register_operand" "r"))
3099 (const_int 0)))
3100 (set (match_operand:GPI 0 "register_operand" "=r")
3101 (minus:GPI (match_dup 1) (match_dup 2)))]
3102 ""
3103 "subs\\t%<w>0, %<w>1, %<w>2"
1d61feeb 3104 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 3105)
3106
3b585938
IB		 3107;; zero_extend version of above
3108(define_insn "*subsi3_compare0_uxtw"
3109 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 3110 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3111 (match_operand:SI 2 "register_operand" "r"))
3112 (const_int 0)))
3113 (set (match_operand:DI 0 "register_operand" "=r")
3114 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3115 ""
3116 "subs\\t%w0, %w1, %w2"
1d61feeb 3117 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3118)
3119
a58fe3c5 3120(define_insn "sub<mode>3_compare1_imm"
30c46053
MC		 3121 [(set (reg:CC CC_REGNUM)
3122 (compare:CC
19199a6f 3123 (match_operand:GPI 1 "register_operand" "rk,rk")
30c46053
MC		 3124 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3125 (set (match_operand:GPI 0 "register_operand" "=r,r")
3126 (plus:GPI
3127 (match_dup 1)
3128 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3129 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3130 "@
a58fe3c5
RE		 3131 subs\\t%<w>0, %<w>1, %2
3132 adds\\t%<w>0, %<w>1, #%n2"
30c46053
MC		 3133 [(set_attr "type" "alus_imm")]
3134)
3135
ef22810a
RH		 3136(define_insn "sub<mode>3_compare1"
3137 [(set (reg:CC CC_REGNUM)
3138 (compare:CC
341fca1b 3139 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
ef22810a
RH		 3140 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3141 (set (match_operand:GPI 0 "register_operand" "=r")
3142 (minus:GPI (match_dup 1) (match_dup 2)))]
3143 ""
3144 "subs\\t%<w>0, %<w>1, %<w>2"
3145 [(set_attr "type" "alus_sreg")]
3146)
3147
e49d8b68 3148(define_peephole2
341fca1b 3149 [(set (match_operand:GPI 0 "aarch64_general_reg")
e49d8b68
KT		 3150 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3151 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3152 (set (reg:CC CC_REGNUM)
3153 (compare:CC
3154 (match_dup 1)
3155 (match_dup 2)))]
3156 "!reg_overlap_mentioned_p (operands[0], operands[1])
3157 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3158 [(const_int 0)]
3159 {
3160 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3161 operands[2]));
3162 DONE;
3163 }
3164)
3165
de47f61f
KT		 3166;; Same as the above peephole but with the compare and minus in
3167;; swapped order. The restriction on overlap between operand 0
3168;; and operands 1 and 2 doesn't apply here.
3169(define_peephole2
3170 [(set (reg:CC CC_REGNUM)
3171 (compare:CC
3172 (match_operand:GPI 1 "aarch64_reg_or_zero")
3173 (match_operand:GPI 2 "aarch64_reg_or_zero")))
341fca1b 3174 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f
KT		 3175 (minus:GPI (match_dup 1)
3176 (match_dup 2)))]
3177 ""
3178 [(const_int 0)]
3179 {
3180 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3181 operands[2]));
3182 DONE;
3183 }
3184)
3185
279dc4b4 3186(define_peephole2
341fca1b 3187 [(set (match_operand:GPI 0 "aarch64_general_reg")
279dc4b4 3188 (plus:GPI (match_operand:GPI 1 "register_operand")
a58fe3c5 3189 (match_operand:GPI 2 "aarch64_plus_immediate")))
279dc4b4
KT		 3190 (set (reg:CC CC_REGNUM)
3191 (compare:CC
3192 (match_dup 1)
3193 (match_operand:GPI 3 "const_int_operand")))]
3194 "!reg_overlap_mentioned_p (operands[0], operands[1])
3195 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3196 [(const_int 0)]
3197 {
3198 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3199 operands[3], operands[2]));
279dc4b4
KT		 3200 DONE;
3201 }
3202)
3203
de47f61f
KT		 3204;; Same as the above peephole but with the compare and minus in
3205;; swapped order. The restriction on overlap between operand 0
3206;; and operands 1 doesn't apply here.
3207(define_peephole2
3208 [(set (reg:CC CC_REGNUM)
3209 (compare:CC
3210 (match_operand:GPI 1 "register_operand")
3211 (match_operand:GPI 3 "const_int_operand")))
341fca1b 3212 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f 3213 (plus:GPI (match_dup 1)
a58fe3c5 3214 (match_operand:GPI 2 "aarch64_plus_immediate")))]
de47f61f
KT		 3215 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3216 [(const_int 0)]
3217 {
3218 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3219 operands[3], operands[2]));
de47f61f
KT		 3220 DONE;
3221 }
3222)
3223
43e9d192 3224(define_insn "*sub_<shift>_<mode>"
24c56925 3225 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 3226 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3227 (ASHIFT:GPI
3228 (match_operand:GPI 1 "register_operand" "r")
3229 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3230 ""
3231 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
ae27ce51 3232 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 3233)
3234
3b585938
IB		 3235;; zero_extend version of above
3236(define_insn "*sub_<shift>_si_uxtw"
24c56925 3237 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 3238 (zero_extend:DI
3239 (minus:SI (match_operand:SI 3 "register_operand" "r")
3240 (ASHIFT:SI
3241 (match_operand:SI 1 "register_operand" "r")
3242 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3243 ""
3244 "sub\\t%w0, %w3, %w1, <shift> %2"
ae27ce51 3245 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 3246)
3247
43e9d192
IB		 3248(define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3249 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3250 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3251 (ANY_EXTEND:GPI
3252 (match_operand:ALLX 2 "register_operand" "r"))))]
3253 ""
d4febc75 3254 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
99e6af15 3255 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3256)
3257
3b585938
IB		 3258;; zero_extend version of above
3259(define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3260 [(set (match_operand:DI 0 "register_operand" "=rk")
3261 (zero_extend:DI
7eb8fb77 3262 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3263 (ANY_EXTEND:SI
3264 (match_operand:SHORT 2 "register_operand" "r")))))]
3265 ""
3266 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
99e6af15 3267 [(set_attr "type" "alu_ext")]
3b585938
IB		 3268)
3269
43e9d192
IB		 3270(define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3271 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3272 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3273 (ashift:GPI (ANY_EXTEND:GPI
3274 (match_operand:ALLX 2 "register_operand" "r"))
3275 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3276 ""
d4febc75 3277 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
99e6af15 3278 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3279)
3280
3b585938
IB		 3281;; zero_extend version of above
3282(define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3283 [(set (match_operand:DI 0 "register_operand" "=rk")
3284 (zero_extend:DI
7eb8fb77 3285 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3286 (ashift:SI (ANY_EXTEND:SI
3287 (match_operand:SHORT 2 "register_operand" "r"))
3288 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3289 ""
3290 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
99e6af15 3291 [(set_attr "type" "alu_ext")]
3b585938
IB		 3292)
3293
ef22810a
RH		 3294;; The hardware description is op1 + ~op2 + C.
3295;; = op1 + (-op2 + 1) + (1 - !C)
3296;; = op1 - op2 - 1 + 1 - !C
3297;; = op1 - op2 - !C.
3298;; We describe the latter.
3299
3300(define_insn "*sub<mode>3_carryin0"
3301 [(set (match_operand:GPI 0 "register_operand" "=r")
3302 (minus:GPI
3303 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3304 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3305 ""
3306 "sbc\\t%<w>0, %<w>1, <w>zr"
3307 [(set_attr "type" "adc_reg")]
3308)
3309
3310;; zero_extend version of the above
3311(define_insn "*subsi3_carryin_uxtw"
3312 [(set (match_operand:DI 0 "register_operand" "=r")
3313 (zero_extend:DI
3314 (minus:SI
3315 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3316 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3317 ""
3318 "sbc\\t%w0, %w1, wzr"
3319 [(set_attr "type" "adc_reg")]
3320)
3321
3322(define_expand "sub<mode>3_carryin"
3323 [(set (match_operand:GPI 0 "register_operand")
3324 (minus:GPI
3325 (minus:GPI
3326 (match_operand:GPI 1 "aarch64_reg_or_zero")
3327 (match_operand:GPI 2 "register_operand"))
3328 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3329 ""
3330 ""
3331)
3332
3333(define_insn "*sub<mode>3_carryin"
3334 [(set (match_operand:GPI 0 "register_operand" "=r")
3335 (minus:GPI
3336 (minus:GPI
3337 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3338 (match_operand:GPI 2 "register_operand" "r"))
3339 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3340
0b8cdc58
IB		 3341 ""
3342 "sbc\\t%<w>0, %<w>1, %<w>2"
99e6af15 3343 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3344)
3345
3346;; zero_extend version of the above
3347(define_insn "*subsi3_carryin_uxtw"
ef22810a
RH		 3348 [(set (match_operand:DI 0 "register_operand" "=r")
3349 (zero_extend:DI
3350 (minus:SI
3351 (minus:SI
3352 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3353 (match_operand:SI 2 "register_operand" "r"))
3354 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3355
3356 ""
3357 "sbc\\t%w0, %w1, %w2"
3358 [(set_attr "type" "adc_reg")]
3359)
3360
3361(define_insn "*sub<mode>3_carryin_alt"
3362 [(set (match_operand:GPI 0 "register_operand" "=r")
3363 (minus:GPI
3364 (minus:GPI
3365 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3366 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3367 (match_operand:GPI 2 "register_operand" "r")))]
3368 ""
3369 "sbc\\t%<w>0, %<w>1, %<w>2"
3370 [(set_attr "type" "adc_reg")]
3371)
3372
3373;; zero_extend version of the above
3374(define_insn "*subsi3_carryin_alt_uxtw"
3375 [(set (match_operand:DI 0 "register_operand" "=r")
3376 (zero_extend:DI
3377 (minus:SI
3378 (minus:SI
3379 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3380 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3381 (match_operand:SI 2 "register_operand" "r"))))]
0b8cdc58
IB		 3382 ""
3383 "sbc\\t%w0, %w1, %w2"
99e6af15 3384 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3385)
3386
a58fe3c5 3387(define_expand "usub<GPI:mode>3_carryinC"
30c46053
MC		 3388 [(parallel
3389 [(set (reg:CC CC_REGNUM)
3390 (compare:CC
a58fe3c5 3391 (zero_extend:<DWI>
1bbffb87 3392 (match_operand:GPI 1 "aarch64_reg_or_zero"))
30c46053 3393 (plus:<DWI>
a58fe3c5 3394 (zero_extend:<DWI>
1bbffb87 3395 (match_operand:GPI 2 "register_operand"))
30c46053 3396 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
1bbffb87 3397 (set (match_operand:GPI 0 "register_operand")
30c46053
MC		 3398 (minus:GPI
3399 (minus:GPI (match_dup 1) (match_dup 2))
3400 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3401 ""
3402)
3403
a58fe3c5 3404(define_insn "*usub<GPI:mode>3_carryinC_z1"
30c46053
MC		 3405 [(set (reg:CC CC_REGNUM)
3406 (compare:CC
3407 (const_int 0)
3408 (plus:<DWI>
a58fe3c5 3409 (zero_extend:<DWI>
30c46053
MC		 3410 (match_operand:GPI 1 "register_operand" "r"))
3411 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3412 (set (match_operand:GPI 0 "register_operand" "=r")
3413 (minus:GPI
3414 (neg:GPI (match_dup 1))
3415 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3416 ""
3417 "sbcs\\t%<w>0, <w>zr, %<w>1"
3418 [(set_attr "type" "adc_reg")]
3419)
3420
a58fe3c5 3421(define_insn "*usub<GPI:mode>3_carryinC_z2"
30c46053
MC		 3422 [(set (reg:CC CC_REGNUM)
3423 (compare:CC
a58fe3c5 3424 (zero_extend:<DWI>
30c46053
MC		 3425 (match_operand:GPI 1 "register_operand" "r"))
3426 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3427 (set (match_operand:GPI 0 "register_operand" "=r")
3428 (minus:GPI
3429 (match_dup 1)
3430 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3431 ""
3432 "sbcs\\t%<w>0, %<w>1, <w>zr"
3433 [(set_attr "type" "adc_reg")]
3434)
3435
a58fe3c5 3436(define_insn "*usub<GPI:mode>3_carryinC"
30c46053
MC		 3437 [(set (reg:CC CC_REGNUM)
3438 (compare:CC
a58fe3c5 3439 (zero_extend:<DWI>
30c46053
MC		 3440 (match_operand:GPI 1 "register_operand" "r"))
3441 (plus:<DWI>
a58fe3c5 3442 (zero_extend:<DWI>
30c46053
MC		 3443 (match_operand:GPI 2 "register_operand" "r"))
3444 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3445 (set (match_operand:GPI 0 "register_operand" "=r")
3446 (minus:GPI
3447 (minus:GPI (match_dup 1) (match_dup 2))
3448 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3449 ""
3450 "sbcs\\t%<w>0, %<w>1, %<w>2"
3451 [(set_attr "type" "adc_reg")]
3452)
3453
a58fe3c5
RE		 3454(define_expand "sub<GPI:mode>3_carryinV"
3455 [(parallel
3456 [(set (reg:CC_V CC_REGNUM)
3457 (compare:CC_V
3458 (minus:<DWI>
3459 (sign_extend:<DWI>
1bbffb87 3460 (match_operand:GPI 1 "aarch64_reg_or_zero"))
a58fe3c5
RE		 3461 (plus:<DWI>
3462 (sign_extend:<DWI>
1bbffb87 3463 (match_operand:GPI 2 "register_operand"))
a58fe3c5
RE		 3464 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3465 (sign_extend:<DWI>
3466 (minus:GPI (match_dup 1)
3467 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3468 (match_dup 2))))))
1bbffb87 3469 (set (match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 3470 (minus:GPI
3471 (minus:GPI (match_dup 1) (match_dup 2))
3472 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3473 ""
3474)
3475
3476(define_insn "*sub<mode>3_carryinV_z2"
3477 [(set (reg:CC_V CC_REGNUM)
3478 (compare:CC_V
3479 (minus:<DWI>
3480 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3481 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3482 (sign_extend:<DWI>
3483 (minus:GPI (match_dup 1)
3484 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3485 (set (match_operand:GPI 0 "register_operand" "=r")
3486 (minus:GPI
3487 (match_dup 1) (match_dup 3)))]
3488 ""
3489 "sbcs\\t%<w>0, %<w>1, <w>zr"
3490 [(set_attr "type" "adc_reg")]
3491)
3492
3493(define_insn "*sub<mode>3_carryinV"
3494 [(set (reg:CC_V CC_REGNUM)
3495 (compare:CC_V
3496 (minus:<DWI>
3497 (sign_extend:<DWI>
3498 (match_operand:GPI 1 "register_operand" "r"))
3499 (plus:<DWI>
3500 (sign_extend:<DWI>
3501 (match_operand:GPI 2 "register_operand" "r"))
3502 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3503 (sign_extend:<DWI>
3504 (minus:GPI
3505 (match_dup 1)
3506 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3507 (match_dup 2))))))
3508 (set (match_operand:GPI 0 "register_operand" "=r")
3509 (minus:GPI
3510 (minus:GPI (match_dup 1) (match_dup 2))
3511 (match_dup 4)))]
3512 ""
3513 "sbcs\\t%<w>0, %<w>1, %<w>2"
3514 [(set_attr "type" "adc_reg")]
3515)
3516
dd1bdce4
VK		 3517(define_insn "*sub_uxt<mode>_shift2"
3518 [(set (match_operand:GPI 0 "register_operand" "=rk")
3519 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3520 (and:GPI
3521 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3522 (match_operand 2 "aarch64_imm3" "Ui3"))
3523 (match_operand 3 "const_int_operand" "n"))))]
3524 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3525 "*
3526 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3527 INTVAL (operands[3])));
d4febc75 3528 return \"sub\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
dd1bdce4
VK		 3529 [(set_attr "type" "alu_ext")]
3530)
3531
3532;; zero_extend version of above
3533(define_insn "*sub_uxtsi_shift2_uxtw"
3534 [(set (match_operand:DI 0 "register_operand" "=rk")
3535 (zero_extend:DI
3536 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3537 (and:SI
3538 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3539 (match_operand 2 "aarch64_imm3" "Ui3"))
3540 (match_operand 3 "const_int_operand" "n")))))]
3541 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3542 "*
3543 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3544 INTVAL (operands[3])));
3545 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3546 [(set_attr "type" "alu_ext")]
3547)
3548
fc8b587d 3549(define_expand "abs<mode>2"
1bbffb87
DZ		 3550 [(match_operand:GPI 0 "register_operand")
3551 (match_operand:GPI 1 "register_operand")]
26366d28 3552 ""
26366d28 3553 {
fc8b587d
WD		 3554 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3555 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3556 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
26366d28
IB		 3557 DONE;
3558 }
26366d28
IB		 3559)
3560
43e9d192 3561(define_insn "neg<mode>2"
040d8a1c
IB		 3562 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3563 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 3564 ""
040d8a1c
IB		 3565 "@
3566 neg\\t%<w>0, %<w>1
3567 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
1d61feeb 3568 [(set_attr "type" "alu_sreg, neon_neg<q>")
488461d8 3569 (set_attr "arch" "*,simd")]
43e9d192
IB		 3570)
3571
3b585938
IB		 3572;; zero_extend version of above
3573(define_insn "*negsi2_uxtw"
3574 [(set (match_operand:DI 0 "register_operand" "=r")
3575 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3576 ""
3577 "neg\\t%w0, %w1"
1d61feeb 3578 [(set_attr "type" "alu_sreg")]
3b585938
IB		 3579)
3580
7e0228bf
N		 3581(define_insn "*ngc<mode>"
3582 [(set (match_operand:GPI 0 "register_operand" "=r")
ef22810a
RH		 3583 (minus:GPI
3584 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3585 (match_operand:GPI 1 "register_operand" "r")))]
7e0228bf
N		 3586 ""
3587 "ngc\\t%<w>0, %<w>1"
99e6af15 3588 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3589)
3590
3591(define_insn "*ngcsi_uxtw"
3592 [(set (match_operand:DI 0 "register_operand" "=r")
3593 (zero_extend:DI
ef22810a
RH		 3594 (minus:SI
3595 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3596 (match_operand:SI 1 "register_operand" "r"))))]
7e0228bf
N		 3597 ""
3598 "ngc\\t%w0, %w1"
99e6af15 3599 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3600)
3601
4f58fe36 3602(define_insn "neg<mode>2_compare0"
43e9d192
IB		 3603 [(set (reg:CC_NZ CC_REGNUM)
3604 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3605 (const_int 0)))
3606 (set (match_operand:GPI 0 "register_operand" "=r")
3607 (neg:GPI (match_dup 1)))]
3608 ""
3609 "negs\\t%<w>0, %<w>1"
1d61feeb 3610 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 3611)
3612
3b585938
IB		 3613;; zero_extend version of above
3614(define_insn "*negsi2_compare0_uxtw"
3615 [(set (reg:CC_NZ CC_REGNUM)
3616 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3617 (const_int 0)))
3618 (set (match_operand:DI 0 "register_operand" "=r")
3619 (zero_extend:DI (neg:SI (match_dup 1))))]
3620 ""
3621 "negs\\t%w0, %w1"
1d61feeb 3622 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3623)
3624
b056c910
N		 3625(define_insn "*neg_<shift><mode>3_compare0"
3626 [(set (reg:CC_NZ CC_REGNUM)
3627 (compare:CC_NZ
3628 (neg:GPI (ASHIFT:GPI
3629 (match_operand:GPI 1 "register_operand" "r")
3630 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3631 (const_int 0)))
3632 (set (match_operand:GPI 0 "register_operand" "=r")
3633 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3634 ""
3635 "negs\\t%<w>0, %<w>1, <shift> %2"
99e6af15 3636 [(set_attr "type" "alus_shift_imm")]
b056c910
N		 3637)
3638
43e9d192
IB		 3639(define_insn "*neg_<shift>_<mode>2"
3640 [(set (match_operand:GPI 0 "register_operand" "=r")
3641 (neg:GPI (ASHIFT:GPI
3642 (match_operand:GPI 1 "register_operand" "r")
3643 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3644 ""
3645 "neg\\t%<w>0, %<w>1, <shift> %2"
ae27ce51 3646 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
43e9d192
IB		 3647)
3648
3b585938
IB		 3649;; zero_extend version of above
3650(define_insn "*neg_<shift>_si2_uxtw"
3651 [(set (match_operand:DI 0 "register_operand" "=r")
3652 (zero_extend:DI
3653 (neg:SI (ASHIFT:SI
3654 (match_operand:SI 1 "register_operand" "r")
3655 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3656 ""
3657 "neg\\t%w0, %w1, <shift> %2"
ae27ce51 3658 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3b585938
IB		 3659)
3660
47f42744
JJ		 3661(define_insn "*neg_asr_si2_extr"
3662 [(set (match_operand:SI 0 "register_operand" "=r")
3663 (neg:SI (match_operator:SI 4 "subreg_lowpart_operator"
3664 [(sign_extract:DI
3665 (match_operand:DI 1 "register_operand" "r")
3666 (match_operand 3 "aarch64_simd_shift_imm_offset_si" "n")
3667 (match_operand 2 "aarch64_simd_shift_imm_offset_si" "n"))])))]
3668 "INTVAL (operands[2]) + INTVAL (operands[3]) == 32"
3669 "neg\\t%w0, %w1, asr %2"
3670 [(set_attr "autodetect_type" "alu_shift_asr_op2")]
3671)
3672
43e9d192
IB		 3673(define_insn "mul<mode>3"
3674 [(set (match_operand:GPI 0 "register_operand" "=r")
3675 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3676 (match_operand:GPI 2 "register_operand" "r")))]
3677 ""
3678 "mul\\t%<w>0, %<w>1, %<w>2"
99e6af15 3679 [(set_attr "type" "mul")]
43e9d192
IB		 3680)
3681
3b585938
IB		 3682;; zero_extend version of above
3683(define_insn "*mulsi3_uxtw"
3684 [(set (match_operand:DI 0 "register_operand" "=r")
3685 (zero_extend:DI
3686 (mult:SI (match_operand:SI 1 "register_operand" "r")
3687 (match_operand:SI 2 "register_operand" "r"))))]
3688 ""
3689 "mul\\t%w0, %w1, %w2"
99e6af15 3690 [(set_attr "type" "mul")]
3b585938
IB		 3691)
3692
2195867f 3693(define_insn "madd<mode>"
43e9d192
IB		 3694 [(set (match_operand:GPI 0 "register_operand" "=r")
3695 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3696 (match_operand:GPI 2 "register_operand" "r"))
3697 (match_operand:GPI 3 "register_operand" "r")))]
3698 ""
3699 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3700 [(set_attr "type" "mla")]
43e9d192
IB		 3701)
3702
3b585938
IB		 3703;; zero_extend version of above
3704(define_insn "*maddsi_uxtw"
3705 [(set (match_operand:DI 0 "register_operand" "=r")
3706 (zero_extend:DI
3707 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3708 (match_operand:SI 2 "register_operand" "r"))
3709 (match_operand:SI 3 "register_operand" "r"))))]
3710 ""
3711 "madd\\t%w0, %w1, %w2, %w3"
99e6af15 3712 [(set_attr "type" "mla")]
3b585938
IB		 3713)
3714
43e9d192
IB		 3715(define_insn "*msub<mode>"
3716 [(set (match_operand:GPI 0 "register_operand" "=r")
3717 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3718 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3719 (match_operand:GPI 2 "register_operand" "r"))))]
3720
3721 ""
3722 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3723 [(set_attr "type" "mla")]
43e9d192
IB		 3724)
3725
3b585938
IB		 3726;; zero_extend version of above
3727(define_insn "*msubsi_uxtw"
3728 [(set (match_operand:DI 0 "register_operand" "=r")
3729 (zero_extend:DI
3730 (minus:SI (match_operand:SI 3 "register_operand" "r")
3731 (mult:SI (match_operand:SI 1 "register_operand" "r")
3732 (match_operand:SI 2 "register_operand" "r")))))]
3733
3734 ""
3735 "msub\\t%w0, %w1, %w2, %w3"
99e6af15 3736 [(set_attr "type" "mla")]
3b585938
IB		 3737)
3738
43e9d192
IB		 3739(define_insn "*mul<mode>_neg"
3740 [(set (match_operand:GPI 0 "register_operand" "=r")
3741 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3742 (match_operand:GPI 2 "register_operand" "r")))]
3743
3744 ""
3745 "mneg\\t%<w>0, %<w>1, %<w>2"
99e6af15 3746 [(set_attr "type" "mul")]
43e9d192
IB		 3747)
3748
3b585938
IB		 3749;; zero_extend version of above
3750(define_insn "*mulsi_neg_uxtw"
3751 [(set (match_operand:DI 0 "register_operand" "=r")
3752 (zero_extend:DI
3753 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3754 (match_operand:SI 2 "register_operand" "r"))))]
3755
3756 ""
3757 "mneg\\t%w0, %w1, %w2"
99e6af15 3758 [(set_attr "type" "mul")]
3b585938
IB		 3759)
3760
43e9d192
IB		 3761(define_insn "<su_optab>mulsidi3"
3762 [(set (match_operand:DI 0 "register_operand" "=r")
3763 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3764 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3765 ""
3766 "<su>mull\\t%0, %w1, %w2"
99e6af15 3767 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3768)
3769
3770(define_insn "<su_optab>maddsidi4"
3771 [(set (match_operand:DI 0 "register_operand" "=r")
3772 (plus:DI (mult:DI
3773 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3774 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3775 (match_operand:DI 3 "register_operand" "r")))]
3776 ""
3777 "<su>maddl\\t%0, %w1, %w2, %3"
99e6af15 3778 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3779)
3780
3781(define_insn "<su_optab>msubsidi4"
3782 [(set (match_operand:DI 0 "register_operand" "=r")
3783 (minus:DI
3784 (match_operand:DI 3 "register_operand" "r")
3785 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3786 (ANY_EXTEND:DI
3787 (match_operand:SI 2 "register_operand" "r")))))]
3788 ""
3789 "<su>msubl\\t%0, %w1, %w2, %3"
99e6af15 3790 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3791)
3792
3793(define_insn "*<su_optab>mulsidi_neg"
3794 [(set (match_operand:DI 0 "register_operand" "=r")
3795 (mult:DI (neg:DI
3796 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3797 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3798 ""
3799 "<su>mnegl\\t%0, %w1, %w2"
99e6af15 3800 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3801)
3802
2195867f
RH		 3803(define_expand "<su_optab>mulditi3"
3804 [(set (match_operand:TI 0 "register_operand")
3805 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3806 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3807 ""
3808{
3809 rtx low = gen_reg_rtx (DImode);
3810 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3811
3812 rtx high = gen_reg_rtx (DImode);
3813 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3814
3815 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3816 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3817 DONE;
3818})
3819
3820;; The default expansion of multi3 using umuldi3_highpart will perform
3821;; the additions in an order that fails to combine into two madd insns.
3822(define_expand "multi3"
3823 [(set (match_operand:TI 0 "register_operand")
3824 (mult:TI (match_operand:TI 1 "register_operand")
3825 (match_operand:TI 2 "register_operand")))]
3826 ""
3827{
3828 rtx l0 = gen_reg_rtx (DImode);
3829 rtx l1 = gen_lowpart (DImode, operands[1]);
3830 rtx l2 = gen_lowpart (DImode, operands[2]);
3831 rtx h0 = gen_reg_rtx (DImode);
3832 rtx h1 = gen_highpart (DImode, operands[1]);
3833 rtx h2 = gen_highpart (DImode, operands[2]);
3834
3835 emit_insn (gen_muldi3 (l0, l1, l2));
3836 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3837 emit_insn (gen_madddi (h0, h1, l2, h0));
3838 emit_insn (gen_madddi (h0, l1, h2, h0));
3839
3840 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3841 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3842 DONE;
3843})
3844
43e9d192
IB		 3845(define_insn "<su>muldi3_highpart"
3846 [(set (match_operand:DI 0 "register_operand" "=r")
3847 (truncate:DI
3848 (lshiftrt:TI
3849 (mult:TI
3850 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3851 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3852 (const_int 64))))]
3853 ""
3854 "<su>mulh\\t%0, %1, %2"
99e6af15 3855 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3856)
3857
3858(define_insn "<su_optab>div<mode>3"
3859 [(set (match_operand:GPI 0 "register_operand" "=r")
3860 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3861 (match_operand:GPI 2 "register_operand" "r")))]
3862 ""
3863 "<su>div\\t%<w>0, %<w>1, %<w>2"
99e6af15 3864 [(set_attr "type" "<su>div")]
43e9d192
IB		 3865)
3866
3b585938
IB		 3867;; zero_extend version of above
3868(define_insn "*<su_optab>divsi3_uxtw"
3869 [(set (match_operand:DI 0 "register_operand" "=r")
3870 (zero_extend:DI
3871 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3872 (match_operand:SI 2 "register_operand" "r"))))]
3873 ""
3874 "<su>div\\t%w0, %w1, %w2"
99e6af15 3875 [(set_attr "type" "<su>div")]
3b585938
IB		 3876)
3877
43e9d192
IB		 3878;; -------------------------------------------------------------------
3879;; Comparison insns
3880;; -------------------------------------------------------------------
3881
c8012fbc 3882(define_insn "cmp<mode>"
43e9d192 3883 [(set (reg:CC CC_REGNUM)
3751345d 3884 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
34b512f5 3885 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
43e9d192
IB		 3886 ""
3887 "@
34b512f5 3888 cmp\\t%<w>0, %<w>1
e2b691c4 3889 cmp\\t%<w>0, %1
43e9d192 3890 cmn\\t%<w>0, #%n1"
1d61feeb 3891 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 3892)
3893
786e3c06 3894(define_insn "fcmp<mode>"
43e9d192
IB		 3895 [(set (reg:CCFP CC_REGNUM)
3896 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3897 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3898 "TARGET_FLOAT"
3899 "@
3900 fcmp\\t%<s>0, #0.0
3901 fcmp\\t%<s>0, %<s>1"
99e6af15 3902 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3903)
3904
786e3c06 3905(define_insn "fcmpe<mode>"
43e9d192
IB		 3906 [(set (reg:CCFPE CC_REGNUM)
3907 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3908 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3909 "TARGET_FLOAT"
3910 "@
3911 fcmpe\\t%<s>0, #0.0
3912 fcmpe\\t%<s>0, %<s>1"
99e6af15 3913 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3914)
3915
3916(define_insn "*cmp_swp_<shift>_reg<mode>"
3917 [(set (reg:CC_SWP CC_REGNUM)
3918 (compare:CC_SWP (ASHIFT:GPI
3919 (match_operand:GPI 0 "register_operand" "r")
3920 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3921 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3922 ""
3923 "cmp\\t%<w>2, %<w>0, <shift> %1"
99e6af15 3924 [(set_attr "type" "alus_shift_imm")]
43e9d192
IB		 3925)
3926
3927(define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3928 [(set (reg:CC_SWP CC_REGNUM)
3929 (compare:CC_SWP (ANY_EXTEND:GPI
3930 (match_operand:ALLX 0 "register_operand" "r"))
3931 (match_operand:GPI 1 "register_operand" "r")))]
3932 ""
d4febc75 3933 "cmp\\t%<GPI:w>1, %w0, <su>xt<ALLX:size>"
99e6af15 3934 [(set_attr "type" "alus_ext")]
43e9d192
IB		 3935)
3936
58cff58c
N		 3937(define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3938 [(set (reg:CC_SWP CC_REGNUM)
3939 (compare:CC_SWP (ashift:GPI
3940 (ANY_EXTEND:GPI
3941 (match_operand:ALLX 0 "register_operand" "r"))
4095f9fa 3942 (match_operand 1 "aarch64_imm3" "Ui3"))
58cff58c
N		 3943 (match_operand:GPI 2 "register_operand" "r")))]
3944 ""
d4febc75 3945 "cmp\\t%<GPI:w>2, %w0, <su>xt<ALLX:size> %1"
99e6af15 3946 [(set_attr "type" "alus_ext")]
58cff58c 3947)
43e9d192
IB		 3948
3949;; -------------------------------------------------------------------
3950;; Store-flag and conditional select insns
3951;; -------------------------------------------------------------------
3952
3953(define_expand "cstore<mode>4"
1bbffb87 3954 [(set (match_operand:SI 0 "register_operand")
43e9d192 3955 (match_operator:SI 1 "aarch64_comparison_operator"
1bbffb87
DZ		 3956 [(match_operand:GPI 2 "register_operand")
3957 (match_operand:GPI 3 "aarch64_plus_operand")]))]
43e9d192
IB		 3958 ""
3959 "
3960 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3961 operands[3]);
3962 operands[3] = const0_rtx;
3963 "
3964)
3965
cf670503
ZC		 3966(define_expand "cstorecc4"
3967 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 3968 (match_operator 1 "aarch64_comparison_operator_mode"
c8012fbc 3969 [(match_operand 2 "cc_register")
cf670503
ZC		 3970 (match_operand 3 "const0_operand")]))]
3971 ""
3972"{
f7df4a84 3973 emit_insn (gen_rtx_SET (operands[0], operands[1]));
cf670503
ZC		 3974 DONE;
3975}")
3976
3977
43e9d192 3978(define_expand "cstore<mode>4"
1bbffb87 3979 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 3980 (match_operator:SI 1 "aarch64_comparison_operator_mode"
1bbffb87
DZ		 3981 [(match_operand:GPF 2 "register_operand")
3982 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
43e9d192
IB		 3983 ""
3984 "
3985 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3986 operands[3]);
3987 operands[3] = const0_rtx;
3988 "
3989)
3990
714e1b3b 3991(define_insn "aarch64_cstore<mode>"
43e9d192 3992 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 3993 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 3994 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3995 ""
3996 "cset\\t%<w>0, %m1"
99e6af15 3997 [(set_attr "type" "csel")]
43e9d192
IB		 3998)
3999
714e1b3b
KT		 4000;; For a 24-bit immediate CST we can optimize the compare for equality
4001;; and branch sequence from:
4002;; mov x0, #imm1
4003;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4004;; cmp x1, x0
4005;; cset x2, <ne,eq>
4006;; into the shorter:
4007;; sub x0, x1, #(CST & 0xfff000)
4008;; subs x0, x0, #(CST & 0x000fff)
4009;; cset x2, <ne, eq>.
4010(define_insn_and_split "*compare_cstore<mode>_insn"
4011 [(set (match_operand:GPI 0 "register_operand" "=r")
4012 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
c8574943
KT		 4013 (match_operand:GPI 2 "aarch64_imm24" "n")))
4014 (clobber (reg:CC CC_REGNUM))]
714e1b3b
KT		 4015 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4016 && !aarch64_plus_operand (operands[2], <MODE>mode)
4017 && !reload_completed"
4018 "#"
4019 "&& true"
4020 [(const_int 0)]
4021 {
4022 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4023 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4024 rtx tmp = gen_reg_rtx (<MODE>mode);
4025 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4026 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4027 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4028 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4029 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4030 DONE;
4031 }
4032 [(set_attr "type" "csel")]
4033)
4034
d0b6bb1b
IB		 4035;; zero_extend version of the above
4036(define_insn "*cstoresi_insn_uxtw"
4037 [(set (match_operand:DI 0 "register_operand" "=r")
4038 (zero_extend:DI
a44e0ebb 4039 (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 4040 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4041 ""
4042 "cset\\t%w0, %m1"
99e6af15 4043 [(set_attr "type" "csel")]
d0b6bb1b
IB		 4044)
4045
889b9412 4046(define_insn "cstore<mode>_neg"
43e9d192 4047 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 4048 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 4049 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4050 ""
4051 "csetm\\t%<w>0, %m1"
99e6af15 4052 [(set_attr "type" "csel")]
43e9d192
IB		 4053)
4054
d0b6bb1b
IB		 4055;; zero_extend version of the above
4056(define_insn "*cstoresi_neg_uxtw"
4057 [(set (match_operand:DI 0 "register_operand" "=r")
4058 (zero_extend:DI
a44e0ebb 4059 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 4060 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4061 ""
4062 "csetm\\t%w0, %m1"
99e6af15 4063 [(set_attr "type" "csel")]
d0b6bb1b
IB		 4064)
4065
43e9d192 4066(define_expand "cmov<mode>6"
1bbffb87 4067 [(set (match_operand:GPI 0 "register_operand")
43e9d192
IB		 4068 (if_then_else:GPI
4069 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 4070 [(match_operand:GPI 2 "register_operand")
4071 (match_operand:GPI 3 "aarch64_plus_operand")])
4072 (match_operand:GPI 4 "register_operand")
4073 (match_operand:GPI 5 "register_operand")))]
43e9d192
IB		 4074 ""
4075 "
4076 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4077 operands[3]);
4078 operands[3] = const0_rtx;
4079 "
4080)
4081
4082(define_expand "cmov<mode>6"
1bbffb87 4083 [(set (match_operand:GPF 0 "register_operand")
43e9d192
IB		 4084 (if_then_else:GPF
4085 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 4086 [(match_operand:GPF 2 "register_operand")
4087 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4088 (match_operand:GPF 4 "register_operand")
4089 (match_operand:GPF 5 "register_operand")))]
43e9d192
IB		 4090 ""
4091 "
4092 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4093 operands[3]);
4094 operands[3] = const0_rtx;
4095 "
4096)
4097
4098(define_insn "*cmov<mode>_insn"
06b1198f 4099 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
43e9d192
IB		 4100 (if_then_else:ALLI
4101 (match_operator 1 "aarch64_comparison_operator"
4102 [(match_operand 2 "cc_register" "") (const_int 0)])
06b1198f
IB		 4103 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4104 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4105 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4106 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4107 ;; Final two alternatives should be unreachable, but included for completeness
43e9d192
IB		 4108 "@
4109 csel\\t%<w>0, %<w>3, %<w>4, %m1
4110 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4111 csinv\\t%<w>0, %<w>4, <w>zr, %M1
06b1198f
IB		 4112 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4113 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4114 mov\\t%<w>0, -1
4115 mov\\t%<w>0, 1"
04e34811 4116 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
43e9d192
IB		 4117)
4118
d0b6bb1b
IB		 4119;; zero_extend version of above
4120(define_insn "*cmovsi_insn_uxtw"
4121 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4122 (zero_extend:DI
4123 (if_then_else:SI
4124 (match_operator 1 "aarch64_comparison_operator"
4125 [(match_operand 2 "cc_register" "") (const_int 0)])
4126 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4127 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4128 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4129 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4130 ;; Final two alternatives should be unreachable, but included for completeness
4131 "@
4132 csel\\t%w0, %w3, %w4, %m1
4133 csinv\\t%w0, %w3, wzr, %m1
4134 csinv\\t%w0, %w4, wzr, %M1
4135 csinc\\t%w0, %w3, wzr, %m1
4136 csinc\\t%w0, %w4, wzr, %M1
4137 mov\\t%w0, -1
4138 mov\\t%w0, 1"
04e34811 4139 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
d0b6bb1b
IB		 4140)
4141
1d3e6ed3
RL		 4142(define_insn "*cmovdi_insn_uxtw"
4143 [(set (match_operand:DI 0 "register_operand" "=r")
4144 (if_then_else:DI
4145 (match_operator 1 "aarch64_comparison_operator"
4146 [(match_operand 2 "cc_register" "") (const_int 0)])
4147 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4148 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4149 ""
4150 "csel\\t%w0, %w3, %w4, %m1"
4151 [(set_attr "type" "csel")]
4152)
4153
43e9d192
IB		 4154(define_insn "*cmov<mode>_insn"
4155 [(set (match_operand:GPF 0 "register_operand" "=w")
4156 (if_then_else:GPF
4157 (match_operator 1 "aarch64_comparison_operator"
4158 [(match_operand 2 "cc_register" "") (const_int 0)])
4159 (match_operand:GPF 3 "register_operand" "w")
4160 (match_operand:GPF 4 "register_operand" "w")))]
4161 "TARGET_FLOAT"
4162 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
99e6af15 4163 [(set_attr "type" "fcsel")]
43e9d192
IB		 4164)
4165
4166(define_expand "mov<mode>cc"
1bbffb87
DZ		 4167 [(set (match_operand:ALLI 0 "register_operand")
4168 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4169 (match_operand:ALLI 2 "register_operand")
4170 (match_operand:ALLI 3 "register_operand")))]
43e9d192
IB		 4171 ""
4172 {
c8012fbc 4173 rtx ccreg;
43e9d192
IB		 4174 enum rtx_code code = GET_CODE (operands[1]);
4175
4176 if (code == UNEQ || code == LTGT)
4177 FAIL;
4178
c8012fbc
WD		 4179 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4180 XEXP (operands[1], 1));
4181 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
43e9d192
IB		 4182 }
4183)
4184
4185(define_expand "mov<GPF:mode><GPI:mode>cc"
1bbffb87
DZ		 4186 [(set (match_operand:GPI 0 "register_operand")
4187 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4188 (match_operand:GPF 2 "register_operand")
4189 (match_operand:GPF 3 "register_operand")))]
43e9d192
IB		 4190 ""
4191 {
4192 rtx ccreg;
4193 enum rtx_code code = GET_CODE (operands[1]);
4194
4195 if (code == UNEQ || code == LTGT)
4196 FAIL;
4197
4198 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4199 XEXP (operands[1], 1));
4200 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4201 }
4202)
4203
d3a3182b 4204(define_expand "mov<mode>cc"
1bbffb87
DZ		 4205 [(set (match_operand:GPF 0 "register_operand")
4206 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4207 (match_operand:GPF 2 "register_operand")
4208 (match_operand:GPF 3 "register_operand")))]
d3a3182b
ZC		 4209 ""
4210 {
4211 rtx ccreg;
4212 enum rtx_code code = GET_CODE (operands[1]);
4213
4214 if (code == UNEQ || code == LTGT)
4215 FAIL;
4216
4217 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4218 XEXP (operands[1], 1));
4219 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4220 }
4221)
4222
3204ac98 4223(define_expand "<neg_not_op><mode>cc"
1bbffb87
DZ		 4224 [(set (match_operand:GPI 0 "register_operand")
4225 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4226 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4227 (match_operand:GPI 3 "register_operand")))]
3204ac98
KT		 4228 ""
4229 {
4230 rtx ccreg;
4231 enum rtx_code code = GET_CODE (operands[1]);
4232
4233 if (code == UNEQ || code == LTGT)
4234 FAIL;
4235
4236 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4237 XEXP (operands[1], 1));
4238 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4239 }
4240)
5d357f26
KT		 4241
4242;; CRC32 instructions.
4243(define_insn "aarch64_<crc_variant>"
4244 [(set (match_operand:SI 0 "register_operand" "=r")
4245 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4246 (match_operand:<crc_mode> 2 "register_operand" "r")]
4247 CRC))]
4248 "TARGET_CRC32"
4249 {
6a70badb 4250 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
5d357f26
KT		 4251 return "<crc_variant>\\t%w0, %w1, %x2";
4252 else
4253 return "<crc_variant>\\t%w0, %w1, %w2";
4254 }
4255 [(set_attr "type" "crc")]
4256)
4257
43e9d192
IB		 4258(define_insn "*csinc2<mode>_insn"
4259 [(set (match_operand:GPI 0 "register_operand" "=r")
cd5660ab
KT		 4260 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4261 (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4262 ""
49f88e43 4263 "cinc\\t%<w>0, %<w>1, %m2"
99e6af15
JG		 4264 [(set_attr "type" "csel")]
4265)
43e9d192
IB		 4266
4267(define_insn "csinc3<mode>_insn"
4268 [(set (match_operand:GPI 0 "register_operand" "=r")
4269 (if_then_else:GPI
cd5660ab
KT		 4270 (match_operand 1 "aarch64_comparison_operation" "")
4271 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
43e9d192 4272 (const_int 1))
cd5660ab 4273 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4274 ""
cd5660ab 4275 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15 4276 [(set_attr "type" "csel")]
43e9d192
IB		 4277)
4278
4279(define_insn "*csinv3<mode>_insn"
4280 [(set (match_operand:GPI 0 "register_operand" "=r")
4281 (if_then_else:GPI
cd5660ab
KT		 4282 (match_operand 1 "aarch64_comparison_operation" "")
4283 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4284 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4285 ""
cd5660ab 4286 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15
JG		 4287 [(set_attr "type" "csel")]
4288)
43e9d192 4289
db95ac77 4290(define_insn "*cs<neg_not_cs>3_uxtw_insn4"
0f6ca79c
RL		 4291 [(set (match_operand:DI 0 "register_operand" "=r")
4292 (zero_extend:DI
4293 (if_then_else:SI
4294 (match_operand 1 "aarch64_comparison_operation" "")
db95ac77 4295 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r"))
0f6ca79c
RL		 4296 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4297 ""
db95ac77 4298 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
0f6ca79c
RL		 4299 [(set_attr "type" "csel")]
4300)
4301
fc8b587d 4302(define_insn "csneg3<mode>_insn"
43e9d192
IB		 4303 [(set (match_operand:GPI 0 "register_operand" "=r")
4304 (if_then_else:GPI
cd5660ab
KT		 4305 (match_operand 1 "aarch64_comparison_operation" "")
4306 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4307 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4308 ""
cd5660ab 4309 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15 4310 [(set_attr "type" "csel")]
d572ad49
AC		 4311)
4312
4313(define_insn "*csinv3_uxtw_insn1"
4314 [(set (match_operand:DI 0 "register_operand" "=r")
4315 (if_then_else:DI
4316 (match_operand 1 "aarch64_comparison_operation" "")
4317 (zero_extend:DI
4318 (match_operand:SI 2 "register_operand" "r"))
4319 (zero_extend:DI
4320 (NEG_NOT:SI (match_operand:SI 3 "register_operand" "r")))))]
4321 ""
4322 "cs<neg_not_cs>\\t%w0, %w2, %w3, %m1"
4323 [(set_attr "type" "csel")]
4324)
4325
4326(define_insn "*csinv3_uxtw_insn2"
4327 [(set (match_operand:DI 0 "register_operand" "=r")
4328 (if_then_else:DI
4329 (match_operand 1 "aarch64_comparison_operation" "")
4330 (zero_extend:DI
4331 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4332 (zero_extend:DI
4333 (match_operand:SI 3 "register_operand" "r"))))]
4334 ""
4335 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
4336 [(set_attr "type" "csel")]
4337)
4338
4339(define_insn "*csinv3_uxtw_insn3"
4340 [(set (match_operand:DI 0 "register_operand" "=r")
4341 (if_then_else:DI
4342 (match_operand 1 "aarch64_comparison_operation" "")
4343 (zero_extend:DI
4344 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4345 (const_int 0)))]
4346 ""
4347 "cs<neg_not_cs>\\t%w0, wzr, %w2, %M1"
4348 [(set_attr "type" "csel")]
99e6af15 4349)
43e9d192 4350
7cfb4d93
RS		 4351;; If X can be loaded by a single CNT[BHWD] instruction,
4352;;
4353;; A = UMAX (B, X)
4354;;
4355;; is equivalent to:
4356;;
4357;; TMP = UQDEC[BHWD] (B, X)
4358;; A = TMP + X
4359;;
4360;; Defining the pattern this way means that:
4361;;
4362;; A = UMAX (B, X) - X
4363;;
4364;; becomes:
4365;;
4366;; TMP1 = UQDEC[BHWD] (B, X)
4367;; TMP2 = TMP1 + X
4368;; A = TMP2 - X
4369;;
4370;; which combine can optimize to:
4371;;
4372;; A = UQDEC[BHWD] (B, X)
4373;;
4374;; We don't use match_operand predicates because the order of the operands
4375;; can vary: the CNT[BHWD] constant will come first if the other operand is
4376;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4377(define_expand "umax<mode>3"
4378 [(set (match_operand:GPI 0 "register_operand")
4379 (umax:GPI (match_operand:GPI 1 "")
4380 (match_operand:GPI 2 "")))]
4381 "TARGET_SVE"
4382 {
4383 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4384 std::swap (operands[1], operands[2]);
4385 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4386 FAIL;
4387 rtx temp = gen_reg_rtx (<MODE>mode);
4388 operands[1] = force_reg (<MODE>mode, operands[1]);
4389 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4390 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4391 DONE;
4392 }
4393)
4394
4395;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4396(define_insn "aarch64_uqdec<mode>"
4397 [(set (match_operand:GPI 0 "register_operand" "=r")
4398 (minus:GPI
4399 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4400 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4401 (match_dup 2)))]
4402 "TARGET_SVE"
4403 {
4404 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4405 }
4406)
4407
43e9d192
IB		 4408;; -------------------------------------------------------------------
4409;; Logical operations
4410;; -------------------------------------------------------------------
4411
43fd192f
MC		 4412
4413(define_insn_and_split "*aarch64_and<mode>_imm2"
4414 [(set (match_operand:GPI 0 "register_operand" "=rk")
4415 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4416 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4417 ""
4418 "#"
4419 "true"
4420 [(const_int 0)]
4421 {
4422 HOST_WIDE_INT val = INTVAL (operands[2]);
4423 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4424 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4425
4426 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4427 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4428 DONE;
4429 }
4430)
4431
43e9d192 4432(define_insn "<optab><mode>3"
fe82d1f2
AL		 4433 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4434 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4435 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
43e9d192 4436 ""
fe82d1f2
AL		 4437 "@
4438 <logical>\\t%<w>0, %<w>1, %<w>2
e2b691c4 4439 <logical>\\t%<w>0, %<w>1, %2
fe82d1f2
AL		 4440 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4441 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
488461d8 4442 (set_attr "arch" "*,*,simd")]
99e6af15 4443)
43e9d192 4444
d0b6bb1b
IB		 4445;; zero_extend version of above
4446(define_insn "*<optab>si3_uxtw"
4447 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4448 (zero_extend:DI
4449 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4450 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4451 ""
954224d7
WD		 4452 "@
4453 <logical>\\t%w0, %w1, %w2
4454 <logical>\\t%w0, %w1, %2"
99e6af15
JG		 4455 [(set_attr "type" "logic_reg,logic_imm")]
4456)
d0b6bb1b 4457
2a1d1991
N		 4458(define_insn "*and<mode>3_compare0"
4459 [(set (reg:CC_NZ CC_REGNUM)
4460 (compare:CC_NZ
4461 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4462 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4463 (const_int 0)))
4464 (set (match_operand:GPI 0 "register_operand" "=r,r")
4465 (and:GPI (match_dup 1) (match_dup 2)))]
4466 ""
954224d7
WD		 4467 "@
4468 ands\\t%<w>0, %<w>1, %<w>2
4469 ands\\t%<w>0, %<w>1, %2"
99e6af15 4470 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4471)
4472
4473;; zero_extend version of above
4474(define_insn "*andsi3_compare0_uxtw"
4475 [(set (reg:CC_NZ CC_REGNUM)
4476 (compare:CC_NZ
4477 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4478 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4479 (const_int 0)))
4480 (set (match_operand:DI 0 "register_operand" "=r,r")
4481 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4482 ""
954224d7
WD		 4483 "@
4484 ands\\t%w0, %w1, %w2
4485 ands\\t%w0, %w1, %2"
99e6af15 4486 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4487)
4488
4489(define_insn "*and_<SHIFT:optab><mode>3_compare0"
4490 [(set (reg:CC_NZ CC_REGNUM)
4491 (compare:CC_NZ
4492 (and:GPI (SHIFT:GPI
4493 (match_operand:GPI 1 "register_operand" "r")
4494 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4495 (match_operand:GPI 3 "register_operand" "r"))
4496 (const_int 0)))
4497 (set (match_operand:GPI 0 "register_operand" "=r")
4498 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4499 ""
4500 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15 4501 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4502)
4503
4504;; zero_extend version of above
4505(define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4506 [(set (reg:CC_NZ CC_REGNUM)
4507 (compare:CC_NZ
4508 (and:SI (SHIFT:SI
4509 (match_operand:SI 1 "register_operand" "r")
4510 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4511 (match_operand:SI 3 "register_operand" "r"))
4512 (const_int 0)))
4513 (set (match_operand:DI 0 "register_operand" "=r")
4514 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4515 (match_dup 3))))]
4516 ""
4517 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15 4518 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4519)
4520
43e9d192
IB		 4521(define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4522 [(set (match_operand:GPI 0 "register_operand" "=r")
4523 (LOGICAL:GPI (SHIFT:GPI
4524 (match_operand:GPI 1 "register_operand" "r")
4525 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4526 (match_operand:GPI 3 "register_operand" "r")))]
4527 ""
4528 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4529 [(set_attr "type" "logic_shift_imm")]
4530)
43e9d192 4531
39d23b79
JJ		 4532(define_split
4533 [(set (match_operand:GPI 0 "register_operand")
4534 (LOGICAL:GPI
4535 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand")
4536 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
4537 (match_operand:GPI 3 "const_int_operand"))
4538 (zero_extend:GPI (match_operand 4 "register_operand"))))]
4539 "can_create_pseudo_p ()
4540 && ((paradoxical_subreg_p (operands[1])
4541 && rtx_equal_p (SUBREG_REG (operands[1]), operands[4]))
4542 || (REG_P (operands[1])
4543 && REG_P (operands[4])
4544 && REGNO (operands[1]) == REGNO (operands[4])))
4545 && (trunc_int_for_mode (GET_MODE_MASK (GET_MODE (operands[4]))
4546 << INTVAL (operands[2]), <MODE>mode)
4547 == INTVAL (operands[3]))"
4548 [(set (match_dup 5) (zero_extend:GPI (match_dup 4)))
4549 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 5) (match_dup 2))
4550 (match_dup 5)))]
4551 "operands[5] = gen_reg_rtx (<MODE>mode);"
4552)
4553
4554(define_split
4555 [(set (match_operand:GPI 0 "register_operand")
4556 (LOGICAL:GPI
4557 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand")
4558 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
4559 (match_operand:GPI 4 "const_int_operand"))
4560 (and:GPI (match_dup 1) (match_operand:GPI 3 "const_int_operand"))))]
4561 "can_create_pseudo_p ()
4562 && pow2_or_zerop (UINTVAL (operands[3]) + 1)
4563 && (trunc_int_for_mode (UINTVAL (operands[3])
4564 << INTVAL (operands[2]), <MODE>mode)
4565 == INTVAL (operands[4]))"
4566 [(set (match_dup 5) (and:GPI (match_dup 1) (match_dup 3)))
4567 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 5) (match_dup 2))
4568 (match_dup 5)))]
4569 "operands[5] = gen_reg_rtx (<MODE>mode);"
4570)
4571
4572(define_split
4573 [(set (match_operand:GPI 0 "register_operand")
4574 (LOGICAL:GPI
4575 (ashift:GPI (sign_extend:GPI (match_operand 1 "register_operand"))
4576 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
4577 (sign_extend:GPI (match_dup 1))))]
4578 "can_create_pseudo_p ()"
4579 [(set (match_dup 3) (sign_extend:GPI (match_dup 1)))
4580 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 3) (match_dup 2))
4581 (match_dup 3)))]
4582 "operands[3] = gen_reg_rtx (<MODE>mode);"
4583)
4584
57b77d46
RE		 4585(define_insn "*<optab>_rol<mode>3"
4586 [(set (match_operand:GPI 0 "register_operand" "=r")
4587 (LOGICAL:GPI (rotate:GPI
4588 (match_operand:GPI 1 "register_operand" "r")
4589 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4590 (match_operand:GPI 3 "register_operand" "r")))]
4591 ""
5051c121 4592 "<logical>\\t%<w>0, %<w>3, %<w>1, ror #(<sizen> - %2)"
57b77d46
RE		 4593 [(set_attr "type" "logic_shift_imm")]
4594)
4595
4596;; zero_extend versions of above
d0b6bb1b
IB		 4597(define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4598 [(set (match_operand:DI 0 "register_operand" "=r")
4599 (zero_extend:DI
4600 (LOGICAL:SI (SHIFT:SI
4601 (match_operand:SI 1 "register_operand" "r")
4602 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4603 (match_operand:SI 3 "register_operand" "r"))))]
4604 ""
4605 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4606 [(set_attr "type" "logic_shift_imm")]
4607)
d0b6bb1b 4608
57b77d46
RE		 4609(define_insn "*<optab>_rolsi3_uxtw"
4610 [(set (match_operand:DI 0 "register_operand" "=r")
4611 (zero_extend:DI
4612 (LOGICAL:SI (rotate:SI
4613 (match_operand:SI 1 "register_operand" "r")
4614 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4615 (match_operand:SI 3 "register_operand" "r"))))]
4616 ""
5051c121 4617 "<logical>\\t%w0, %w3, %w1, ror #(32 - %2)"
57b77d46
RE		 4618 [(set_attr "type" "logic_shift_imm")]
4619)
4620
43e9d192 4621(define_insn "one_cmpl<mode>2"
fe82d1f2
AL		 4622 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4623 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 4624 ""
fe82d1f2
AL		 4625 "@
4626 mvn\\t%<w>0, %<w>1
4627 mvn\\t%0.8b, %1.8b"
4628 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4629 (set_attr "arch" "*,simd")]
99e6af15 4630)
43e9d192 4631
1bd3a8af
AC		 4632(define_insn "*one_cmpl_zero_extend"
4633 [(set (match_operand:DI 0 "register_operand" "=r")
4634 (zero_extend:DI
4635 (not:SI (match_operand:SI 1 "register_operand" "r"))))]
4636 ""
4637 "mvn\\t%w0, %w1"
4638 [(set_attr "type" "logic_reg")]
4639)
4640
43e9d192
IB		 4641(define_insn "*one_cmpl_<optab><mode>2"
4642 [(set (match_operand:GPI 0 "register_operand" "=r")
4643 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4644 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4645 ""
4646 "mvn\\t%<w>0, %<w>1, <shift> %2"
99e6af15
JG		 4647 [(set_attr "type" "logic_shift_imm")]
4648)
43e9d192 4649
84be6032
AL		 4650;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4651
4652(define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4653 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4654 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4655 (match_operand:GPI 2 "register_operand" "r,w")))]
43e9d192 4656 ""
84be6032
AL		 4657 "@
4658 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4659 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4660 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4661 (set_attr "arch" "*,simd")]
84be6032
AL		 4662)
4663
8409e468
KT		 4664(define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4665 [(set (match_operand:DI 0 "register_operand" "=r")
4666 (zero_extend:DI
4667 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4668 (match_operand:SI 2 "register_operand" "r"))))]
4669 ""
4670 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4671 [(set_attr "type" "logic_reg")]
4672)
4673
4674(define_insn "*xor_one_cmplsidi3_ze"
4675 [(set (match_operand:DI 0 "register_operand" "=r")
4676 (zero_extend:DI
4677 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4678 (match_operand:SI 2 "register_operand" "r")))))]
4679 ""
4680 "eon\\t%w0, %w1, %w2"
4681 [(set_attr "type" "logic_reg")]
4682)
4683
84be6032
AL		 4684;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4685;; eon does not operate on SIMD registers so the vector variant must be split.
4686(define_insn_and_split "*xor_one_cmpl<mode>3"
4687 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4688 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4689 (match_operand:GPI 2 "register_operand" "r,w"))))]
4690 ""
eeaccc07
AL		 4691 "@
4692 eon\\t%<w>0, %<w>1, %<w>2
4693 #"
4694 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
84be6032
AL		 4695 [(set (match_operand:GPI 0 "register_operand" "=w")
4696 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4697 (match_operand:GPI 2 "register_operand" "w")))
4698 (set (match_dup 0) (not:GPI (match_dup 0)))]
4699 ""
4700 [(set_attr "type" "logic_reg,multiple")
488461d8 4701 (set_attr "arch" "*,simd")]
99e6af15 4702)
43e9d192 4703
add3c965
IB		 4704(define_insn "*and_one_cmpl<mode>3_compare0"
4705 [(set (reg:CC_NZ CC_REGNUM)
4706 (compare:CC_NZ
4707 (and:GPI (not:GPI
4708 (match_operand:GPI 1 "register_operand" "r"))
4709 (match_operand:GPI 2 "register_operand" "r"))
4710 (const_int 0)))
4711 (set (match_operand:GPI 0 "register_operand" "=r")
4712 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4713 ""
4714 "bics\\t%<w>0, %<w>2, %<w>1"
99e6af15
JG		 4715 [(set_attr "type" "logics_reg")]
4716)
add3c965
IB		 4717
4718;; zero_extend version of above
4719(define_insn "*and_one_cmplsi3_compare0_uxtw"
4720 [(set (reg:CC_NZ CC_REGNUM)
4721 (compare:CC_NZ
4722 (and:SI (not:SI
4723 (match_operand:SI 1 "register_operand" "r"))
4724 (match_operand:SI 2 "register_operand" "r"))
4725 (const_int 0)))
4726 (set (match_operand:DI 0 "register_operand" "=r")
4727 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4728 ""
4729 "bics\\t%w0, %w2, %w1"
99e6af15
JG		 4730 [(set_attr "type" "logics_reg")]
4731)
add3c965 4732
cd8fc5ee
AV		 4733(define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4734 [(set (reg:CC_NZ CC_REGNUM)
4735 (compare:CC_NZ
4736 (and:GPI (not:GPI
4737 (match_operand:GPI 0 "register_operand" "r"))
4738 (match_operand:GPI 1 "register_operand" "r"))
4739 (const_int 0)))]
4740 ""
4741 "bics\\t<w>zr, %<w>1, %<w>0"
4742 [(set_attr "type" "logics_reg")]
4743)
4744
cc05c2c3 4745(define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
43e9d192
IB		 4746 [(set (match_operand:GPI 0 "register_operand" "=r")
4747 (LOGICAL:GPI (not:GPI
4748 (SHIFT:GPI
4749 (match_operand:GPI 1 "register_operand" "r")
4750 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4751 (match_operand:GPI 3 "register_operand" "r")))]
4752 ""
4753 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
dba606bf 4754 [(set_attr "type" "logic_shift_imm")]
99e6af15 4755)
43e9d192 4756
6365da9e
KT		 4757(define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4758 [(set (match_operand:GPI 0 "register_operand" "=r")
4759 (not:GPI (xor:GPI
4760 (SHIFT:GPI
4761 (match_operand:GPI 1 "register_operand" "r")
4762 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4763 (match_operand:GPI 3 "register_operand" "r"))))]
4764 ""
4765 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4766 [(set_attr "type" "logic_shift_imm")]
4767)
4768
4769;; Zero-extend version of the above.
4770(define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4771 [(set (match_operand:DI 0 "register_operand" "=r")
4772 (zero_extend:DI
4773 (not:SI (xor:SI
4774 (SHIFT:SI
4775 (match_operand:SI 1 "register_operand" "r")
4776 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4777 (match_operand:SI 3 "register_operand" "r")))))]
4778 ""
4779 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4780 [(set_attr "type" "logic_shift_imm")]
4781)
4782
add3c965
IB		 4783(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4784 [(set (reg:CC_NZ CC_REGNUM)
4785 (compare:CC_NZ
4786 (and:GPI (not:GPI
4787 (SHIFT:GPI
4788 (match_operand:GPI 1 "register_operand" "r")
4789 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4790 (match_operand:GPI 3 "register_operand" "r"))
4791 (const_int 0)))
4792 (set (match_operand:GPI 0 "register_operand" "=r")
4793 (and:GPI (not:GPI
4794 (SHIFT:GPI
4795 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4796 ""
4797 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4798 [(set_attr "type" "logics_shift_imm")]
4799)
add3c965
IB		 4800
4801;; zero_extend version of above
4802(define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4803 [(set (reg:CC_NZ CC_REGNUM)
4804 (compare:CC_NZ
4805 (and:SI (not:SI
4806 (SHIFT:SI
4807 (match_operand:SI 1 "register_operand" "r")
4808 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4809 (match_operand:SI 3 "register_operand" "r"))
4810 (const_int 0)))
4811 (set (match_operand:DI 0 "register_operand" "=r")
4812 (zero_extend:DI (and:SI
4813 (not:SI
4814 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4815 ""
4816 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4817 [(set_attr "type" "logics_shift_imm")]
4818)
add3c965 4819
cd8fc5ee
AV		 4820(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4821 [(set (reg:CC_NZ CC_REGNUM)
4822 (compare:CC_NZ
4823 (and:GPI (not:GPI
4824 (SHIFT:GPI
4825 (match_operand:GPI 0 "register_operand" "r")
4826 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4827 (match_operand:GPI 2 "register_operand" "r"))
4828 (const_int 0)))]
4829 ""
4830 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4831 [(set_attr "type" "logics_shift_imm")]
4832)
4833
43e9d192
IB		 4834(define_insn "clz<mode>2"
4835 [(set (match_operand:GPI 0 "register_operand" "=r")
4836 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4837 ""
4838 "clz\\t%<w>0, %<w>1"
99e6af15
JG		 4839 [(set_attr "type" "clz")]
4840)
43e9d192
IB		 4841
4842(define_expand "ffs<mode>2"
4843 [(match_operand:GPI 0 "register_operand")
4844 (match_operand:GPI 1 "register_operand")]
4845 ""
4846 {
4847 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4848 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4849
4850 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4851 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
cd5660ab 4852 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
43e9d192
IB		 4853 DONE;
4854 }
4855)
4856
9d5019a1
AP		 4857;; Pop count be done via the "CNT" instruction in AdvSIMD.
4858;;
4859;; MOV v.1d, x0
4860;; CNT v1.8b, v.8b
4861;; ADDV b2, v1.8b
4862;; MOV w0, v2.b[0]
4863
4864(define_expand "popcount<mode>2"
4865 [(match_operand:GPI 0 "register_operand")
4866 (match_operand:GPI 1 "register_operand")]
4867 "TARGET_SIMD"
4868{
4869 rtx v = gen_reg_rtx (V8QImode);
4870 rtx v1 = gen_reg_rtx (V8QImode);
9d5019a1
AP		 4871 rtx in = operands[1];
4872 rtx out = operands[0];
4873 if(<MODE>mode == SImode)
4874 {
4875 rtx tmp;
4876 tmp = gen_reg_rtx (DImode);
4877 /* If we have SImode, zero extend to DImode, pop count does
4878 not change if we have extra zeros. */
4879 emit_insn (gen_zero_extendsidi2 (tmp, in));
4880 in = tmp;
4881 }
4882 emit_move_insn (v, gen_lowpart (V8QImode, in));
4883 emit_insn (gen_popcountv8qi2 (v1, v));
9921bbf9 4884 emit_insn (gen_aarch64_zero_extend<mode>_reduc_plus_v8qi (out, v1));
9d5019a1
AP		 4885 DONE;
4886})
4887
43e9d192
IB		 4888(define_insn "clrsb<mode>2"
4889 [(set (match_operand:GPI 0 "register_operand" "=r")
dfc98d99 4890 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192
IB		 4891 ""
4892 "cls\\t%<w>0, %<w>1"
99e6af15
JG		 4893 [(set_attr "type" "clz")]
4894)
43e9d192
IB		 4895
4896(define_insn "rbit<mode>2"
4897 [(set (match_operand:GPI 0 "register_operand" "=r")
4898 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4899 ""
4900 "rbit\\t%<w>0, %<w>1"
99e6af15
JG		 4901 [(set_attr "type" "rbit")]
4902)
43e9d192 4903
e8536e2b
KT		 4904;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4905;; it is unlikely to fold with any other operation, so keep this as a CTZ
4906;; expression and split after reload to enable scheduling them apart if
4907;; needed.
4908
4909(define_insn_and_split "ctz<mode>2"
4910 [(set (match_operand:GPI 0 "register_operand" "=r")
4911 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4912 ""
e8536e2b
KT		 4913 "#"
4914 "reload_completed"
4915 [(const_int 0)]
4916 "
4917 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4918 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4919 DONE;
4920")
43e9d192 4921
2b8568fe
KT		 4922(define_insn "*and<mode>_compare0"
4923 [(set (reg:CC_NZ CC_REGNUM)
4924 (compare:CC_NZ
4925 (match_operand:SHORT 0 "register_operand" "r")
4926 (const_int 0)))]
4927 ""
4928 "tst\\t%<w>0, <short_mask>"
4929 [(set_attr "type" "alus_imm")]
4930)
4931
e7ba492a 4932(define_insn "*ands<GPI:mode>_compare0"
b06335f9
KT		 4933 [(set (reg:CC_NZ CC_REGNUM)
4934 (compare:CC_NZ
4935 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4936 (const_int 0)))
4937 (set (match_operand:GPI 0 "register_operand" "=r")
4938 (zero_extend:GPI (match_dup 1)))]
4939 ""
4940 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4941 [(set_attr "type" "alus_imm")]
4942)
4943
43e9d192 4944(define_insn "*and<mode>3nr_compare0"
d803a491
IB		 4945 [(set (reg:CC_NZ CC_REGNUM)
4946 (compare:CC_NZ
43e9d192
IB		 4947 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4948 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4949 (const_int 0)))]
4950 ""
954224d7
WD		 4951 "@
4952 tst\\t%<w>0, %<w>1
4953 tst\\t%<w>0, %1"
1d70b920 4954 [(set_attr "type" "logics_reg,logics_imm")]
99e6af15 4955)
43e9d192 4956
2c2789d5
KT		 4957(define_split
4958 [(set (reg:CC_NZ CC_REGNUM)
4959 (compare:CC_NZ
4960 (and:GPI (match_operand:GPI 0 "register_operand")
4961 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4962 (const_int 0)))
4963 (clobber (match_operand:SI 2 "register_operand"))]
4964 ""
4965 [(set (match_dup 2) (match_dup 1))
4966 (set (reg:CC_NZ CC_REGNUM)
4967 (compare:CC_NZ
4968 (and:GPI (match_dup 0)
4969 (match_dup 2))
4970 (const_int 0)))]
4971)
4972
7325d85a
KT		 4973(define_insn "*and<mode>3nr_compare0_zextract"
4974 [(set (reg:CC_NZ CC_REGNUM)
4975 (compare:CC_NZ
4976 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4977 (match_operand:GPI 1 "const_int_operand" "n")
4978 (match_operand:GPI 2 "const_int_operand" "n"))
4979 (const_int 0)))]
4980 "INTVAL (operands[1]) > 0
4981 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4982 <= GET_MODE_BITSIZE (<MODE>mode))
4983 && aarch64_bitmask_imm (
4984 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4985 operands[2])),
4986 <MODE>mode)"
4987 {
4988 operands[1]
4989 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4990 return "tst\\t%<w>0, %1";
4991 }
4992 [(set_attr "type" "logics_shift_imm")]
4993)
4994
43e9d192 4995(define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
d803a491
IB		 4996 [(set (reg:CC_NZ CC_REGNUM)
4997 (compare:CC_NZ
43e9d192
IB		 4998 (and:GPI (SHIFT:GPI
4999 (match_operand:GPI 0 "register_operand" "r")
5000 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
5001 (match_operand:GPI 2 "register_operand" "r"))
5002 (const_int 0)))]
5003 ""
5004 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
99e6af15
JG		 5005 [(set_attr "type" "logics_shift_imm")]
5006)
43e9d192 5007
2c2789d5
KT		 5008(define_split
5009 [(set (reg:CC_NZ CC_REGNUM)
5010 (compare:CC_NZ
5011 (and:GPI (SHIFT:GPI
5012 (match_operand:GPI 0 "register_operand")
5013 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
5014 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
5015 (const_int 0)))
5016 (clobber (match_operand:SI 3 "register_operand"))]
5017 ""
5018 [(set (match_dup 3) (match_dup 2))
5019 (set (reg:CC_NZ CC_REGNUM)
5020 (compare:CC_NZ
5021 (and:GPI (SHIFT:GPI
5022 (match_dup 0)
5023 (match_dup 1))
5024 (match_dup 3))
5025 (const_int 0)))]
5026)
5027
43e9d192
IB		 5028;; -------------------------------------------------------------------
5029;; Shifts
5030;; -------------------------------------------------------------------
5031
5032(define_expand "<optab><mode>3"
5033 [(set (match_operand:GPI 0 "register_operand")
5034 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 5035 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 5036 ""
5037 {
5038 if (CONST_INT_P (operands[2]))
5039 {
5040 operands[2] = GEN_INT (INTVAL (operands[2])
5041 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5042
5043 if (operands[2] == const0_rtx)
5044 {
5045 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5046 DONE;
5047 }
5048 }
5049 }
5050)
5051
5052(define_expand "ashl<mode>3"
5053 [(set (match_operand:SHORT 0 "register_operand")
5054 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4bc72f35 5055 (match_operand:QI 2 "const_int_operand")))]
43e9d192
IB		 5056 ""
5057 {
4bc72f35 5058 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
43e9d192 5059
4bc72f35
KT		 5060 if (operands[2] == const0_rtx)
5061 {
5062 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5063 DONE;
43e9d192
IB		 5064 }
5065 }
5066)
5067
5068(define_expand "rotr<mode>3"
5069 [(set (match_operand:GPI 0 "register_operand")
5070 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 5071 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 5072 ""
5073 {
5074 if (CONST_INT_P (operands[2]))
5075 {
5076 operands[2] = GEN_INT (INTVAL (operands[2])
5077 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5078
5079 if (operands[2] == const0_rtx)
5080 {
5081 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5082 DONE;
5083 }
5084 }
5085 }
5086)
5087
5088(define_expand "rotl<mode>3"
5089 [(set (match_operand:GPI 0 "register_operand")
5090 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 5091 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 5092 ""
5093 {
5094 /* (SZ - cnt) % SZ == -cnt % SZ */
5095 if (CONST_INT_P (operands[2]))
5096 {
618ae596 5097 operands[2] = GEN_INT ((-UINTVAL (operands[2]))
43e9d192
IB		 5098 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5099 if (operands[2] == const0_rtx)
5100 {
5101 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5102 DONE;
5103 }
5104 }
5105 else
5106 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5107 NULL_RTX, 1);
5108 }
5109)
5110
7813b280
KT		 5111;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5112;; they truncate the shift/rotate amount by the size of the registers they
5113;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5114;; such redundant masking instructions. GCC can do that automatically when
5115;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5116;; because some of the SISD shift alternatives don't perform this truncations.
5117;; So this pattern exists to catch such cases.
5118
5119(define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5120 [(set (match_operand:GPI 0 "register_operand" "=r")
5121 (SHIFT:GPI
5122 (match_operand:GPI 1 "register_operand" "r")
5123 (match_operator 4 "subreg_lowpart_operator"
5124 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5125 (match_operand 3 "const_int_operand" "n"))])))]
5126 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5127 "<shift>\t%<w>0, %<w>1, %<w>2"
5128 [(set_attr "type" "shift_reg")]
5129)
5130
cdeba3e0 5131(define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
7813b280
KT		 5132 [(set (match_operand:GPI 0 "register_operand" "=&r")
5133 (SHIFT:GPI
5134 (match_operand:GPI 1 "register_operand" "r")
5135 (match_operator 4 "subreg_lowpart_operator"
5136 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5137 (match_operand 3 "const_int_operand" "n")))])))]
5138 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5139 "#"
5140 "&& true"
5141 [(const_int 0)]
5142 {
5143 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
cdeba3e0 5144 : lowpart_subreg (SImode, operands[0], <MODE>mode));
7813b280
KT		 5145 emit_insn (gen_negsi2 (tmp, operands[2]));
5146
5147 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5148 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5149 SUBREG_BYTE (operands[4]));
5150 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5151 DONE;
5152 }
5153)
5154
cdeba3e0 5155(define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
7813b280
KT		 5156 [(set (match_operand:GPI 0 "register_operand" "=&r")
5157 (ashift:GPI
5158 (match_operand:GPI 1 "register_operand" "r")
5159 (minus:QI (match_operand 2 "const_int_operand" "n")
5160 (match_operator 5 "subreg_lowpart_operator"
5161 [(and:SI (match_operand:SI 3 "register_operand" "r")
5162 (match_operand 4 "const_int_operand" "n"))]))))]
5163 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5164 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5165 "#"
5166 "&& true"
5167 [(const_int 0)]
5168 {
5169 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5170 : operands[0]);
5171
5172 emit_insn (gen_negsi2 (tmp, operands[3]));
5173
5174 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5175 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5176 SUBREG_BYTE (operands[5]));
5177
5178 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5179 DONE;
5180 }
5181)
5182
5183(define_insn "*aarch64_<optab>_reg_di3_mask2"
5184 [(set (match_operand:DI 0 "register_operand" "=r")
5185 (SHIFT:DI
5186 (match_operand:DI 1 "register_operand" "r")
5187 (match_operator 4 "subreg_lowpart_operator"
5188 [(and:SI (match_operand:SI 2 "register_operand" "r")
cdeba3e0
JJ		 5189 (match_operand 3 "const_int_operand" "n"))])))]
5190 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
7813b280
KT		 5191{
5192 rtx xop[3];
5193 xop[0] = operands[0];
5194 xop[1] = operands[1];
5195 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5196 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5197 return "";
5198}
5199 [(set_attr "type" "shift_reg")]
5200)
5201
cdeba3e0 5202(define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
59abe903
MC		 5203 [(set (match_operand:GPI 0 "register_operand" "=&r")
5204 (ASHIFT:GPI
5205 (match_operand:GPI 1 "register_operand" "r")
5206 (minus:QI (match_operand 2 "const_int_operand" "n")
5207 (match_operand:QI 3 "register_operand" "r"))))]
5208 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5209 "#"
5210 "&& true"
5211 [(const_int 0)]
5212 {
5213 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5214
5215 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5216 : gen_lowpart (SImode, operands[0]));
5217
5218 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5219
5220 rtx and_op = gen_rtx_AND (SImode, tmp,
5221 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5222
5223 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5224
5225 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5226 DONE;
5227 }
5228 [(set_attr "length" "8")]
5229)
5230
2e100703
VP		 5231;; Logical left shift using SISD or Integer instruction
5232(define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3362d764
WD		 5233 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5234 (ashift:GPI
5235 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5236 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
2e100703
VP		 5237 ""
5238 "@
3362d764 5239 lsl\t%<w>0, %<w>1, %2
d99dacc9 5240 lsl\t%<w>0, %<w>1, %<w>2
2e100703 5241 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
d99dacc9 5242 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
488461d8
MM		 5243 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5244 (set_attr "arch" "*,*,simd,simd")]
2e100703
VP		 5245)
5246
5247;; Logical right shift using SISD or Integer instruction
5248(define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3362d764
WD		 5249 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5250 (lshiftrt:GPI
5251 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
0603375c
KT		 5252 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5253 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5254 ""
5255 "@
3362d764 5256 lsr\t%<w>0, %<w>1, %2
ad7b853a 5257 lsr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5258 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5259 #
ad7b853a 5260 #"
488461d8
MM		 5261 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5262 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5263)
5264
5265(define_split
5266 [(set (match_operand:DI 0 "aarch64_simd_register")
5267 (lshiftrt:DI
5268 (match_operand:DI 1 "aarch64_simd_register")
5269 (match_operand:QI 2 "aarch64_simd_register")))]
5270 "TARGET_SIMD && reload_completed"
498b13e2 5271 [(set (match_dup 3)
2e100703
VP		 5272 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5273 (set (match_dup 0)
498b13e2
KT		 5274 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5275 {
5276 operands[3] = gen_lowpart (QImode, operands[0]);
5277 }
2e100703
VP		 5278)
5279
5280(define_split
5281 [(set (match_operand:SI 0 "aarch64_simd_register")
5282 (lshiftrt:SI
5283 (match_operand:SI 1 "aarch64_simd_register")
5284 (match_operand:QI 2 "aarch64_simd_register")))]
5285 "TARGET_SIMD && reload_completed"
498b13e2 5286 [(set (match_dup 3)
2e100703
VP		 5287 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5288 (set (match_dup 0)
498b13e2
KT		 5289 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5290 {
5291 operands[3] = gen_lowpart (QImode, operands[0]);
5292 }
2e100703
VP		 5293)
5294
5295;; Arithmetic right shift using SISD or Integer instruction
5296(define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3362d764 5297 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
0603375c
KT		 5298 (ashiftrt:GPI
5299 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5300 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5301 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5302 ""
5303 "@
3362d764 5304 asr\t%<w>0, %<w>1, %2
ad7b853a 5305 asr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5306 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5307 #
ad7b853a 5308 #"
488461d8
MM		 5309 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5310 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5311)
5312
5313(define_split
5314 [(set (match_operand:DI 0 "aarch64_simd_register")
5315 (ashiftrt:DI
5316 (match_operand:DI 1 "aarch64_simd_register")
5317 (match_operand:QI 2 "aarch64_simd_register")))]
5318 "TARGET_SIMD && reload_completed"
21a7b5e1 5319 [(set (match_dup 3)
2e100703
VP		 5320 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5321 (set (match_dup 0)
21a7b5e1
MS		 5322 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5323{
5324 operands[3] = gen_lowpart (QImode, operands[0]);
5325}
2e100703
VP		 5326)
5327
5328(define_split
5329 [(set (match_operand:SI 0 "aarch64_simd_register")
5330 (ashiftrt:SI
5331 (match_operand:SI 1 "aarch64_simd_register")
5332 (match_operand:QI 2 "aarch64_simd_register")))]
5333 "TARGET_SIMD && reload_completed"
21a7b5e1 5334 [(set (match_dup 3)
2e100703
VP		 5335 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5336 (set (match_dup 0)
21a7b5e1
MS		 5337 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5338{
5339 operands[3] = gen_lowpart (QImode, operands[0]);
5340}
2e100703
VP		 5341)
5342
5343(define_insn "*aarch64_sisd_ushl"
5344 [(set (match_operand:DI 0 "register_operand" "=w")
5345 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5346 (match_operand:QI 2 "register_operand" "w")]
5347 UNSPEC_SISD_USHL))]
5348 "TARGET_SIMD"
5349 "ushl\t%d0, %d1, %d2"
488461d8 5350 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5351)
5352
5353(define_insn "*aarch64_ushl_2s"
5354 [(set (match_operand:SI 0 "register_operand" "=w")
5355 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5356 (match_operand:QI 2 "register_operand" "w")]
5357 UNSPEC_USHL_2S))]
5358 "TARGET_SIMD"
5359 "ushl\t%0.2s, %1.2s, %2.2s"
488461d8 5360 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5361)
5362
5363(define_insn "*aarch64_sisd_sshl"
5364 [(set (match_operand:DI 0 "register_operand" "=w")
5365 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5366 (match_operand:QI 2 "register_operand" "w")]
5367 UNSPEC_SISD_SSHL))]
5368 "TARGET_SIMD"
5369 "sshl\t%d0, %d1, %d2"
488461d8 5370 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5371)
5372
5373(define_insn "*aarch64_sshl_2s"
5374 [(set (match_operand:SI 0 "register_operand" "=w")
5375 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5376 (match_operand:QI 2 "register_operand" "w")]
5377 UNSPEC_SSHL_2S))]
5378 "TARGET_SIMD"
5379 "sshl\t%0.2s, %1.2s, %2.2s"
488461d8 5380 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5381)
5382
5383(define_insn "*aarch64_sisd_neg_qi"
5384 [(set (match_operand:QI 0 "register_operand" "=w")
5385 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5386 UNSPEC_SISD_NEG))]
5387 "TARGET_SIMD"
5388 "neg\t%d0, %d1"
488461d8 5389 [(set_attr "type" "neon_neg")]
2e100703
VP		 5390)
5391
5392;; Rotate right
5393(define_insn "*ror<mode>3_insn"
b81f1ee3
JG		 5394 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5395 (rotatert:GPI
5396 (match_operand:GPI 1 "register_operand" "r,r")
3362d764 5397 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
43e9d192 5398 ""
3362d764
WD		 5399 "@
5400 ror\\t%<w>0, %<w>1, %2
5401 ror\\t%<w>0, %<w>1, %<w>2"
5402 [(set_attr "type" "rotate_imm,shift_reg")]
43e9d192
IB		 5403)
5404
d0b6bb1b
IB		 5405;; zero_extend version of above
5406(define_insn "*<optab>si3_insn_uxtw"
3362d764 5407 [(set (match_operand:DI 0 "register_operand" "=r,r")
d0b6bb1b 5408 (zero_extend:DI (SHIFT:SI
3362d764
WD		 5409 (match_operand:SI 1 "register_operand" "r,r")
5410 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
d0b6bb1b 5411 ""
3362d764
WD		 5412 "@
5413 <shift>\\t%w0, %w1, %2
5414 <shift>\\t%w0, %w1, %w2"
94f7a25e 5415 [(set_attr "type" "bfx,shift_reg")]
d0b6bb1b
IB		 5416)
5417
43e9d192
IB		 5418(define_insn "*<optab><mode>3_insn"
5419 [(set (match_operand:SHORT 0 "register_operand" "=r")
5420 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5421 (match_operand 2 "const_int_operand" "n")))]
5422 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5423{
5424 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5425 return "<bfshift>\t%w0, %w1, %2, %3";
5426}
94f7a25e 5427 [(set_attr "type" "bfx")]
43e9d192
IB		 5428)
5429
bd83ff2c
IB		 5430(define_insn "*extr<mode>5_insn"
5431 [(set (match_operand:GPI 0 "register_operand" "=r")
5432 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5433 (match_operand 3 "const_int_operand" "n"))
5434 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5435 (match_operand 4 "const_int_operand" "n"))))]
5436 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5437 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5438 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5439 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5440)
5441
fb0cb7fa
KT		 5442;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5443;; so we have to match both orderings.
5444(define_insn "*extr<mode>5_insn_alt"
5445 [(set (match_operand:GPI 0 "register_operand" "=r")
5446 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5447 (match_operand 4 "const_int_operand" "n"))
5448 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5449 (match_operand 3 "const_int_operand" "n"))))]
5450 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5451 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5452 == GET_MODE_BITSIZE (<MODE>mode))"
5453 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5454 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5455)
5456
bd83ff2c
IB		 5457;; zero_extend version of the above
5458(define_insn "*extrsi5_insn_uxtw"
5459 [(set (match_operand:DI 0 "register_operand" "=r")
5460 (zero_extend:DI
5461 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5462 (match_operand 3 "const_int_operand" "n"))
5463 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5464 (match_operand 4 "const_int_operand" "n")))))]
5465 "UINTVAL (operands[3]) < 32 &&
5466 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5467 "extr\\t%w0, %w1, %w2, %4"
3362d764 5468 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5469)
5470
fb0cb7fa
KT		 5471(define_insn "*extrsi5_insn_uxtw_alt"
5472 [(set (match_operand:DI 0 "register_operand" "=r")
5473 (zero_extend:DI
5474 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5475 (match_operand 4 "const_int_operand" "n"))
5476 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5477 (match_operand 3 "const_int_operand" "n")))))]
5478 "UINTVAL (operands[3]) < 32 &&
5479 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5480 "extr\\t%w0, %w1, %w2, %4"
3362d764 5481 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5482)
5483
47f42744
JJ		 5484(define_insn "*extrsi5_insn_di"
5485 [(set (match_operand:SI 0 "register_operand" "=r")
5486 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5487 (match_operand 3 "const_int_operand" "n"))
5488 (match_operator:SI 6 "subreg_lowpart_operator"
5489 [(zero_extract:DI
5490 (match_operand:DI 2 "register_operand" "r")
5491 (match_operand 5 "const_int_operand" "n")
5492 (match_operand 4 "const_int_operand" "n"))])))]
5493 "UINTVAL (operands[3]) < 32
5494 && UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32
5495 && INTVAL (operands[3]) == INTVAL (operands[5])"
5496 "extr\\t%w0, %w1, %w2, %4"
5497 [(set_attr "type" "rotate_imm")]
5498)
5499
5977a10d
IB		 5500(define_insn "*ror<mode>3_insn"
5501 [(set (match_operand:GPI 0 "register_operand" "=r")
5502 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5503 (match_operand 2 "const_int_operand" "n")))]
5504 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5505{
5506 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5507 return "ror\\t%<w>0, %<w>1, %3";
5508}
b81f1ee3 5509 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5510)
5511
5512;; zero_extend version of the above
5513(define_insn "*rorsi3_insn_uxtw"
5514 [(set (match_operand:DI 0 "register_operand" "=r")
5515 (zero_extend:DI
5516 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5517 (match_operand 2 "const_int_operand" "n"))))]
5518 "UINTVAL (operands[2]) < 32"
5519{
5520 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5521 return "ror\\t%w0, %w1, %3";
5522}
b81f1ee3 5523 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5524)
5525
43e9d192
IB		 5526(define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5527 [(set (match_operand:GPI 0 "register_operand" "=r")
5528 (ANY_EXTEND:GPI
5529 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5530 (match_operand 2 "const_int_operand" "n"))))]
5531 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5532{
5533 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5534 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5535}
94f7a25e 5536 [(set_attr "type" "bfx")]
43e9d192
IB		 5537)
5538
5539(define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5540 [(set (match_operand:GPI 0 "register_operand" "=r")
5541 (zero_extend:GPI
5542 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5543 (match_operand 2 "const_int_operand" "n"))))]
5544 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5545{
5546 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5547 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5548}
94f7a25e 5549 [(set_attr "type" "bfx")]
43e9d192
IB		 5550)
5551
5552(define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5553 [(set (match_operand:GPI 0 "register_operand" "=r")
5554 (sign_extend:GPI
5555 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5556 (match_operand 2 "const_int_operand" "n"))))]
5557 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5558{
5559 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5560 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5561}
94f7a25e 5562 [(set_attr "type" "bfx")]
43e9d192
IB		 5563)
5564
5565;; -------------------------------------------------------------------
5566;; Bitfields
5567;; -------------------------------------------------------------------
5568
5569(define_expand "<optab>"
1bbffb87 5570 [(set (match_operand:DI 0 "register_operand")
e27bfda2
KT		 5571 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5572 (match_operand 2
5573 "aarch64_simd_shift_imm_offset_di")
5574 (match_operand 3 "aarch64_simd_shift_imm_di")))]
43e9d192 5575 ""
e27bfda2
KT		 5576 {
5577 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5578 1, GET_MODE_BITSIZE (DImode) - 1))
5579 FAIL;
5580 }
43e9d192
IB		 5581)
5582
e27bfda2 5583
43e9d192
IB		 5584(define_insn "*<optab><mode>"
5585 [(set (match_operand:GPI 0 "register_operand" "=r")
5586 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
e27bfda2
KT		 5587 (match_operand 2
5588 "aarch64_simd_shift_imm_offset_<mode>" "n")
5589 (match_operand 3
5590 "aarch64_simd_shift_imm_<mode>" "n")))]
5591 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5592 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
43e9d192 5593 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
94f7a25e 5594 [(set_attr "type" "bfx")]
65f9e789
IB		 5595)
5596
bcb036c5
KT		 5597;; When the bit position and width add up to 32 we can use a W-reg LSR
5598;; instruction taking advantage of the implicit zero-extension of the X-reg.
5599(define_split
5600 [(set (match_operand:DI 0 "register_operand")
5601 (zero_extract:DI (match_operand:DI 1 "register_operand")
5602 (match_operand 2
5603 "aarch64_simd_shift_imm_offset_di")
5604 (match_operand 3
5605 "aarch64_simd_shift_imm_di")))]
5606 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5607 GET_MODE_BITSIZE (DImode) - 1)
5608 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5609 == GET_MODE_BITSIZE (SImode)"
5610 [(set (match_dup 0)
5611 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5612 {
5613 operands[4] = gen_lowpart (SImode, operands[1]);
5614 }
5615)
5616
65f9e789
IB		 5617;; Bitfield Insert (insv)
5618(define_expand "insv<mode>"
5619 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5620 (match_operand 1 "const_int_operand")
5621 (match_operand 2 "const_int_operand"))
5622 (match_operand:GPI 3 "general_operand"))]
5623 ""
5624{
5625 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5626 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5627 rtx value = operands[3];
5628
5629 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5630 FAIL;
5631
5632 if (CONST_INT_P (value))
5633 {
5634 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5635
5636 /* Prefer AND/OR for inserting all zeros or all ones. */
5637 if ((UINTVAL (value) & mask) == 0
5638 || (UINTVAL (value) & mask) == mask)
5639 FAIL;
5640
5641 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5642 if (width == 16 && (pos % 16) == 0)
5643 DONE;
5644 }
5645 operands[3] = force_reg (<MODE>mode, value);
5646})
5647
5648(define_insn "*insv_reg<mode>"
5649 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5650 (match_operand 1 "const_int_operand" "n")
5651 (match_operand 2 "const_int_operand" "n"))
5652 (match_operand:GPI 3 "register_operand" "r"))]
5653 "!(UINTVAL (operands[1]) == 0
5654 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5655 > GET_MODE_BITSIZE (<MODE>mode)))"
5656 "bfi\\t%<w>0, %<w>3, %2, %1"
99e6af15 5657 [(set_attr "type" "bfm")]
2879bb2b
IB		 5658)
5659
e4f019d3
RL		 5660(define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5661 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5662 (match_operand 1 "const_int_operand" "n")
5663 (match_operand 2 "const_int_operand" "n"))
5664 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5665 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5666 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5667 [(set_attr "type" "bfm")]
5668)
5669
6a0d3939
SE		 5670;; Match a bfi instruction where the shift of OP3 means that we are
5671;; actually copying the least significant bits of OP3 into OP0 by way
5672;; of the AND masks and the IOR instruction. A similar instruction
5673;; with the two parts of the IOR swapped around was never triggered
5674;; in a bootstrap build and test of GCC so it was not included.
5675
5676(define_insn "*aarch64_bfi<GPI:mode>5_shift"
5677 [(set (match_operand:GPI 0 "register_operand" "=r")
5678 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5679 (match_operand:GPI 2 "const_int_operand" "n"))
5680 (and:GPI (ashift:GPI
5681 (match_operand:GPI 3 "register_operand" "r")
5682 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5683 (match_operand:GPI 5 "const_int_operand" "n"))))]
5684 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5685 UINTVAL (operands[4]),
5686 UINTVAL(operands[5]))"
5687 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5688 [(set_attr "type" "bfm")]
5689)
5690
5691(define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5692 [(set (match_operand:GPI 0 "register_operand" "=r")
5693 (ior:GPI (and:GPI (ashift:GPI
5694 (match_operand:GPI 1 "register_operand" "r")
5695 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5696 (match_operand:GPI 3 "const_int_operand" "n"))
5697 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5698 (match_operand:GPI 5 "const_int_operand" "n"))))]
5699 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5700 UINTVAL (operands[2]),
5701 UINTVAL(operands[3]))"
5702 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5703 [(set_attr "type" "bfm")]
5704)
5705
5706;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5707;; the shift is large enough to remove the need for an AND instruction.
5708
5709(define_insn "*aarch64_bfi<GPI:mode>4_noand"
5710 [(set (match_operand:GPI 0 "register_operand" "=r")
5711 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5712 (match_operand:GPI 2 "const_int_operand" "n"))
5713 (ashift:GPI
5714 (match_operand:GPI 3 "register_operand" "r")
5715 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5716 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5717 UINTVAL (operands[4]),
5718 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5719{
5720 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5721 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5722}
5723 [(set_attr "type" "bfm")]
5724)
5725
5726(define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5727 [(set (match_operand:GPI 0 "register_operand" "=r")
5728 (ior:GPI (ashift:GPI
5729 (match_operand:GPI 1 "register_operand" "r")
5730 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5731 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5732 (match_operand:GPI 4 "const_int_operand" "n"))))]
5733 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5734 UINTVAL (operands[2]),
5735 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5736{
5737 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5738 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5739}
5740 [(set_attr "type" "bfm")]
5741)
5742
5743;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
f0a8f6bc
SE		 5744;; copying the least significant bits of OP3 to OP0. We need two versions
5745;; of the instruction to handle different checks on the constant values.
6a0d3939
SE		 5746
5747(define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5748 [(set (match_operand:GPI 0 "register_operand" "=r")
5749 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5750 (match_operand:GPI 2 "const_int_operand" "n"))
5751 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5752 (match_operand:GPI 4 "const_int_operand" "n"))))]
5753 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5754 UINTVAL (operands[4]))"
5755 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5756 [(set_attr "type" "bfm")]
5757)
5758
f0a8f6bc
SE		 5759(define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5760 [(set (match_operand:GPI 0 "register_operand" "=r")
5761 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5762 (match_operand:GPI 4 "const_int_operand" "n"))
5763 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5764 (match_operand:GPI 2 "const_int_operand" "n"))))]
5765 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5766 UINTVAL (operands[4]))"
5767 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5768 [(set_attr "type" "bfm")]
5769)
5770
f6ba5d03
JJ		 5771(define_insn "*aarch64_bfxil<mode>_extr"
5772 [(set (match_operand:GPI 0 "register_operand" "=r")
5773 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5774 (match_operand:GPI 2 "const_int_operand" "n"))
5775 (zero_extract:GPI
5776 (match_operand:GPI 3 "register_operand" "r")
5777 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n")
5778 (match_operand:GPI 5 "aarch64_simd_shift_imm_<mode>" "n"))))]
5779 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4])
5780 && INTVAL (operands[4])
5781 && (UINTVAL (operands[4]) + UINTVAL (operands[5])
5782 <= GET_MODE_BITSIZE (<MODE>mode))"
5783 "bfxil\t%<GPI:w>0, %<GPI:w>3, %5, %4"
5784 [(set_attr "type" "bfm")]
5785)
5786
5787(define_insn "*aarch64_bfxilsi_extrdi"
5788 [(set (match_operand:SI 0 "register_operand" "=r")
5789 (ior:SI (and:SI (match_operand:SI 1 "register_operand" "0")
5790 (match_operand:SI 2 "const_int_operand" "n"))
5791 (match_operator:SI 6 "subreg_lowpart_operator"
5792 [(zero_extract:DI
5793 (match_operand:DI 3 "register_operand" "r")
5794 (match_operand:SI 4 "aarch64_simd_shift_imm_si" "n")
5795 (match_operand:SI 5 "aarch64_simd_shift_imm_si" "n"))])))]
5796 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4])
5797 && INTVAL (operands[4])
5798 && UINTVAL (operands[4]) + UINTVAL (operands[5]) <= 32"
5799 "bfxil\t%w0, %w3, %5, %4"
5800 [(set_attr "type" "bfm")]
5801)
5802
2879bb2b
IB		 5803(define_insn "*extr_insv_lower_reg<mode>"
5804 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5805 (match_operand 1 "const_int_operand" "n")
5806 (const_int 0))
98fa5ef3 5807 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
2879bb2b
IB		 5808 (match_dup 1)
5809 (match_operand 3 "const_int_operand" "n")))]
5810 "!(UINTVAL (operands[1]) == 0
5811 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5812 > GET_MODE_BITSIZE (<MODE>mode)))"
5813 "bfxil\\t%<w>0, %<w>2, %3, %1"
99e6af15 5814 [(set_attr "type" "bfm")]
43e9d192
IB		 5815)
5816
5817(define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5818 [(set (match_operand:GPI 0 "register_operand" "=r")
5819 (ashift:GPI (ANY_EXTEND:GPI
5820 (match_operand:ALLX 1 "register_operand" "r"))
5821 (match_operand 2 "const_int_operand" "n")))]
5822 "UINTVAL (operands[2]) < <GPI:sizen>"
5823{
5824 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5825 ? GEN_INT (<ALLX:sizen>)
5826 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5827 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5828}
94f7a25e 5829 [(set_attr "type" "bfx")]
43e9d192
IB		 5830)
5831
5832;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5833
5834(define_insn "*andim_ashift<mode>_bfiz"
5835 [(set (match_operand:GPI 0 "register_operand" "=r")
5836 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5837 (match_operand 2 "const_int_operand" "n"))
5838 (match_operand 3 "const_int_operand" "n")))]
8c83f71d 5839 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
43e9d192 5840 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
94f7a25e 5841 [(set_attr "type" "bfx")]
43e9d192
IB		 5842)
5843
58c2ad42
LM		 5844;; Match sbfiz pattern in a shift left + shift right operation.
5845
5846(define_insn "*ashift<mode>_extv_bfiz"
5847 [(set (match_operand:GPI 0 "register_operand" "=r")
5848 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5849 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5850 (const_int 0))
5851 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5852 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5853 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5854 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5855 [(set_attr "type" "bfx")]
5856)
5857
b65a1eb3
RS		 5858(define_insn "*ashiftsi_extvdi_bfiz"
5859 [(set (match_operand:SI 0 "register_operand" "=r")
5860 (ashift:SI
5861 (match_operator:SI 4 "subreg_lowpart_operator"
5862 [(sign_extract:DI
5863 (match_operand:DI 1 "register_operand" "r")
5864 (match_operand 2 "aarch64_simd_shift_imm_offset_si")
5865 (const_int 0))])
5866 (match_operand 3 "aarch64_simd_shift_imm_si")))]
5867 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5868 1, GET_MODE_BITSIZE (SImode) - 1)"
5869 "sbfiz\\t%w0, %w1, %3, %2"
5870 [(set_attr "type" "bfx")]
5871)
5872
680153bd
KT		 5873;; When the bit position and width of the equivalent extraction add up to 32
5874;; we can use a W-reg LSL instruction taking advantage of the implicit
5875;; zero-extension of the X-reg.
5876(define_split
5877 [(set (match_operand:DI 0 "register_operand")
5878 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5879 (match_operand 2 "const_int_operand"))
5880 (match_operand 3 "const_int_operand")))]
5881 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5882 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5883 == GET_MODE_BITSIZE (SImode)"
5884 [(set (match_dup 0)
5885 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5886 {
5887 operands[4] = gen_lowpart (SImode, operands[1]);
5888 }
5889)
5890
43e9d192
IB		 5891(define_insn "bswap<mode>2"
5892 [(set (match_operand:GPI 0 "register_operand" "=r")
5893 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5894 ""
5895 "rev\\t%<w>0, %<w>1"
99e6af15 5896 [(set_attr "type" "rev")]
43e9d192
IB		 5897)
5898
c16c63bb
IB		 5899(define_insn "bswaphi2"
5900 [(set (match_operand:HI 0 "register_operand" "=r")
5901 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5902 ""
5903 "rev16\\t%w0, %w1"
99e6af15 5904 [(set_attr "type" "rev")]
c16c63bb
IB		 5905)
5906
231c52ae
ST		 5907(define_insn "*aarch64_bfxil<mode>"
5908 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5909 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5910 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5911 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5912 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5913 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5914 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5915 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5916 {
5917 switch (which_alternative)
5918 {
5919 case 0:
5920 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5921 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5922 case 1:
5923 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5924 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5925 default:
5926 gcc_unreachable ();
5927 }
5928 }
5929 [(set_attr "type" "bfm")]
5930)
5931
2dc09f66
ST		 5932; Zero-extended version of above (aarch64_bfxil)
5933(define_insn "*aarch64_bfxilsi_uxtw"
5934 [(set (match_operand:DI 0 "register_operand" "=r,r")
5935 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5936 "r,0")
5937 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5938 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5939 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5940 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5941 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5942 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5943 {
5944 switch (which_alternative)
5945 {
5946 case 0:
5947 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
55163419 5948 return "bfxil\\t%w0, %w1, 0, %3";
2dc09f66
ST		 5949 case 1:
5950 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
55163419 5951 return "bfxil\\t%w0, %w2, 0, %3";
2dc09f66
ST		 5952 default:
5953 gcc_unreachable ();
5954 }
5955 }
5956 [(set_attr "type" "bfm")]
5957)
5958
f7d5cf8d
KT		 5959;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5960;; operations within an IOR/AND RTX, therefore we have two patterns matching
5961;; each valid permutation.
5962
5963(define_insn "rev16<mode>2"
5964 [(set (match_operand:GPI 0 "register_operand" "=r")
5965 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5966 (const_int 8))
5967 (match_operand:GPI 3 "const_int_operand" "n"))
5968 (and:GPI (lshiftrt:GPI (match_dup 1)
5969 (const_int 8))
5970 (match_operand:GPI 2 "const_int_operand" "n"))))]
5971 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5972 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5973 "rev16\\t%<w>0, %<w>1"
5974 [(set_attr "type" "rev")]
5975)
5976
5977(define_insn "rev16<mode>2_alt"
5978 [(set (match_operand:GPI 0 "register_operand" "=r")
5979 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5980 (const_int 8))
5981 (match_operand:GPI 2 "const_int_operand" "n"))
5982 (and:GPI (ashift:GPI (match_dup 1)
5983 (const_int 8))
5984 (match_operand:GPI 3 "const_int_operand" "n"))))]
5985 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5986 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5987 "rev16\\t%<w>0, %<w>1"
5988 [(set_attr "type" "rev")]
5989)
5990
d0b6bb1b
IB		 5991;; zero_extend version of above
5992(define_insn "*bswapsi2_uxtw"
5993 [(set (match_operand:DI 0 "register_operand" "=r")
5994 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5995 ""
5996 "rev\\t%w0, %w1"
99e6af15 5997 [(set_attr "type" "rev")]
d0b6bb1b
IB		 5998)
5999
43e9d192
IB		 6000;; -------------------------------------------------------------------
6001;; Floating-point intrinsics
6002;; -------------------------------------------------------------------
6003
42fc9a7f 6004;; frint floating-point round to integral standard patterns.
16ce822e 6005;; Expands to btrunc, ceil, floor, nearbyint, rint, round, roundeven.
43e9d192 6006
42fc9a7f 6007(define_insn "<frint_pattern><mode>2"
d7f33f07
JW		 6008 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6009 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
42fc9a7f 6010 FRINT))]
43e9d192 6011 "TARGET_FLOAT"
42fc9a7f 6012 "frint<frint_suffix>\\t%<s>0, %<s>1"
d7f33f07 6013 [(set_attr "type" "f_rint<stype>")]
43e9d192
IB		 6014)
6015
42fc9a7f
JG		 6016;; frcvt floating-point round to integer and convert standard patterns.
6017;; Expands to lbtrunc, lceil, lfloor, lround.
d7f33f07 6018(define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
43e9d192 6019 [(set (match_operand:GPI 0 "register_operand" "=r")
d7f33f07
JW		 6020 (FIXUORS:GPI
6021 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
6022 FCVT)))]
43e9d192 6023 "TARGET_FLOAT"
d7f33f07 6024 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
99e6af15 6025 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 6026)
6027
39252973
KT		 6028(define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
6029 [(set (match_operand:GPI 0 "register_operand" "=r")
6030 (FIXUORS:GPI
6031 (mult:GPF
6032 (match_operand:GPF 1 "register_operand" "w")
6033 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
6034 "TARGET_FLOAT
6035 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
6036 GET_MODE_BITSIZE (<GPI:MODE>mode))"
6037 {
6038 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
6039 char buf[64];
6040 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
6041 output_asm_insn (buf, operands);
6042 return "";
6043 }
6044 [(set_attr "type" "f_cvtf2i")]
6045)
6046
d6e6e8b6
WD		 6047;; fma - expand fma into patterns with the accumulator operand first since
6048;; reusing the accumulator results in better register allocation.
6049;; The register allocator considers copy preferences in operand order,
6050;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
6051
6052(define_expand "fma<mode>4"
6053 [(set (match_operand:GPF_F16 0 "register_operand")
6054 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
6055 (match_operand:GPF_F16 2 "register_operand")
6056 (match_operand:GPF_F16 3 "register_operand")))]
6057 "TARGET_FLOAT"
6058)
43e9d192 6059
d6e6e8b6 6060(define_insn "*aarch64_fma<mode>4"
9a594ad6 6061 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
d6e6e8b6
WD		 6062 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
6063 (match_operand:GPF_F16 3 "register_operand" "w")
6064 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 6065 "TARGET_FLOAT"
d6e6e8b6 6066 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 6067 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 6068)
6069
d6e6e8b6
WD		 6070(define_expand "fnma<mode>4"
6071 [(set (match_operand:GPF_F16 0 "register_operand")
6072 (fma:GPF_F16
6073 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
6074 (match_operand:GPF_F16 2 "register_operand")
6075 (match_operand:GPF_F16 3 "register_operand")))]
6076 "TARGET_FLOAT"
6077)
6078
6079(define_insn "*aarch64_fnma<mode>4"
9a594ad6
JW		 6080 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6081 (fma:GPF_F16
d6e6e8b6
WD		 6082 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
6083 (match_operand:GPF_F16 3 "register_operand" "w")
6084 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 6085 "TARGET_FLOAT"
d6e6e8b6 6086 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 6087 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 6088)
6089
d6e6e8b6
WD		 6090
6091(define_expand "fms<mode>4"
6092 [(set (match_operand:GPF 0 "register_operand")
6093 (fma:GPF (match_operand:GPF 1 "register_operand")
6094 (match_operand:GPF 2 "register_operand")
6095 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6096 "TARGET_FLOAT"
6097)
6098
6099(define_insn "*aarch64_fms<mode>4"
43e9d192 6100 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 6101 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6102 (match_operand:GPF 3 "register_operand" "w")
6103 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 6104 "TARGET_FLOAT"
d6e6e8b6 6105 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 6106 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 6107)
6108
d6e6e8b6
WD		 6109(define_expand "fnms<mode>4"
6110 [(set (match_operand:GPF 0 "register_operand")
6111 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
6112 (match_operand:GPF 2 "register_operand")
6113 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6114 "TARGET_FLOAT"
6115)
6116
6117(define_insn "*aarch64_fnms<mode>4"
43e9d192 6118 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 6119 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
6120 (match_operand:GPF 3 "register_operand" "w")
6121 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 6122 "TARGET_FLOAT"
d6e6e8b6 6123 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 6124 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 6125)
6126
6127;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
d6e6e8b6 6128(define_insn "*aarch64_fnmadd<mode>4"
43e9d192 6129 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 6130 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6131 (match_operand:GPF 3 "register_operand" "w")
6132 (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 6133 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
d6e6e8b6 6134 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 6135 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 6136)
6137
6138;; -------------------------------------------------------------------
6139;; Floating-point conversions
6140;; -------------------------------------------------------------------
6141
6142(define_insn "extendsfdf2"
6143 [(set (match_operand:DF 0 "register_operand" "=w")
6144 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6145 "TARGET_FLOAT"
6146 "fcvt\\t%d0, %s1"
99e6af15 6147 [(set_attr "type" "f_cvt")]
43e9d192
IB		 6148)
6149
c2ec330c
AL		 6150(define_insn "extendhfsf2"
6151 [(set (match_operand:SF 0 "register_operand" "=w")
6152 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6153 "TARGET_FLOAT"
6154 "fcvt\\t%s0, %h1"
6155 [(set_attr "type" "f_cvt")]
6156)
6157
6158(define_insn "extendhfdf2"
6159 [(set (match_operand:DF 0 "register_operand" "=w")
6160 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6161 "TARGET_FLOAT"
6162 "fcvt\\t%d0, %h1"
6163 [(set_attr "type" "f_cvt")]
6164)
6165
43e9d192
IB		 6166(define_insn "truncdfsf2"
6167 [(set (match_operand:SF 0 "register_operand" "=w")
6168 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6169 "TARGET_FLOAT"
6170 "fcvt\\t%s0, %d1"
99e6af15 6171 [(set_attr "type" "f_cvt")]
43e9d192
IB		 6172)
6173
c2ec330c
AL		 6174(define_insn "truncsfhf2"
6175 [(set (match_operand:HF 0 "register_operand" "=w")
6176 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6177 "TARGET_FLOAT"
6178 "fcvt\\t%h0, %s1"
6179 [(set_attr "type" "f_cvt")]
6180)
6181
6182(define_insn "truncdfhf2"
6183 [(set (match_operand:HF 0 "register_operand" "=w")
6184 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6185 "TARGET_FLOAT"
6186 "fcvt\\t%h0, %d1"
6187 [(set_attr "type" "f_cvt")]
6188)
6189
22be0d08
MC		 6190;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6191;; and making r = w more expensive
6192
6193(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
43d0a8ee 6194 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
22be0d08
MC		 6195 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6196 "TARGET_FLOAT"
6197 "@
43d0a8ee
WD		 6198 fcvtz<su>\t%<s>0, %<s>1
6199 fcvtz<su>\t%<w>0, %<s>1"
cc2fda13
MS		 6200 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")
6201 (set_attr "arch" "simd,fp")]
22be0d08
MC		 6202)
6203
6204;; Convert HF -> SI or DI
6205
6206(define_insn "<optab>_trunchf<GPI:mode>2"
6207 [(set (match_operand:GPI 0 "register_operand" "=r")
6208 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6209 "TARGET_FP_F16INST"
6210 "fcvtz<su>\t%<w>0, %h1"
6211 [(set_attr "type" "f_cvtf2i")]
6212)
6213
6214;; Convert DF -> SI or SF -> DI which can only be accomplished with
6215;; input in a fp register and output in a integer register
6216
6217(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
43e9d192 6218 [(set (match_operand:GPI 0 "register_operand" "=r")
22be0d08 6219 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
43e9d192 6220 "TARGET_FLOAT"
22be0d08 6221 "fcvtz<su>\t%<w>0, %<fpw>1"
99e6af15 6222 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 6223)
6224
eeb59c16
MC		 6225(define_insn "*fix_to_zero_extend<mode>di2"
6226 [(set (match_operand:DI 0 "register_operand" "=r")
6227 (zero_extend:DI
6228 (unsigned_fix:SI
6229 (match_operand:GPF 1 "register_operand" "w"))))]
6230 "TARGET_FLOAT"
6231 "fcvtzu\t%w0, %<s>1"
6232 [(set_attr "type" "f_cvtf2i")]
6233)
6234
188d0079
JH		 6235;; Equal width integer to fp and multiply combine.
6236(define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6237 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6238 (mult:GPF (FLOATUORS:GPF
6239 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6240 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6241 "TARGET_FLOAT"
6242 {
6243 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6244 switch (which_alternative)
6245 {
6246 case 0:
6247 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6248 case 1:
6249 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6250 default:
6251 gcc_unreachable ();
6252 }
6253 }
6254 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6255 (set_attr "arch" "simd,fp")]
6256)
6257
6258;; Unequal width integer to fp and multiply combine.
6259(define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6260 [(set (match_operand:GPF 0 "register_operand" "=w")
6261 (mult:GPF (FLOATUORS:GPF
6262 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6263 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6264 "TARGET_FLOAT"
6265 {
6266 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6267 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6268 }
6269 [(set_attr "type" "f_cvti2f")]
6270)
6271
6272;; Equal width integer to fp conversion.
0d35c5c2
VP		 6273(define_insn "<optab><fcvt_target><GPF:mode>2"
6274 [(set (match_operand:GPF 0 "register_operand" "=w,w")
43d0a8ee 6275 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
31ca7cba 6276 "TARGET_FLOAT"
0d35c5c2
VP		 6277 "@
6278 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6279 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
488461d8
MM		 6280 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6281 (set_attr "arch" "simd,fp")]
43e9d192
IB		 6282)
6283
188d0079 6284;; Unequal width integer to fp conversions.
0d35c5c2 6285(define_insn "<optab><fcvt_iesize><GPF:mode>2"
43e9d192 6286 [(set (match_operand:GPF 0 "register_operand" "=w")
0d35c5c2 6287 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
43e9d192 6288 "TARGET_FLOAT"
0d35c5c2
VP		 6289 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6290 [(set_attr "type" "f_cvti2f")]
43e9d192
IB		 6291)
6292
bf1e3646
JG		 6293;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6294;; midend will arrange for an SImode conversion to HFmode to first go
6295;; through DFmode, then to HFmode. But first it will try converting
6296;; to DImode then down, which would match our DImode pattern below and
6297;; give very poor code-generation. So, we must provide our own emulation
6298;; of the mid-end logic.
6299
6300(define_insn "aarch64_fp16_<optab><mode>hf2"
d7f33f07
JW		 6301 [(set (match_operand:HF 0 "register_operand" "=w")
6302 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6303 "TARGET_FP_F16INST"
6304 "<su_optab>cvtf\t%h0, %<w>1"
6305 [(set_attr "type" "f_cvti2f")]
6306)
6307
bf1e3646
JG		 6308(define_expand "<optab>sihf2"
6309 [(set (match_operand:HF 0 "register_operand")
6310 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6311 "TARGET_FLOAT"
6312{
6313 if (TARGET_FP_F16INST)
6314 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6315 else
6316 {
6317 rtx convert_target = gen_reg_rtx (DFmode);
6318 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6319 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6320 }
6321 DONE;
6322}
6323)
6324
6325;; For DImode there is no wide enough floating-point mode that we
6326;; can convert through natively (TFmode would work, but requires a library
6327;; call). However, we know that any value >= 65504 will be rounded
6328;; to infinity on conversion. This is well within the range of SImode, so
6329;; we can:
6330;; Saturate to SImode.
6331;; Convert from that to DFmode
6332;; Convert from that to HFmode (phew!).
6333;; Note that the saturation to SImode requires the SIMD extensions. If
6334;; we ever need to provide this pattern where the SIMD extensions are not
6335;; available, we would need a different approach.
6336
6337(define_expand "<optab>dihf2"
6338 [(set (match_operand:HF 0 "register_operand")
6339 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6340 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6341{
6342 if (TARGET_FP_F16INST)
6343 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6344 else
6345 {
6346 rtx sat_target = gen_reg_rtx (SImode);
6347 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6348 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6349 }
6350
6351 DONE;
6352}
6353)
6354
3f598afe
JW		 6355;; Convert between fixed-point and floating-point (scalar modes)
6356
6357(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6358 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6359 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6360 (match_operand:SI 2 "immediate_operand" "i, i")]
6361 FCVT_F2FIXED))]
6362 ""
6363 "@
1f0e9e34
JG		 6364 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6365 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
3f598afe 6366 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
488461d8 6367 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6368)
6369
6370(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6371 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6372 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6373 (match_operand:SI 2 "immediate_operand" "i, i")]
6374 FCVT_FIXED2F))]
6375 ""
6376 "@
1f0e9e34
JG		 6377 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6378 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
3f598afe 6379 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
488461d8 6380 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6381)
6382
68ad28c3
JW		 6383(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6384 [(set (match_operand:GPI 0 "register_operand" "=r")
6385 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6386 (match_operand:SI 2 "immediate_operand" "i")]
6387 FCVT_F2FIXED))]
6388 "TARGET_FP_F16INST"
6389 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6390 [(set_attr "type" "f_cvtf2i")]
6391)
6392
6393(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6394 [(set (match_operand:HF 0 "register_operand" "=w")
6395 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6396 (match_operand:SI 2 "immediate_operand" "i")]
6397 FCVT_FIXED2F))]
6398 "TARGET_FP_F16INST"
6399 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6400 [(set_attr "type" "f_cvti2f")]
6401)
6402
6403(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6404 [(set (match_operand:HI 0 "register_operand" "=w")
6405 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6406 (match_operand:SI 2 "immediate_operand" "i")]
6407 FCVT_F2FIXED))]
6408 "TARGET_SIMD"
6409 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6410 [(set_attr "type" "neon_fp_to_int_s")]
6411)
6412
6413(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6414 [(set (match_operand:HF 0 "register_operand" "=w")
6415 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6416 (match_operand:SI 2 "immediate_operand" "i")]
6417 FCVT_FIXED2F))]
6418 "TARGET_SIMD"
6419 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6420 [(set_attr "type" "neon_int_to_fp_s")]
6421)
6422
43e9d192
IB		 6423;; -------------------------------------------------------------------
6424;; Floating-point arithmetic
6425;; -------------------------------------------------------------------
6426
6427(define_insn "add<mode>3"
68ad28c3
JW		 6428 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6429 (plus:GPF_F16
6430 (match_operand:GPF_F16 1 "register_operand" "w")
6431 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6432 "TARGET_FLOAT"
6433 "fadd\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6434 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6435)
6436
6437(define_insn "sub<mode>3"
68ad28c3
JW		 6438 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6439 (minus:GPF_F16
6440 (match_operand:GPF_F16 1 "register_operand" "w")
6441 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6442 "TARGET_FLOAT"
6443 "fsub\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6444 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6445)
6446
6447(define_insn "mul<mode>3"
68ad28c3
JW		 6448 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6449 (mult:GPF_F16
6450 (match_operand:GPF_F16 1 "register_operand" "w")
6451 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6452 "TARGET_FLOAT"
6453 "fmul\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6454 [(set_attr "type" "fmul<stype>")]
43e9d192
IB		 6455)
6456
6457(define_insn "*fnmul<mode>3"
6458 [(set (match_operand:GPF 0 "register_operand" "=w")
6459 (mult:GPF
6460 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6461 (match_operand:GPF 2 "register_operand" "w")))]
2b3fe2e9
SN		 6462 "TARGET_FLOAT && !flag_rounding_math"
6463 "fnmul\\t%<s>0, %<s>1, %<s>2"
6464 [(set_attr "type" "fmul<s>")]
6465)
6466
6467(define_insn "*fnmul<mode>3"
6468 [(set (match_operand:GPF 0 "register_operand" "=w")
6469 (neg:GPF (mult:GPF
6470 (match_operand:GPF 1 "register_operand" "w")
6471 (match_operand:GPF 2 "register_operand" "w"))))]
43e9d192
IB		 6472 "TARGET_FLOAT"
6473 "fnmul\\t%<s>0, %<s>1, %<s>2"
99e6af15 6474 [(set_attr "type" "fmul<s>")]
43e9d192
IB		 6475)
6476
79a2bc2d 6477(define_expand "div<mode>3"
68ad28c3
JW		 6478 [(set (match_operand:GPF_F16 0 "register_operand")
6479 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6480 (match_operand:GPF_F16 2 "register_operand")))]
1be49a38 6481 "TARGET_FLOAT"
79a2bc2d
EM		 6482{
6483 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6484 DONE;
6485
6486 operands[1] = force_reg (<MODE>mode, operands[1]);
6487})
6488
6489(define_insn "*div<mode>3"
68ad28c3
JW		 6490 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6491 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6492 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6493 "TARGET_FLOAT"
6494 "fdiv\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6495 [(set_attr "type" "fdiv<stype>")]
43e9d192
IB		 6496)
6497
6498(define_insn "neg<mode>2"
d7f33f07
JW		 6499 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6500 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6501 "TARGET_FLOAT"
6502 "fneg\\t%<s>0, %<s>1"
d7f33f07 6503 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6504)
6505
98daafa0 6506(define_expand "sqrt<mode>2"
1bbffb87
DZ		 6507 [(set (match_operand:GPF_F16 0 "register_operand")
6508 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
98daafa0
EM		 6509 "TARGET_FLOAT"
6510{
6511 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6512 DONE;
6513})
6514
6515(define_insn "*sqrt<mode>2"
d7f33f07
JW		 6516 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6517 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6518 "TARGET_FLOAT"
6519 "fsqrt\\t%<s>0, %<s>1"
d7f33f07 6520 [(set_attr "type" "fsqrt<stype>")]
43e9d192
IB		 6521)
6522
6523(define_insn "abs<mode>2"
d7f33f07
JW		 6524 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6525 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6526 "TARGET_FLOAT"
6527 "fabs\\t%<s>0, %<s>1"
d7f33f07 6528 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6529)
6530
6531;; Given that smax/smin do not specify the result when either input is NaN,
6532;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6533;; for smin.
6534
6535(define_insn "smax<mode>3"
6536 [(set (match_operand:GPF 0 "register_operand" "=w")
6537 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6538 (match_operand:GPF 2 "register_operand" "w")))]
6539 "TARGET_FLOAT"
6540 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6541 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6542)
6543
6544(define_insn "smin<mode>3"
6545 [(set (match_operand:GPF 0 "register_operand" "=w")
6546 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6547 (match_operand:GPF 2 "register_operand" "w")))]
6548 "TARGET_FLOAT"
6549 "fminnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6550 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6551)
6552
1efafef3
TC		 6553;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6554;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6555;; which implement the IEEE fmax ()/fmin () functions.
6d331688 6556(define_insn "<fmaxmin><mode>3"
68ad28c3
JW		 6557 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6558 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6559 (match_operand:GPF_F16 2 "register_operand" "w")]
1efafef3 6560 FMAXMIN_UNS))]
202d0c11 6561 "TARGET_FLOAT"
1efafef3 6562 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6563 [(set_attr "type" "f_minmax<stype>")]
202d0c11
DS		 6564)
6565
f66e6423
TC		 6566(define_expand "lrint<GPF:mode><GPI:mode>2"
6567 [(match_operand:GPI 0 "register_operand")
6568 (match_operand:GPF 1 "register_operand")]
1013465f 6569 "TARGET_FLOAT
fb802d91 6570 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
1013465f 6571 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
f66e6423
TC		 6572{
6573 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6574 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6575 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6576 DONE;
6577}
6578)
6579
fb0f04fe
JG		 6580;; For copysign (x, y), we want to generate:
6581;;
6582;; LDR d2, #(1 << 63)
6583;; BSL v2.8b, [y], [x]
6584;;
315fdae8
RE		 6585;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6586;; we expect these operations to nearly always operate on
6587;; floating-point values, we do not want the operation to be
6588;; simplified into a bit-field insert operation that operates on the
6589;; integer side, since typically that would involve three inter-bank
6590;; register copies. As we do not expect copysign to be followed by
6591;; other logical operations on the result, it seems preferable to keep
6592;; this as an unspec operation, rather than exposing the underlying
6593;; logic to the compiler.
6594
6595(define_expand "copysign<GPF:mode>3"
6596 [(match_operand:GPF 0 "register_operand")
6597 (match_operand:GPF 1 "register_operand")
6598 (match_operand:GPF 2 "register_operand")]
fb0f04fe
JG		 6599 "TARGET_FLOAT && TARGET_SIMD"
6600{
315fdae8
RE		 6601 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6602 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6603 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6604 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6605 bitmask));
fb0f04fe
JG		 6606 DONE;
6607}
6608)
6609
315fdae8
RE		 6610(define_insn "copysign<GPF:mode>3_insn"
6611 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6612 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6613 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6614 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6615 UNSPEC_COPYSIGN))]
fb0f04fe 6616 "TARGET_FLOAT && TARGET_SIMD"
315fdae8
RE		 6617 "@
6618 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6619 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6620 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6621 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6622 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
fb0f04fe
JG		 6623)
6624
315fdae8 6625
4261463d
TC		 6626;; For xorsign (x, y), we want to generate:
6627;;
6628;; LDR d2, #1<<63
6629;; AND v3.8B, v1.8B, v2.8B
6630;; EOR v0.8B, v0.8B, v3.8B
6631;;
6632
6633(define_expand "xorsign<mode>3"
6634 [(match_operand:GPF 0 "register_operand")
6635 (match_operand:GPF 1 "register_operand")
6636 (match_operand:GPF 2 "register_operand")]
6637 "TARGET_FLOAT && TARGET_SIMD"
6638{
6639
5f565314 6640 machine_mode imode = <V_INT_EQUIV>mode;
4261463d
TC		 6641 rtx mask = gen_reg_rtx (imode);
6642 rtx op1x = gen_reg_rtx (imode);
6643 rtx op2x = gen_reg_rtx (imode);
6644
6645 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6646 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6647 imode)));
6648
5f565314
RS		 6649 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6650 lowpart_subreg (imode, operands[2],
6651 <MODE>mode)));
6652 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6653 lowpart_subreg (imode, operands[1],
6654 <MODE>mode),
6655 op2x));
4261463d
TC		 6656 emit_move_insn (operands[0],
6657 lowpart_subreg (<MODE>mode, op1x, imode));
6658 DONE;
6659}
6660)
6661
43e9d192
IB		 6662;; -------------------------------------------------------------------
6663;; Reload support
6664;; -------------------------------------------------------------------
b4f50fd4
RR		 6665;; Reload Scalar Floating point modes from constant pool.
6666;; The AArch64 port doesn't have __int128 constant move support.
1bbffb87 6667;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
0016d8d9 6668(define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
b4f50fd4
RR		 6669 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6670 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6671 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6672 "TARGET_FLOAT"
b4f50fd4
RR		 6673 {
6674 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6675 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6676 DONE;
6677 }
6678)
6679
6680;; Reload Vector modes from constant pool.
0016d8d9 6681(define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
b4f50fd4
RR		 6682 [(set (match_operand:VALL 0 "register_operand" "=w")
6683 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6684 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6685 "TARGET_FLOAT"
b4f50fd4
RR		 6686 {
6687 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6688 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6689 DONE;
6690 }
6691)
43e9d192 6692
0016d8d9 6693(define_expand "@aarch64_reload_mov<mode>"
43e9d192
IB		 6694 [(set (match_operand:TX 0 "register_operand" "=w")
6695 (match_operand:TX 1 "register_operand" "w"))
6696 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6697 ]
a636bb40 6698 "TARGET_FLOAT"
43e9d192
IB		 6699 {
6700 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6701 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6702 gen_aarch64_movtilow_tilow (op0, op1);
6703 gen_aarch64_movdi_tihigh (operands[2], op1);
6704 gen_aarch64_movtihigh_di (op0, operands[2]);
6705 DONE;
6706 }
6707)
6708
6709;; The following secondary reload helpers patterns are invoked
6710;; after or during reload as we don't want these patterns to start
6711;; kicking in during the combiner.
2a9704d0 6712
0016d8d9 6713(define_insn "@aarch64_movdi_<mode>low"
43e9d192 6714 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6715 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6716 (const_int 64) (const_int 0)))]
a636bb40 6717 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6718 "fmov\\t%x0, %d1"
c1dfcec7 6719 [(set_attr "type" "f_mrc")
43e9d192
IB		 6720 (set_attr "length" "4")
6721 ])
6722
0016d8d9 6723(define_insn "@aarch64_movdi_<mode>high"
43e9d192 6724 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6725 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6726 (const_int 64) (const_int 64)))]
a636bb40 6727 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6728 "fmov\\t%x0, %1.d[1]"
c1dfcec7 6729 [(set_attr "type" "f_mrc")
43e9d192
IB		 6730 (set_attr "length" "4")
6731 ])
6732
0016d8d9 6733(define_insn "@aarch64_mov<mode>high_di"
12dc6974 6734 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
43e9d192 6735 (const_int 64) (const_int 64))
12dc6974 6736 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6737 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6738 "fmov\\t%0.d[1], %x1"
c1dfcec7 6739 [(set_attr "type" "f_mcr")
43e9d192
IB		 6740 (set_attr "length" "4")
6741 ])
6742
0016d8d9 6743(define_insn "@aarch64_mov<mode>low_di"
12dc6974
SN		 6744 [(set (match_operand:TX 0 "register_operand" "=w")
6745 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6746 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6747 "fmov\\t%d0, %x1"
c1dfcec7 6748 [(set_attr "type" "f_mcr")
43e9d192
IB		 6749 (set_attr "length" "4")
6750 ])
6751
6752(define_insn "aarch64_movtilow_tilow"
6753 [(set (match_operand:TI 0 "register_operand" "=w")
a636bb40 6754 (zero_extend:TI
43e9d192 6755 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
a636bb40 6756 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6757 "fmov\\t%d0, %d1"
d7256bb8 6758 [(set_attr "type" "fmov")
43e9d192
IB		 6759 (set_attr "length" "4")
6760 ])
6761
6762;; There is a deliberate reason why the parameters of high and lo_sum's
6763;; don't have modes for ADRP and ADD instructions. This is to allow high
6764;; and lo_sum's to be used with the labels defining the jump tables in
6765;; rodata section.
6766
28514dda 6767(define_expand "add_losym"
1bbffb87
DZ		 6768 [(set (match_operand 0 "register_operand")
6769 (lo_sum (match_operand 1 "register_operand")
6770 (match_operand 2 "aarch64_valid_symref")))]
43e9d192 6771 ""
28514dda 6772{
ef4bddc2 6773 machine_mode mode = GET_MODE (operands[0]);
28514dda
YZ		 6774
6775 emit_insn ((mode == DImode
6776 ? gen_add_losym_di
6777 : gen_add_losym_si) (operands[0],
6778 operands[1],
6779 operands[2]));
6780 DONE;
6781})
6782
6783(define_insn "add_losym_<mode>"
6784 [(set (match_operand:P 0 "register_operand" "=r")
6785 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6786 (match_operand 2 "aarch64_valid_symref" "S")))]
6787 ""
d6591257 6788 "add\\t%<w>0, %<w>1, :lo12:%c2"
d8e11262 6789 [(set_attr "type" "alu_imm")]
28514dda 6790)
43e9d192 6791
1b1e81f8
JW		 6792(define_insn "ldr_got_small_28k_<mode>"
6793 [(set (match_operand:PTR 0 "register_operand" "=r")
6794 (unspec:PTR [(mem:PTR (lo_sum:PTR
6795 (match_operand:PTR 1 "register_operand" "r")
6796 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6797 UNSPEC_GOTSMALLPIC28K))]
6798 ""
d6591257 6799 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
db46a2e6 6800 [(set_attr "type" "load_<ldst_sz>")]
1b1e81f8
JW		 6801)
6802
6803(define_insn "ldr_got_small_28k_sidi"
6804 [(set (match_operand:DI 0 "register_operand" "=r")
6805 (zero_extend:DI
6806 (unspec:SI [(mem:SI (lo_sum:DI
6807 (match_operand:DI 1 "register_operand" "r")
6808 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6809 UNSPEC_GOTSMALLPIC28K)))]
6810 "TARGET_ILP32"
d6591257 6811 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
89b2133e 6812 [(set_attr "type" "load_4")]
1b1e81f8
JW		 6813)
6814
d91480de
D		 6815(define_insn "@ldr_got_tiny_<mode>"
6816 [(set (match_operand:PTR 0 "register_operand" "=r")
6817 (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")]
6818 UNSPEC_GOTTINYPIC))]
87dd8ab0 6819 ""
d91480de
D		 6820 "ldr\t%<w>0, %L1"
6821 [(set_attr "type" "load_<ldst_sz>")]
6822)
6823
6824(define_insn "ldr_got_tiny_sidi"
6825 [(set (match_operand:DI 0 "register_operand" "=r")
6826 (zero_extend:DI
6827 (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6828 UNSPEC_GOTTINYPIC)))]
6829 "TARGET_ILP32"
6830 "ldr\t%w0, %L1"
6831 [(set_attr "type" "load_4")]
87dd8ab0
MS		 6832)
6833
43e9d192
IB		 6834(define_insn "aarch64_load_tp_hard"
6835 [(set (match_operand:DI 0 "register_operand" "=r")
6836 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6837 ""
6838 "mrs\\t%0, tpidr_el0"
99e6af15 6839 [(set_attr "type" "mrs")]
43e9d192
IB		 6840)
6841
6842;; The TLS ABI specifically requires that the compiler does not schedule
6843;; instructions in the TLS stubs, in order to enable linker relaxation.
6844;; Therefore we treat the stubs as an atomic sequence.
23b88fda 6845(define_expand "tlsgd_small_<mode>"
87ca615a 6846 [(parallel [(set (match_operand:PTR 0 "register_operand")
43e9d192 6847 (call (mem:DI (match_dup 2)) (const_int 1)))
08cc4d92 6848 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
87ca615a 6849 (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6850 (clobber (reg:DI LR_REGNUM))])]
6851 ""
6852{
6853 operands[2] = aarch64_tls_get_addr ();
6854})
6855
23b88fda 6856(define_insn "*tlsgd_small_<mode>"
87ca615a 6857 [(set (match_operand:PTR 0 "register_operand" "")
43e9d192 6858 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
08cc4d92 6859 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
87ca615a 6860 (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6861 (clobber (reg:DI LR_REGNUM))
6862 ]
6863 ""
6864 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
c1dfcec7 6865 [(set_attr "type" "call")
43e9d192
IB		 6866 (set_attr "length" "16")])
6867
621ad2de
AP		 6868(define_insn "tlsie_small_<mode>"
6869 [(set (match_operand:PTR 0 "register_operand" "=r")
6870 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
43e9d192
IB		 6871 UNSPEC_GOTSMALLTLS))]
6872 ""
621ad2de 6873 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
89b2133e 6874 [(set_attr "type" "load_4")
43e9d192
IB		 6875 (set_attr "length" "8")]
6876)
6877
621ad2de 6878(define_insn "tlsie_small_sidi"
43e9d192 6879 [(set (match_operand:DI 0 "register_operand" "=r")
621ad2de
AP		 6880 (zero_extend:DI
6881 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6882 UNSPEC_GOTSMALLTLS)))]
6883 ""
6884 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
89b2133e 6885 [(set_attr "type" "load_4")
621ad2de
AP		 6886 (set_attr "length" "8")]
6887)
6888
5ae7caad
JW		 6889(define_insn "tlsie_tiny_<mode>"
6890 [(set (match_operand:PTR 0 "register_operand" "=&r")
6891 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6892 (match_operand:PTR 2 "register_operand" "r")]
6893 UNSPEC_GOTTINYTLS))]
6894 ""
6895 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6896 [(set_attr "type" "multiple")
6897 (set_attr "length" "8")]
6898)
6899
6900(define_insn "tlsie_tiny_sidi"
6901 [(set (match_operand:DI 0 "register_operand" "=&r")
6902 (zero_extend:DI
6903 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6904 (match_operand:DI 2 "register_operand" "r")
6905 ]
6906 UNSPEC_GOTTINYTLS)))]
6907 ""
5b295a81 6908 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5ae7caad
JW		 6909 [(set_attr "type" "multiple")
6910 (set_attr "length" "8")]
6911)
6912
cbf5629e
JW		 6913(define_insn "tlsle12_<mode>"
6914 [(set (match_operand:P 0 "register_operand" "=r")
6915 (unspec:P [(match_operand:P 1 "register_operand" "r")
6916 (match_operand 2 "aarch64_tls_le_symref" "S")]
6917 UNSPEC_TLSLE12))]
621ad2de 6918 ""
cbf5629e
JW		 6919 "add\\t%<w>0, %<w>1, #%L2";
6920 [(set_attr "type" "alu_sreg")
6921 (set_attr "length" "4")]
6922)
621ad2de 6923
cbf5629e 6924(define_insn "tlsle24_<mode>"
621ad2de 6925 [(set (match_operand:P 0 "register_operand" "=r")
cbf5629e
JW		 6926 (unspec:P [(match_operand:P 1 "register_operand" "r")
6927 (match_operand 2 "aarch64_tls_le_symref" "S")]
6928 UNSPEC_TLSLE24))]
43e9d192 6929 ""
0699caae 6930 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
cbf5629e 6931 [(set_attr "type" "multiple")
43e9d192
IB		 6932 (set_attr "length" "8")]
6933)
6934
cbf5629e
JW		 6935(define_insn "tlsle32_<mode>"
6936 [(set (match_operand:P 0 "register_operand" "=r")
6937 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6938 UNSPEC_TLSLE32))]
6939 ""
6940 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6941 [(set_attr "type" "multiple")
6942 (set_attr "length" "8")]
6943)
6944
6945(define_insn "tlsle48_<mode>"
6946 [(set (match_operand:P 0 "register_operand" "=r")
6947 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6948 UNSPEC_TLSLE48))]
6949 ""
6950 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6951 [(set_attr "type" "multiple")
6952 (set_attr "length" "12")]
6953)
6954
80c13ac5
RS		 6955(define_expand "tlsdesc_small_<mode>"
6956 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6957 "TARGET_TLS_DESC"
6958 {
6959 if (TARGET_SVE)
bb6ce448
RS		 6960 {
6961 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6962 rtx_insn *call
6963 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6964 RTL_CONST_CALL_P (call) = 1;
6965 }
80c13ac5
RS		 6966 else
6967 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6968 DONE;
6969 }
6970)
6971
6972;; tlsdesc calls preserve all core and Advanced SIMD registers except
6973;; R0 and LR.
6974(define_insn "tlsdesc_small_advsimd_<mode>"
621ad2de
AP		 6975 [(set (reg:PTR R0_REGNUM)
6976 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
80c13ac5 6977 UNSPEC_TLSDESC))
43e9d192 6978 (clobber (reg:DI LR_REGNUM))
0bc11714 6979 (clobber (reg:CC CC_REGNUM))
0e510d18
ST		 6980 (clobber (match_scratch:DI 1 "=r"))
6981 (use (reg:DI FP_REGNUM))]
80c13ac5
RS		 6982 "TARGET_TLS_DESC && !TARGET_SVE"
6983 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6984 [(set_attr "type" "call")
6985 (set_attr "length" "16")])
6986
bb6ce448
RS		 6987;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6988;; describing the extra call-preserved guarantees. This would work
6989;; for non-SVE too, but avoiding a call is probably better if we can.
80c13ac5
RS		 6990(define_insn "tlsdesc_small_sve_<mode>"
6991 [(set (reg:PTR R0_REGNUM)
bb6ce448
RS		 6992 (call (mem:DI (unspec:PTR
6993 [(match_operand 0 "aarch64_valid_symref")]
6994 UNSPEC_TLSDESC))
6995 (const_int 0)))
6996 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
ba2913f6 6997 (clobber (reg:DI LR_REGNUM))]
80c13ac5 6998 "TARGET_TLS_DESC && TARGET_SVE"
ba2913f6 6999 "adrp\\tx0, %A0\;ldr\\t<w>30, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\tx30"
c1dfcec7 7000 [(set_attr "type" "call")
43e9d192
IB		 7001 (set_attr "length" "16")])
7002
7003(define_insn "stack_tie"
7004 [(set (mem:BLK (scratch))
7005 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
7006 (match_operand:DI 1 "register_operand" "rk")]
7007 UNSPEC_PRLG_STK))]
7008 ""
7009 ""
7010 [(set_attr "length" "0")]
7011)
7012
e1d5d19e
KT		 7013(define_insn "aarch64_fjcvtzs"
7014 [(set (match_operand:SI 0 "register_operand" "=r")
7015 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
d2b86e14
AC		 7016 UNSPEC_FJCVTZS))
7017 (clobber (reg:CC CC_REGNUM))]
e1d5d19e
KT		 7018 "TARGET_JSCVT"
7019 "fjcvtzs\\t%w0, %d1"
7020 [(set_attr "type" "f_cvtf2i")]
7021)
7022
db58fd89
JW		 7023;; Pointer authentication patterns are always provided. In architecture
7024;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
7025;; This lets the user write portable software which authenticates pointers
7026;; when run on something which implements ARMv8.3-A, and which runs
7027;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
7028;; implemented.
7029
7030;; Signing/Authenticating R30 using SP as the salt.
7031
7032(define_insn "<pauth_mnem_prefix>sp"
7033 [(set (reg:DI R30_REGNUM)
7034 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
7035 ""
8fc16d72 7036 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
db58fd89
JW		 7037)
7038
7039;; Signing/Authenticating X17 using X16 as the salt.
7040
7041(define_insn "<pauth_mnem_prefix>1716"
7042 [(set (reg:DI R17_REGNUM)
7043 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
7044 ""
8fc16d72 7045 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
db58fd89
JW		 7046)
7047
7048;; Stripping the signature in R30.
7049
7050(define_insn "xpaclri"
7051 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
7052 ""
7053 "hint\t7 // xpaclri"
7054)
7055
a3eb8a52
EB		 7056;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
7057;; all of memory. This blocks insns from being moved across this point.
7058
7059(define_insn "blockage"
7060 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
7061 ""
7062 ""
7063 [(set_attr "length" "0")
7064 (set_attr "type" "block")]
7065)
7066
5f5c5e0f 7067(define_insn "probe_stack_range"
cd1bef27 7068 [(set (match_operand:DI 0 "register_operand" "=rk")
5f5c5e0f
EB		 7069 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
7070 (match_operand:DI 2 "register_operand" "r")]
7071 UNSPECV_PROBE_STACK_RANGE))]
a3eb8a52
EB		 7072 ""
7073{
7074 return aarch64_output_probe_stack_range (operands[0], operands[2]);
7075}
7076 [(set_attr "length" "32")]
7077)
7078
eb471ba3
TC		 7079;; This instruction is used to generate the stack clash stack adjustment and
7080;; probing loop. We can't change the control flow during prologue and epilogue
7081;; code generation. So we must emit a volatile unspec and expand it later on.
7082
7083(define_insn "@probe_sve_stack_clash_<mode>"
7084 [(set (match_operand:P 0 "register_operand" "=rk")
7085 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
7086 (match_operand:P 2 "register_operand" "r")
7087 (match_operand:P 3 "const_int_operand" "n")
7088 (match_operand:P 4 "aarch64_plus_immediate" "L")]
7089 UNSPECV_PROBE_STACK_RANGE))]
7090 "TARGET_SVE"
7091{
7092 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
7093 operands[3], operands[4]);
7094}
7095 [(set_attr "length" "28")]
7096)
7097
a5bc806c
AP		 7098;; Named pattern for expanding thread pointer reference.
7099(define_expand "get_thread_pointerdi"
1bbffb87 7100 [(match_operand:DI 0 "register_operand")]
a5bc806c
AP		 7101 ""
7102{
7103 rtx tmp = aarch64_load_tp (operands[0]);
7104 if (tmp != operands[0])
7105 emit_move_insn (operands[0], tmp);
7106 DONE;
7107})
7108
74b27d8e
RS		 7109;; Defined for -mstack-protector-guard=sysreg, which goes through this
7110;; pattern rather than stack_protect_combined_set. Our implementation
7111;; of the latter can handle both.
36e17020
VK		 7112(define_expand "stack_protect_set"
7113 [(match_operand 0 "memory_operand")
74b27d8e 7114 (match_operand 1 "")]
36e17020
VK		 7115 ""
7116{
74b27d8e
RS		 7117 emit_insn (gen_stack_protect_combined_set (operands[0], operands[1]));
7118 DONE;
7119})
36e17020 7120
74b27d8e
RS		 7121(define_expand "stack_protect_combined_set"
7122 [(match_operand 0 "memory_operand")
7123 (match_operand 1 "")]
7124 ""
7125{
7126 machine_mode mode = GET_MODE (operands[0]);
7127 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
7128 AARCH64_SALT_SSP_SET);
36e17020
VK		 7129 emit_insn ((mode == DImode
7130 ? gen_stack_protect_set_di
7131 : gen_stack_protect_set_si) (operands[0], operands[1]));
7132 DONE;
7133})
7134
74b27d8e 7135;; Operand 1 is either AARCH64_SALT_SSP_SET or AARCH64_SALT_SSP_TEST.
cd0b2d36
RR		 7136(define_insn "reg_stack_protect_address_<mode>"
7137 [(set (match_operand:PTR 0 "register_operand" "=r")
74b27d8e
RS		 7138 (unspec:PTR [(match_operand 1 "const_int_operand")]
7139 UNSPEC_SSP_SYSREG))]
cd0b2d36
RR		 7140 "aarch64_stack_protector_guard != SSP_GLOBAL"
7141 {
7142 char buf[150];
7143 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7144 aarch64_stack_protector_guard_reg_str);
7145 output_asm_insn (buf, operands);
7146 return "";
7147 }
7148 [(set_attr "type" "mrs")])
7149
a7e73b41
RE		 7150;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7151;; canary value does not live beyond the life of this sequence.
36e17020
VK		 7152(define_insn "stack_protect_set_<mode>"
7153 [(set (match_operand:PTR 0 "memory_operand" "=m")
7154 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7155 UNSPEC_SP_SET))
7156 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7157 ""
a7e73b41 7158 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
36e17020
VK		 7159 [(set_attr "length" "12")
7160 (set_attr "type" "multiple")])
7161
74b27d8e
RS		 7162;; Defined for -mstack-protector-guard=sysreg, which goes through this
7163;; pattern rather than stack_protect_combined_test. Our implementation
7164;; of the latter can handle both.
36e17020
VK		 7165(define_expand "stack_protect_test"
7166 [(match_operand 0 "memory_operand")
74b27d8e 7167 (match_operand 1 "")
36e17020
VK		 7168 (match_operand 2)]
7169 ""
7170{
74b27d8e
RS		 7171 emit_insn (gen_stack_protect_combined_test (operands[0], operands[1],
7172 operands[2]));
7173 DONE;
7174})
9c6ab05f 7175
74b27d8e
RS		 7176(define_expand "stack_protect_combined_test"
7177 [(match_operand 0 "memory_operand")
7178 (match_operand 1 "")
7179 (match_operand 2)]
7180 ""
7181{
7182 machine_mode mode = GET_MODE (operands[0]);
7183 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
7184 AARCH64_SALT_SSP_TEST);
36e17020 7185 emit_insn ((mode == DImode
fe1a2642
RS		 7186 ? gen_stack_protect_test_di
7187 : gen_stack_protect_test_si) (operands[0], operands[1]));
7188
7189 rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
7190 emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx),
7191 cc_reg, operands[2]));
36e17020
VK		 7192 DONE;
7193})
7194
fe1a2642
RS		 7195;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7196;; canary value does not live beyond the end of this sequence.
36e17020 7197(define_insn "stack_protect_test_<mode>"
fe1a2642
RS		 7198 [(set (reg:CC CC_REGNUM)
7199 (unspec:CC [(match_operand:PTR 0 "memory_operand" "m")
7200 (match_operand:PTR 1 "memory_operand" "m")]
7201 UNSPEC_SP_TEST))
7202 (clobber (match_scratch:PTR 2 "=&r"))
b4e9251d 7203 (clobber (match_scratch:PTR 3 "=&r"))]
36e17020 7204 ""
fe1a2642
RS		 7205 "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0"
7206 [(set_attr "length" "16")
36e17020
VK		 7207 (set_attr "type" "multiple")])
7208
0d7e5fa6
AC		 7209;; Write into the Floating-point Status or Control Register.
7210(define_insn "@aarch64_set_<fpscr_name><GPI:mode>"
7211 [(unspec_volatile [(match_operand:GPI 0 "register_operand" "r")] SET_FPSCR)]
aa87aced 7212 ""
0d7e5fa6 7213 "msr\\t<fpscr_name>, %0"
aa87aced
KV		 7214 [(set_attr "type" "mrs")])
7215
0d7e5fa6
AC		 7216;; Read into the Floating-point Status or Control Register.
7217(define_insn "@aarch64_get_<fpscr_name><GPI:mode>"
7218 [(set (match_operand:GPI 0 "register_operand" "=r")
7219 (unspec_volatile:GPI [(const_int 0)] GET_FPSCR))]
aa87aced 7220 ""
0d7e5fa6 7221 "mrs\\t%0, <fpscr_name>"
aa87aced
KV		 7222 [(set_attr "type" "mrs")])
7223
594bdd53
FY		 7224;; Define the subtract-one-and-jump insns so loop.c
7225;; knows what to generate.
7226(define_expand "doloop_end"
7227 [(use (match_operand 0 "" "")) ; loop pseudo
7228 (use (match_operand 1 "" ""))] ; label
7229 "optimize > 0 && flag_modulo_sched"
7230{
7231 rtx s0;
7232 rtx bcomp;
7233 rtx loc_ref;
7234 rtx cc_reg;
7235 rtx insn;
7236 rtx cmp;
7237
7238 /* Currently SMS relies on the do-loop pattern to recognize loops
7239 where (1) the control part consists of all insns defining and/or
7240 using a certain 'count' register and (2) the loop count can be
7241 adjusted by modifying this register prior to the loop.
7242 ??? The possible introduction of a new block to initialize the
7243 new IV can potentially affect branch optimizations. */
7244
7245 if (GET_MODE (operands[0]) != DImode)
7246 FAIL;
7247
7248 s0 = operands [0];
7249 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7250
7251 cmp = XVECEXP (PATTERN (insn), 0, 0);
7252 cc_reg = SET_DEST (cmp);
7253 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7254 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
f7df4a84 7255 emit_jump_insn (gen_rtx_SET (pc_rtx,
594bdd53
FY		 7256 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7257 loc_ref, pc_rtx)));
7258 DONE;
7259})
7260
3751345d
RE		 7261;; Track speculation through conditional branches. We assume that
7262;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7263(define_insn "speculation_tracker"
7264 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
996bea89 7265 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
3751345d
RE		 7266 UNSPEC_SPECULATION_TRACKER))]
7267 ""
7268 {
7269 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7270 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7271 return "";
7272 }
7273 [(set_attr "type" "csel")]
7274)
7275
2214053f
RS		 7276;; Like speculation_tracker, but track the inverse condition.
7277(define_insn "speculation_tracker_rev"
7278 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7279 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7280 UNSPEC_SPECULATION_TRACKER_REV))]
7281 ""
7282 {
7283 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7284 output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands);
7285 return "";
7286 }
7287 [(set_attr "type" "csel")]
7288)
7289
b5f794b4
SD		 7290;; BTI <target> instructions
7291(define_insn "bti_noarg"
7292 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7293 ""
7294 "hint\t32 // bti"
7295 [(set_attr "type" "no_insn")]
7296)
7297
7298(define_insn "bti_c"
7299 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7300 ""
7301 "hint\t34 // bti c"
7302 [(set_attr "type" "no_insn")]
7303)
7304
7305(define_insn "bti_j"
7306 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7307 ""
7308 "hint\t36 // bti j"
7309 [(set_attr "type" "no_insn")]
7310)
7311
7312(define_insn "bti_jc"
7313 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7314 ""
7315 "hint\t38 // bti jc"
7316 [(set_attr "type" "no_insn")]
7317)
7318
def745b2
RE		 7319;; Hard speculation barrier.
7320(define_insn "speculation_barrier"
7321 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7322 ""
7323 "isb\;dsb\\tsy"
7324 [(set_attr "length" "8")
3751345d
RE		 7325 (set_attr "type" "block")
7326 (set_attr "speculation_barrier" "true")]
def745b2
RE		 7327)
7328
c0111dc4
RE		 7329;; Support for __builtin_speculation_safe_value when we have speculation
7330;; tracking enabled. Use the speculation tracker to decide whether to
7331;; copy operand 1 to the target, or to copy the fail value (operand 2).
21cebf90 7332(define_expand "@despeculate_copy<ALLI_TI:mode>"
1bbffb87 7333 [(set (match_operand:ALLI_TI 0 "register_operand")
c0111dc4 7334 (unspec_volatile:ALLI_TI
1bbffb87
DZ		 7335 [(match_operand:ALLI_TI 1 "register_operand")
7336 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
c0111dc4
RE		 7337 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7338 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7339 ""
7340 "
7341 {
7342 if (operands[2] == const0_rtx)
7343 {
7344 rtx tracker;
7345 if (<MODE>mode == TImode)
7346 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7347 else
7348 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7349
7350 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7351 tracker));
7352 DONE;
7353 }
7354 }
7355 "
7356)
7357
7358;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7359;; encoding for CSDB, but will work in older versions of the assembler.
7360(define_insn "*despeculate_copy<ALLI:mode>_insn"
7361 [(set (match_operand:ALLI 0 "register_operand" "=r")
7362 (unspec_volatile:ALLI
7363 [(match_operand:ALLI 1 "register_operand" "r")
7364 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7365 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7366 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7367 ""
7368 {
7369 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7370 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7371 operands);
7372 return "";
7373 }
7374 [(set_attr "length" "12")
7375 (set_attr "type" "block")
7376 (set_attr "speculation_barrier" "true")]
7377)
7378
7379;; Pattern to match despeculate_copyti
7380(define_insn "*despeculate_copyti_insn"
7381 [(set (match_operand:TI 0 "register_operand" "=r")
7382 (unspec_volatile:TI
7383 [(match_operand:TI 1 "register_operand" "r")
7384 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7385 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7386 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7387 ""
7388 {
7389 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7390 output_asm_insn
7391 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7392 operands);
7393 return "";
7394 }
7395 [(set_attr "length" "16")
7396 (set_attr "type" "block")
7397 (set_attr "speculation_barrier" "true")]
7398)
7399
7400(define_insn "despeculate_simple<ALLI:mode>"
7401 [(set (match_operand:ALLI 0 "register_operand" "=r")
7402 (unspec_volatile:ALLI
7403 [(match_operand:ALLI 1 "register_operand" "r")
7404 (use (match_operand:ALLI 2 "register_operand" ""))]
7405 UNSPECV_SPECULATION_BARRIER))]
7406 ""
7407 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7408 [(set_attr "type" "block")
7409 (set_attr "length" "8")
7410 (set_attr "speculation_barrier" "true")]
7411)
7412
7413(define_insn "despeculate_simpleti"
7414 [(set (match_operand:TI 0 "register_operand" "=r")
7415 (unspec_volatile:TI
7416 [(match_operand:TI 1 "register_operand" "r")
7417 (use (match_operand:DI 2 "register_operand" ""))]
7418 UNSPECV_SPECULATION_BARRIER))]
7419 ""
7420 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7421 [(set_attr "type" "block")
7422 (set_attr "length" "12")
7423 (set_attr "speculation_barrier" "true")]
7424)
7425
10bd1d96
KT		 7426(define_insn "aarch64_<frintnzs_op><mode>"
7427 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7428 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7429 FRINTNZX))]
7430 "TARGET_FRINT && TARGET_FLOAT
7431 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7432 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7433 [(set_attr "type" "f_rint<stype>")]
7434)
7435
89626179
SD		 7436;; Transactional Memory Extension (TME) instructions.
7437
7438(define_insn "tstart"
7439 [(set (match_operand:DI 0 "register_operand" "=r")
7440 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7441 (clobber (mem:BLK (scratch)))]
7442 "TARGET_TME"
7443 "tstart\\t%0"
7444 [(set_attr "type" "tme")]
7445)
7446
7447(define_insn "ttest"
7448 [(set (match_operand:DI 0 "register_operand" "=r")
7449 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7450 (clobber (mem:BLK (scratch)))]
7451 "TARGET_TME"
7452 "ttest\\t%0"
7453 [(set_attr "type" "tme")]
7454)
7455
7456(define_insn "tcommit"
7457 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7458 (clobber (mem:BLK (scratch)))]
7459 "TARGET_TME"
7460 "tcommit"
7461 [(set_attr "type" "tme")]
7462)
7463
7464(define_insn "tcancel"
7465 [(unspec_volatile:BLK
7466 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7467 (clobber (mem:BLK (scratch)))]
7468 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7469 "tcancel\\t#%0"
7470 [(set_attr "type" "tme")]
7471)
7472
c5dc215d
KT		 7473(define_insn "aarch64_rndr"
7474 [(set (match_operand:DI 0 "register_operand" "=r")
7475 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7476 (set (reg:CC_Z CC_REGNUM)
7477 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7478 "TARGET_RNG"
7479 "mrs\t%0, RNDR"
7480 [(set_attr "type" "mrs")]
7481)
7482
7483(define_insn "aarch64_rndrrs"
7484 [(set (match_operand:DI 0 "register_operand" "=r")
7485 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7486 (set (reg:CC_Z CC_REGNUM)
7487 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7488 "TARGET_RNG"
7489 "mrs\t%0, RNDRRS"
7490 [(set_attr "type" "mrs")]
7491)
7492
ef01e6bb
DZ		 7493;; Memory Tagging Extension (MTE) instructions.
7494
7495(define_insn "irg"
7496 [(set (match_operand:DI 0 "register_operand" "=rk")
7497 (ior:DI
7498 (and:DI (match_operand:DI 1 "register_operand" "rk")
7499 (const_int -1080863910568919041)) ;; 0xf0ff...
7500 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")]
7501 UNSPEC_GEN_TAG_RND)
7502 (const_int 56))))]
7503 "TARGET_MEMTAG"
7504 "irg\\t%0, %1, %2"
7505 [(set_attr "type" "memtag")]
7506)
7507
7508(define_insn "gmi"
7509 [(set (match_operand:DI 0 "register_operand" "=r")
7510 (ior:DI (ashift:DI
7511 (const_int 1)
7512 (and:QI (lshiftrt:DI
7513 (match_operand:DI 1 "register_operand" "rk")
7514 (const_int 56)) (const_int 15)))
7515 (match_operand:DI 2 "register_operand" "r")))]
7516 "TARGET_MEMTAG"
7517 "gmi\\t%0, %1, %2"
7518 [(set_attr "type" "memtag")]
7519)
7520
7521(define_insn "addg"
7522 [(set (match_operand:DI 0 "register_operand" "=rk")
7523 (ior:DI
7524 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7525 (match_operand:DI 2 "aarch64_granule16_uimm6" "i"))
7526 (const_int -1080863910568919041)) ;; 0xf0ff...
7527 (ashift:DI
7528 (unspec:QI
7529 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15))
7530 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")]
7531 UNSPEC_GEN_TAG)
7532 (const_int 56))))]
7533 "TARGET_MEMTAG"
7534 "addg\\t%0, %1, #%2, #%3"
7535 [(set_attr "type" "memtag")]
7536)
7537
7538(define_insn "subp"
7539 [(set (match_operand:DI 0 "register_operand" "=r")
7540 (minus:DI
7541 (and:DI (match_operand:DI 1 "register_operand" "rk")
7542 (const_int 72057594037927935)) ;; 0x00ff...
7543 (and:DI (match_operand:DI 2 "register_operand" "rk")
7544 (const_int 72057594037927935))))] ;; 0x00ff...
7545 "TARGET_MEMTAG"
7546 "subp\\t%0, %1, %2"
7547 [(set_attr "type" "memtag")]
7548)
7549
7550;; LDG will use the 16-byte aligned value of the address.
7551(define_insn "ldg"
7552 [(set (match_operand:DI 0 "register_operand" "+r")
7553 (ior:DI
7554 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff...
7555 (ashift:DI
7556 (mem:QI (unspec:DI
7557 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7558 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))
7559 (const_int -16))] UNSPEC_TAG_SPACE))
7560 (const_int 56))))]
7561 "TARGET_MEMTAG"
7562 "ldg\\t%0, [%1, #%2]"
7563 [(set_attr "type" "memtag")]
7564)
7565
7566;; STG doesn't align the address but aborts with alignment fault
7567;; when the address is not 16-byte aligned.
7568(define_insn "stg"
7569 [(set (mem:QI (unspec:DI
7570 [(plus:DI (match_operand:DI 1 "register_operand" "rk")
7571 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))]
7572 UNSPEC_TAG_SPACE))
7573 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk")
7574 (const_int 56)) (const_int 15)))]
7575 "TARGET_MEMTAG"
7576 "stg\\t%0, [%1, #%2]"
7577 [(set_attr "type" "memtag")]
7578)
7579
fdcddba8
PW		 7580;; Load/Store 64-bit (LS64) instructions.
7581(define_insn "ld64b"
7582 [(set (match_operand:V8DI 0 "register_operand" "=r")
7583 (unspec_volatile:V8DI
7584 [(mem:V8DI (match_operand:DI 1 "register_operand" "r"))]
7585 UNSPEC_LD64B)
7586 )]
7587 "TARGET_LS64"
7588 "ld64b\\t%0, [%1]"
7589 [(set_attr "type" "ls64")]
7590)
7591
7592(define_insn "st64b"
7593 [(set (mem:V8DI (match_operand:DI 0 "register_operand" "=r"))
7594 (unspec_volatile:V8DI [(match_operand:V8DI 1 "register_operand" "r")]
7595 UNSPEC_ST64B)
7596 )]
7597 "TARGET_LS64"
7598 "st64b\\t%1, [%0]"
7599 [(set_attr "type" "ls64")]
7600)
7601
7602(define_insn "st64bv"
7603 [(set (match_operand:DI 0 "register_operand" "=r")
7604 (unspec_volatile:DI [(const_int 0)] UNSPEC_ST64BV_RET))
7605 (set (mem:V8DI (match_operand:DI 1 "register_operand" "r"))
7606 (unspec_volatile:V8DI [(match_operand:V8DI 2 "register_operand" "r")]
7607 UNSPEC_ST64BV)
7608 )]
7609 "TARGET_LS64"
7610 "st64bv\\t%0, %2, [%1]"
7611 [(set_attr "type" "ls64")]
7612)
7613
7614(define_insn "st64bv0"
7615 [(set (match_operand:DI 0 "register_operand" "=r")
7616 (unspec_volatile:DI [(const_int 0)] UNSPEC_ST64BV0_RET))
7617 (set (mem:V8DI (match_operand:DI 1 "register_operand" "r"))
7618 (unspec_volatile:V8DI [(match_operand:V8DI 2 "register_operand" "r")]
7619 UNSPEC_ST64BV0)
7620 )]
7621 "TARGET_LS64"
7622 "st64bv0\\t%0, %2, [%1]"
7623 [(set_attr "type" "ls64")]
7624)
7625
43e9d192
IB		 7626;; AdvSIMD Stuff
7627(include "aarch64-simd.md")
7628
0462169c
SN		 7629;; Atomic Operations
7630(include "atomics.md")
350013bc
BC		 7631
7632;; ldp/stp peephole patterns
7633(include "aarch64-ldpstp.md")
43cacb12
RS		 7634
7635;; SVE.
7636(include "aarch64-sve.md")
0617e23c
AM		 7637
7638;; SVE2.
7639(include "aarch64-sve2.md")
Mirror of https://gcc.gnu.org/git/gcc.git