1 ;; Machine description for AArch64 SVE.
2 ;; Copyright (C) 2009-2020 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; The file is organised into the following sections (search for the full
25 ;; ---- Note on the handling of big-endian SVE
26 ;; ---- Description of UNSPEC_PTEST
27 ;; ---- Description of UNSPEC_PRED_Z
28 ;; ---- Note on predicated integer arithemtic and UNSPEC_PRED_X
29 ;; ---- Note on predicated FP arithmetic patterns and GP "strictness"
30 ;; ---- Note on FFR handling
33 ;; ---- Moves of single vectors
34 ;; ---- Moves of multiple vectors
35 ;; ---- Moves of predicates
36 ;; ---- Moves relating to the FFR
39 ;; ---- Normal contiguous loads
40 ;; ---- Extending contiguous loads
41 ;; ---- First-faulting contiguous loads
42 ;; ---- First-faulting extending contiguous loads
43 ;; ---- Non-temporal contiguous loads
44 ;; ---- Normal gather loads
45 ;; ---- Extending gather loads
46 ;; ---- First-faulting gather loads
47 ;; ---- First-faulting extending gather loads
50 ;; ---- Contiguous prefetches
51 ;; ---- Gather prefetches
54 ;; ---- Normal contiguous stores
55 ;; ---- Truncating contiguous stores
56 ;; ---- Non-temporal contiguous stores
57 ;; ---- Normal scatter stores
58 ;; ---- Truncating scatter stores
61 ;; ---- [INT,FP] Duplicate element
62 ;; ---- [INT,FP] Initialize from individual elements
63 ;; ---- [INT] Linear series
64 ;; ---- [PRED] Duplicate element
66 ;; == Vector decomposition
67 ;; ---- [INT,FP] Extract index
68 ;; ---- [INT,FP] Extract active element
69 ;; ---- [PRED] Extract index
71 ;; == Unary arithmetic
72 ;; ---- [INT] General unary arithmetic corresponding to rtx codes
73 ;; ---- [INT] General unary arithmetic corresponding to unspecs
74 ;; ---- [INT] Sign and zero extension
75 ;; ---- [INT] Truncation
76 ;; ---- [INT] Logical inverse
77 ;; ---- [FP<-INT] General unary arithmetic that maps to unspecs
78 ;; ---- [FP] General unary arithmetic corresponding to unspecs
79 ;; ---- [PRED] Inverse
81 ;; == Binary arithmetic
82 ;; ---- [INT] General binary arithmetic corresponding to rtx codes
83 ;; ---- [INT] Addition
84 ;; ---- [INT] Subtraction
85 ;; ---- [INT] Take address
86 ;; ---- [INT] Absolute difference
87 ;; ---- [INT] Saturating addition and subtraction
88 ;; ---- [INT] Highpart multiplication
89 ;; ---- [INT] Division
90 ;; ---- [INT] Binary logical operations
91 ;; ---- [INT] Binary logical operations (inverted second input)
92 ;; ---- [INT] Shifts (rounding towards -Inf)
93 ;; ---- [INT] Shifts (rounding towards 0)
94 ;; ---- [FP<-INT] General binary arithmetic corresponding to unspecs
95 ;; ---- [FP] General binary arithmetic corresponding to rtx codes
96 ;; ---- [FP] General binary arithmetic corresponding to unspecs
98 ;; ---- [FP] Complex addition
99 ;; ---- [FP] Subtraction
100 ;; ---- [FP] Absolute difference
101 ;; ---- [FP] Multiplication
102 ;; ---- [FP] Binary logical operations
103 ;; ---- [FP] Sign copying
104 ;; ---- [FP] Maximum and minimum
105 ;; ---- [PRED] Binary logical operations
106 ;; ---- [PRED] Binary logical operations (inverted second input)
107 ;; ---- [PRED] Binary logical operations (inverted result)
109 ;; == Ternary arithmetic
110 ;; ---- [INT] MLA and MAD
111 ;; ---- [INT] MLS and MSB
112 ;; ---- [INT] Dot product
113 ;; ---- [INT] Sum of absolute differences
114 ;; ---- [FP] General ternary arithmetic corresponding to unspecs
115 ;; ---- [FP] Complex multiply-add
116 ;; ---- [FP] Trigonometric multiply-add
118 ;; == Comparisons and selects
119 ;; ---- [INT,FP] Select based on predicates
120 ;; ---- [INT,FP] Compare and select
121 ;; ---- [INT] Comparisons
122 ;; ---- [INT] While tests
123 ;; ---- [FP] Direct comparisons
124 ;; ---- [FP] Absolute comparisons
125 ;; ---- [PRED] Select
126 ;; ---- [PRED] Test bits
129 ;; ---- [INT,FP] Conditional reductions
130 ;; ---- [INT] Tree reductions
131 ;; ---- [FP] Tree reductions
132 ;; ---- [FP] Left-to-right reductions
135 ;; ---- [INT,FP] General permutes
136 ;; ---- [INT,FP] Special-purpose unary permutes
137 ;; ---- [INT,FP] Special-purpose binary permutes
138 ;; ---- [PRED] Special-purpose unary permutes
139 ;; ---- [PRED] Special-purpose binary permutes
142 ;; ---- [INT<-INT] Packs
143 ;; ---- [INT<-INT] Unpacks
144 ;; ---- [INT<-FP] Conversions
145 ;; ---- [INT<-FP] Packs
146 ;; ---- [INT<-FP] Unpacks
147 ;; ---- [FP<-INT] Conversions
148 ;; ---- [FP<-INT] Packs
149 ;; ---- [FP<-INT] Unpacks
150 ;; ---- [FP<-FP] Packs
151 ;; ---- [FP<-FP] Unpacks
152 ;; ---- [PRED<-PRED] Packs
153 ;; ---- [PRED<-PRED] Unpacks
155 ;; == Vector partitioning
156 ;; ---- [PRED] Unary partitioning
157 ;; ---- [PRED] Binary partitioning
158 ;; ---- [PRED] Scalarization
160 ;; == Counting elements
161 ;; ---- [INT] Count elements in a pattern (scalar)
162 ;; ---- [INT] Increment by the number of elements in a pattern (scalar)
163 ;; ---- [INT] Increment by the number of elements in a pattern (vector)
164 ;; ---- [INT] Decrement by the number of elements in a pattern (scalar)
165 ;; ---- [INT] Decrement by the number of elements in a pattern (vector)
166 ;; ---- [INT] Count elements in a predicate (scalar)
167 ;; ---- [INT] Increment by the number of elements in a predicate (scalar)
168 ;; ---- [INT] Increment by the number of elements in a predicate (vector)
169 ;; ---- [INT] Decrement by the number of elements in a predicate (scalar)
170 ;; ---- [INT] Decrement by the number of elements in a predicate (vector)
172 ;; =========================================================================
174 ;; =========================================================================
176 ;; -------------------------------------------------------------------------
177 ;; ---- Note on the handling of big-endian SVE
178 ;; -------------------------------------------------------------------------
180 ;; On big-endian systems, Advanced SIMD mov<mode> patterns act in the
181 ;; same way as movdi or movti would: the first byte of memory goes
182 ;; into the most significant byte of the register and the last byte
183 ;; of memory goes into the least significant byte of the register.
184 ;; This is the most natural ordering for Advanced SIMD and matches
185 ;; the ABI layout for 64-bit and 128-bit vector types.
187 ;; As a result, the order of bytes within the register is what GCC
188 ;; expects for a big-endian target, and subreg offsets therefore work
189 ;; as expected, with the first element in memory having subreg offset 0
190 ;; and the last element in memory having the subreg offset associated
191 ;; with a big-endian lowpart. However, this ordering also means that
192 ;; GCC's lane numbering does not match the architecture's numbering:
193 ;; GCC always treats the element at the lowest address in memory
194 ;; (subreg offset 0) as element 0, while the architecture treats
195 ;; the least significant end of the register as element 0.
197 ;; The situation for SVE is different. We want the layout of the
198 ;; SVE register to be same for mov<mode> as it is for maskload<mode>:
199 ;; logically, a mov<mode> load must be indistinguishable from a
200 ;; maskload<mode> whose mask is all true. We therefore need the
201 ;; register layout to match LD1 rather than LDR. The ABI layout of
202 ;; SVE types also matches LD1 byte ordering rather than LDR byte ordering.
204 ;; As a result, the architecture lane numbering matches GCC's lane
205 ;; numbering, with element 0 always being the first in memory.
208 ;; - Applying a subreg offset to a register does not give the element
209 ;; that GCC expects: the first element in memory has the subreg offset
210 ;; associated with a big-endian lowpart while the last element in memory
211 ;; has subreg offset 0. We handle this via TARGET_CAN_CHANGE_MODE_CLASS.
213 ;; - We cannot use LDR and STR for spill slots that might be accessed
214 ;; via subregs, since although the elements have the order GCC expects,
215 ;; the order of the bytes within the elements is different. We instead
216 ;; access spill slots via LD1 and ST1, using secondary reloads to
217 ;; reserve a predicate register.
219 ;; -------------------------------------------------------------------------
220 ;; ---- Description of UNSPEC_PTEST
221 ;; -------------------------------------------------------------------------
223 ;; SVE provides a PTEST instruction for testing the active lanes of a
224 ;; predicate and setting the flags based on the result. The associated
225 ;; condition code tests are:
227 ;; - any (= ne): at least one active bit is set
228 ;; - none (= eq): all active bits are clear (*)
229 ;; - first (= mi): the first active bit is set
230 ;; - nfrst (= pl): the first active bit is clear (*)
231 ;; - last (= cc): the last active bit is set
232 ;; - nlast (= cs): the last active bit is clear (*)
234 ;; where the conditions marked (*) are also true when there are no active
235 ;; lanes (i.e. when the governing predicate is a PFALSE). The flags results
236 ;; of a PTEST use the condition code mode CC_NZC.
238 ;; PTEST is always a .B operation (i.e. it always operates on VNx16BI).
239 ;; This means that for other predicate modes, we need a governing predicate
240 ;; in which all bits are defined.
242 ;; For example, most predicated .H operations ignore the odd bits of the
243 ;; governing predicate, so that an active lane is represented by the
244 ;; bits "1x" and an inactive lane by the bits "0x", where "x" can be
245 ;; any value. To test a .H predicate, we instead need "10" and "00"
246 ;; respectively, so that the condition only tests the even bits of the
249 ;; Several instructions set the flags as a side-effect, in the same way
250 ;; that a separate PTEST would. It's important for code quality that we
251 ;; use these flags results as often as possible, particularly in the case
252 ;; of WHILE* and RDFFR.
254 ;; Also, some of the instructions that set the flags are unpredicated
255 ;; and instead implicitly test all .B, .H, .S or .D elements, as though
256 ;; they were predicated on a PTRUE of that size. For example, a .S
257 ;; WHILELO sets the flags in the same way as a PTEST with a .S PTRUE
260 ;; We therefore need to represent PTEST operations in a way that
261 ;; makes it easy to combine them with both predicated and unpredicated
262 ;; operations, while using a VNx16BI governing predicate for all
263 ;; predicate modes. We do this using:
265 ;; (unspec:CC_NZC [gp cast_gp ptrue_flag op] UNSPEC_PTEST)
269 ;; - GP is the real VNx16BI governing predicate
271 ;; - CAST_GP is GP cast to the mode of OP. All bits dropped by casting
272 ;; GP to CAST_GP are guaranteed to be clear in GP.
274 ;; - PTRUE_FLAG is a CONST_INT (conceptually of mode SI) that has the value
275 ;; SVE_KNOWN_PTRUE if we know that CAST_GP (rather than GP) is all-true and
276 ;; SVE_MAYBE_NOT_PTRUE otherwise.
278 ;; - OP is the predicate we want to test, of the same mode as CAST_GP.
280 ;; -------------------------------------------------------------------------
281 ;; ---- Description of UNSPEC_PRED_Z
282 ;; -------------------------------------------------------------------------
284 ;; SVE integer comparisons are predicated and return zero for inactive
285 ;; lanes. Sometimes we use them with predicates that are all-true and
286 ;; sometimes we use them with general predicates.
288 ;; The integer comparisons also set the flags and so build-in the effect
289 ;; of a PTEST. We therefore want to be able to combine integer comparison
290 ;; patterns with PTESTs of the result. One difficulty with doing this is
291 ;; that (as noted above) the PTEST is always a .B operation and so can place
292 ;; stronger requirements on the governing predicate than the comparison does.
294 ;; For example, when applying a separate PTEST to the result of a full-vector
295 ;; .H comparison, the PTEST must be predicated on a .H PTRUE instead of a
296 ;; .B PTRUE. In constrast, the comparison might be predicated on either
297 ;; a .H PTRUE or a .B PTRUE, since the values of odd-indexed predicate
298 ;; bits don't matter for .H operations.
300 ;; We therefore can't rely on a full-vector comparison using the same
301 ;; predicate register as a following PTEST. We instead need to remember
302 ;; whether a comparison is known to be a full-vector comparison and use
303 ;; this information in addition to a check for equal predicate registers.
304 ;; At the same time, it's useful to have a common representation for all
305 ;; integer comparisons, so that they can be handled by a single set of
308 ;; We therefore take a similar approach to UNSPEC_PTEST above and use:
310 ;; (unspec:<M:VPRED> [gp ptrue_flag (code:M op0 op1)] UNSPEC_PRED_Z)
314 ;; - GP is the governing predicate, of mode <M:VPRED>
316 ;; - PTRUE_FLAG is a CONST_INT (conceptually of mode SI) that has the value
317 ;; SVE_KNOWN_PTRUE if we know that GP is all-true and SVE_MAYBE_NOT_PTRUE
320 ;; - CODE is the comparison code
322 ;; - OP0 and OP1 are the values being compared, of mode M
324 ;; The "Z" in UNSPEC_PRED_Z indicates that inactive lanes are zero.
326 ;; -------------------------------------------------------------------------
327 ;; ---- Note on predicated integer arithemtic and UNSPEC_PRED_X
328 ;; -------------------------------------------------------------------------
330 ;; Many SVE integer operations are predicated. We can generate them
331 ;; from four sources:
333 ;; (1) Using normal unpredicated optabs. In this case we need to create
334 ;; an all-true predicate register to act as the governing predicate
335 ;; for the SVE instruction. There are no inactive lanes, and thus
336 ;; the values of inactive lanes don't matter.
338 ;; (2) Using _x ACLE functions. In this case the function provides a
339 ;; specific predicate and some lanes might be inactive. However,
340 ;; as for (1), the values of the inactive lanes don't matter.
341 ;; We can make extra lanes active without changing the behavior
342 ;; (although for code-quality reasons we should avoid doing so
345 ;; (3) Using cond_* optabs that correspond to IFN_COND_* internal functions.
346 ;; These optabs have a predicate operand that specifies which lanes are
347 ;; active and another operand that provides the values of inactive lanes.
349 ;; (4) Using _m and _z ACLE functions. These functions map to the same
350 ;; patterns as (3), with the _z functions setting inactive lanes to zero
351 ;; and the _m functions setting the inactive lanes to one of the function
354 ;; For (1) and (2) we need a way of attaching the predicate to a normal
355 ;; unpredicated integer operation. We do this using:
357 ;; (unspec:M [pred (code:M (op0 op1 ...))] UNSPEC_PRED_X)
359 ;; where (code:M (op0 op1 ...)) is the normal integer operation and PRED
360 ;; is a predicate of mode <M:VPRED>. PRED might or might not be a PTRUE;
361 ;; it always is for (1), but might not be for (2).
363 ;; The unspec as a whole has the same value as (code:M ...) when PRED is
364 ;; all-true. It is always semantically valid to replace PRED with a PTRUE,
365 ;; but as noted above, we should only do so if there's a specific benefit.
367 ;; (The "_X" in the unspec is named after the ACLE functions in (2).)
369 ;; For (3) and (4) we can simply use the SVE port's normal representation
370 ;; of a predicate-based select:
372 ;; (unspec:M [pred (code:M (op0 op1 ...)) inactive] UNSPEC_SEL)
374 ;; where INACTIVE specifies the values of inactive lanes.
376 ;; We can also use the UNSPEC_PRED_X wrapper in the UNSPEC_SEL rather
377 ;; than inserting the integer operation directly. This is mostly useful
378 ;; if we want the combine pass to merge an integer operation with an explicit
379 ;; vcond_mask (in other words, with a following SEL instruction). However,
380 ;; it's generally better to merge such operations at the gimple level
383 ;; -------------------------------------------------------------------------
384 ;; ---- Note on predicated FP arithmetic patterns and GP "strictness"
385 ;; -------------------------------------------------------------------------
387 ;; Most SVE floating-point operations are predicated. We can generate
388 ;; them from four sources:
390 ;; (1) Using normal unpredicated optabs. In this case we need to create
391 ;; an all-true predicate register to act as the governing predicate
392 ;; for the SVE instruction. There are no inactive lanes, and thus
393 ;; the values of inactive lanes don't matter.
395 ;; (2) Using _x ACLE functions. In this case the function provides a
396 ;; specific predicate and some lanes might be inactive. However,
397 ;; as for (1), the values of the inactive lanes don't matter.
399 ;; The instruction must have the same exception behavior as the
400 ;; function call unless things like command-line flags specifically
401 ;; allow otherwise. For example, with -ffast-math, it is OK to
402 ;; raise exceptions for inactive lanes, but normally it isn't.
404 ;; (3) Using cond_* optabs that correspond to IFN_COND_* internal functions.
405 ;; These optabs have a predicate operand that specifies which lanes are
406 ;; active and another operand that provides the values of inactive lanes.
408 ;; (4) Using _m and _z ACLE functions. These functions map to the same
409 ;; patterns as (3), with the _z functions setting inactive lanes to zero
410 ;; and the _m functions setting the inactive lanes to one of the function
415 ;; - In (1), the predicate is known to be all true and the pattern can use
416 ;; unpredicated operations where available.
418 ;; - In (2), the predicate might or might not be all true. The pattern can
419 ;; use unpredicated instructions if the predicate is all-true or if things
420 ;; like command-line flags allow exceptions for inactive lanes.
422 ;; - (3) and (4) represent a native SVE predicated operation. Some lanes
423 ;; might be inactive and inactive lanes of the result must have specific
424 ;; values. There is no scope for using unpredicated instructions (and no
425 ;; reason to want to), so the question about command-line flags doesn't
428 ;; It would be inaccurate to model (2) as an rtx code like (sqrt ...)
429 ;; in combination with a separate predicate operand, e.g.
431 ;; (unspec [(match_operand:<VPRED> 1 "register_operand" "Upl")
432 ;; (sqrt:SVE_FULL_F 2 "register_operand" "w")]
435 ;; because (sqrt ...) can raise an exception for any lane, including
436 ;; inactive ones. We therefore need to use an unspec instead.
438 ;; Also, (2) requires some way of distinguishing the case in which the
439 ;; predicate might have inactive lanes and cannot be changed from the
440 ;; case in which the predicate has no inactive lanes or can be changed.
441 ;; This information is also useful when matching combined FP patterns
442 ;; in which the predicates might not be equal.
444 ;; We therefore model FP operations as an unspec of the form:
446 ;; (unspec [pred strictness op0 op1 ...] UNSPEC_COND_<MNEMONIC>)
450 ;; - PRED is the governing predicate.
452 ;; - STRICTNESS is a CONST_INT that conceptually has mode SI. It has the
453 ;; value SVE_STRICT_GP if PRED might have inactive lanes and if those
454 ;; lanes must remain inactive. It has the value SVE_RELAXED_GP otherwise.
456 ;; - OP0 OP1 ... are the normal input operands to the operation.
458 ;; - MNEMONIC is the mnemonic of the associated SVE instruction.
460 ;; -------------------------------------------------------------------------
461 ;; ---- Note on FFR handling
462 ;; -------------------------------------------------------------------------
464 ;; Logically we want to divide FFR-related instructions into regions
465 ;; that contain exactly one of:
467 ;; - a single write to the FFR
468 ;; - any number of reads from the FFR (but only one read is likely)
469 ;; - any number of LDFF1 and LDNF1 instructions
471 ;; However, LDFF1 and LDNF1 instructions should otherwise behave like
472 ;; normal loads as far as possible. This means that they should be
473 ;; schedulable within a region in the same way that LD1 would be,
474 ;; and they should be deleted as dead if the result is unused. The loads
475 ;; should therefore not write to the FFR, since that would both serialize
476 ;; the loads with respect to each other and keep the loads live for any
479 ;; We get around this by using a fake "FFR token" (FFRT) to help describe
480 ;; the dependencies. Writing to the FFRT starts a new "FFRT region",
481 ;; while using the FFRT keeps the instruction within its region.
484 ;; - Writes start a new FFRT region as well as setting the FFR:
486 ;; W1: parallel (FFRT = <new value>, FFR = <actual FFR value>)
488 ;; - Loads use an LD1-like instruction that also uses the FFRT, so that the
489 ;; loads stay within the same FFRT region:
491 ;; L1: load data while using the FFRT
493 ;; In addition, any FFRT region that includes a load also has at least one
496 ;; L2: FFR = update(FFR, FFRT) [type == no_insn]
498 ;; to make it clear that the region both reads from and writes to the FFR.
500 ;; - Reads do the following:
502 ;; R1: FFRT = FFR [type == no_insn]
503 ;; R2: read from the FFRT
504 ;; R3: FFRT = update(FFRT) [type == no_insn]
506 ;; R1 and R3 both create new FFRT regions, so that previous LDFF1s and
507 ;; LDNF1s cannot move forwards across R1 and later LDFF1s and LDNF1s
508 ;; cannot move backwards across R3.
510 ;; This way, writes are only kept alive by later loads or reads,
511 ;; and write/read pairs fold normally. For two consecutive reads,
512 ;; the first R3 is made dead by the second R1, which in turn becomes
513 ;; redundant with the first R1. We then have:
515 ;; first R1: FFRT = FFR
516 ;; first read from the FFRT
517 ;; second read from the FFRT
518 ;; second R3: FFRT = update(FFRT)
520 ;; i.e. the two FFRT regions collapse into a single one with two
521 ;; independent reads.
523 ;; The model still prevents some valid optimizations though. For example,
524 ;; if all loads in an FFRT region are deleted as dead, nothing would remove
525 ;; the L2 instructions.
527 ;; =========================================================================
529 ;; =========================================================================
531 ;; -------------------------------------------------------------------------
532 ;; ---- Moves of single vectors
533 ;; -------------------------------------------------------------------------
535 ;; - MOV (including aliases)
536 ;; - LD1B (contiguous form)
541 ;; - ST1B (contiguous form)
546 ;; -------------------------------------------------------------------------
548 (define_expand "mov<mode>"
549 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
550 (match_operand:SVE_ALL 1 "general_operand"))]
553 /* Use the predicated load and store patterns where possible.
554 This is required for big-endian targets (see the comment at the
555 head of the file) and increases the addressing choices for
557 if ((MEM_P (operands[0]) || MEM_P (operands[1]))
558 && can_create_pseudo_p ())
560 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
564 if (CONSTANT_P (operands[1]))
566 aarch64_expand_mov_immediate (operands[0], operands[1]);
570 /* Optimize subregs on big-endian targets: we can use REV[BHW]
571 instead of going through memory. */
573 && aarch64_maybe_expand_sve_subreg_move (operands[0], operands[1]))
578 (define_expand "movmisalign<mode>"
579 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
580 (match_operand:SVE_ALL 1 "general_operand"))]
583 /* Equivalent to a normal move for our purpooses. */
584 emit_move_insn (operands[0], operands[1]);
589 ;; Unpredicated moves that can use LDR and STR, i.e. full vectors for which
590 ;; little-endian ordering is acceptable. Only allow memory operations during
591 ;; and after RA; before RA we want the predicated load and store patterns to
593 (define_insn "*aarch64_sve_mov<mode>_ldr_str"
594 [(set (match_operand:SVE_FULL 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
595 (match_operand:SVE_FULL 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
597 && (<MODE>mode == VNx16QImode || !BYTES_BIG_ENDIAN)
598 && ((lra_in_progress || reload_completed)
599 || (register_operand (operands[0], <MODE>mode)
600 && nonmemory_operand (operands[1], <MODE>mode)))"
605 * return aarch64_output_sve_mov_immediate (operands[1]);"
608 ;; Unpredicated moves that cannot use LDR and STR, i.e. partial vectors
609 ;; or vectors for which little-endian ordering isn't acceptable. Memory
610 ;; accesses require secondary reloads.
611 (define_insn "*aarch64_sve_mov<mode>_no_ldr_str"
612 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w")
613 (match_operand:SVE_ALL 1 "aarch64_nonmemory_operand" "w, Dn"))]
615 && <MODE>mode != VNx16QImode
617 || maybe_ne (BYTES_PER_SVE_VECTOR, GET_MODE_SIZE (<MODE>mode)))"
620 * return aarch64_output_sve_mov_immediate (operands[1]);"
623 ;; Handle memory reloads for modes that can't use LDR and STR. We use
624 ;; byte PTRUE for all modes to try to encourage reuse. This pattern
625 ;; needs constraints because it is returned by TARGET_SECONDARY_RELOAD.
626 (define_expand "aarch64_sve_reload_mem"
628 [(set (match_operand 0)
630 (clobber (match_operand:VNx16BI 2 "register_operand" "=Upl"))])]
633 /* Create a PTRUE. */
634 emit_move_insn (operands[2], CONSTM1_RTX (VNx16BImode));
636 /* Refer to the PTRUE in the appropriate mode for this move. */
637 machine_mode mode = GET_MODE (operands[0]);
638 rtx pred = gen_lowpart (aarch64_sve_pred_mode (mode), operands[2]);
640 /* Emit a predicated load or store. */
641 aarch64_emit_sve_pred_move (operands[0], pred, operands[1]);
646 ;; A predicated move in which the predicate is known to be all-true.
647 ;; Note that this pattern is generated directly by aarch64_emit_sve_pred_move,
648 ;; so changes to this pattern will need changes there as well.
649 (define_insn_and_split "@aarch64_pred_mov<mode>"
650 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand" "=w, w, m")
652 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
653 (match_operand:SVE_ALL 2 "nonimmediate_operand" "w, m, w")]
656 && (register_operand (operands[0], <MODE>mode)
657 || register_operand (operands[2], <MODE>mode))"
660 ld1<Vesize>\t%0.<Vctype>, %1/z, %2
661 st1<Vesize>\t%2.<Vctype>, %1, %0"
662 "&& register_operand (operands[0], <MODE>mode)
663 && register_operand (operands[2], <MODE>mode)"
664 [(set (match_dup 0) (match_dup 2))]
667 ;; A pattern for optimizing SUBREGs that have a reinterpreting effect
668 ;; on big-endian targets; see aarch64_maybe_expand_sve_subreg_move
669 ;; for details. We use a special predicate for operand 2 to reduce
670 ;; the number of patterns.
671 (define_insn_and_split "*aarch64_sve_mov<mode>_subreg_be"
672 [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w")
674 [(match_operand:VNx16BI 1 "register_operand" "Upl")
675 (match_operand 2 "aarch64_any_register_operand" "w")]
677 "TARGET_SVE && BYTES_BIG_ENDIAN"
679 "&& reload_completed"
682 aarch64_split_sve_subreg_move (operands[0], operands[1], operands[2]);
687 ;; Reinterpret operand 1 in operand 0's mode, without changing its contents.
688 ;; This is equivalent to a subreg on little-endian targets but not for
689 ;; big-endian; see the comment at the head of the file for details.
690 (define_expand "@aarch64_sve_reinterpret<mode>"
691 [(set (match_operand:SVE_ALL 0 "register_operand")
693 [(match_operand 1 "aarch64_any_register_operand")]
694 UNSPEC_REINTERPRET))]
697 machine_mode src_mode = GET_MODE (operands[1]);
698 if (targetm.can_change_mode_class (<MODE>mode, src_mode, FP_REGS))
700 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, operands[1]));
706 ;; A pattern for handling type punning on big-endian targets. We use a
707 ;; special predicate for operand 1 to reduce the number of patterns.
708 (define_insn_and_split "*aarch64_sve_reinterpret<mode>"
709 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
711 [(match_operand 1 "aarch64_any_register_operand" "w")]
712 UNSPEC_REINTERPRET))]
715 "&& reload_completed"
716 [(set (match_dup 0) (match_dup 1))]
718 operands[1] = aarch64_replace_reg_mode (operands[1], <MODE>mode);
722 ;; -------------------------------------------------------------------------
723 ;; ---- Moves of multiple vectors
724 ;; -------------------------------------------------------------------------
725 ;; All patterns in this section are synthetic and split to real
726 ;; instructions after reload.
727 ;; -------------------------------------------------------------------------
729 (define_expand "mov<mode>"
730 [(set (match_operand:SVE_STRUCT 0 "nonimmediate_operand")
731 (match_operand:SVE_STRUCT 1 "general_operand"))]
734 /* Big-endian loads and stores need to be done via LD1 and ST1;
735 see the comment at the head of the file for details. */
736 if ((MEM_P (operands[0]) || MEM_P (operands[1]))
739 gcc_assert (can_create_pseudo_p ());
740 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
744 if (CONSTANT_P (operands[1]))
746 aarch64_expand_mov_immediate (operands[0], operands[1]);
752 ;; Unpredicated structure moves (little-endian).
753 (define_insn "*aarch64_sve_mov<mode>_le"
754 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
755 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
756 "TARGET_SVE && !BYTES_BIG_ENDIAN"
758 [(set_attr "length" "<insn_length>")]
761 ;; Unpredicated structure moves (big-endian). Memory accesses require
762 ;; secondary reloads.
763 (define_insn "*aarch64_sve_mov<mode>_be"
764 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w, w")
765 (match_operand:SVE_STRUCT 1 "aarch64_nonmemory_operand" "w, Dn"))]
766 "TARGET_SVE && BYTES_BIG_ENDIAN"
768 [(set_attr "length" "<insn_length>")]
771 ;; Split unpredicated structure moves into pieces. This is the same
772 ;; for both big-endian and little-endian code, although it only needs
773 ;; to handle memory operands for little-endian code.
775 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand")
776 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand"))]
777 "TARGET_SVE && reload_completed"
780 rtx dest = operands[0];
781 rtx src = operands[1];
782 if (REG_P (dest) && REG_P (src))
783 aarch64_simd_emit_reg_reg_move (operands, <VSINGLE>mode, <vector_count>);
785 for (unsigned int i = 0; i < <vector_count>; ++i)
787 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, dest, <MODE>mode,
788 i * BYTES_PER_SVE_VECTOR);
789 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, src, <MODE>mode,
790 i * BYTES_PER_SVE_VECTOR);
791 emit_insn (gen_rtx_SET (subdest, subsrc));
797 ;; Predicated structure moves. This works for both endiannesses but in
798 ;; practice is only useful for big-endian.
799 (define_insn_and_split "@aarch64_pred_mov<mode>"
800 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_struct_nonimmediate_operand" "=w, w, Utx")
802 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
803 (match_operand:SVE_STRUCT 2 "aarch64_sve_struct_nonimmediate_operand" "w, Utx, w")]
806 && (register_operand (operands[0], <MODE>mode)
807 || register_operand (operands[2], <MODE>mode))"
809 "&& reload_completed"
812 for (unsigned int i = 0; i < <vector_count>; ++i)
814 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, operands[0],
816 i * BYTES_PER_SVE_VECTOR);
817 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, operands[2],
819 i * BYTES_PER_SVE_VECTOR);
820 aarch64_emit_sve_pred_move (subdest, operands[1], subsrc);
824 [(set_attr "length" "<insn_length>")]
827 ;; -------------------------------------------------------------------------
828 ;; ---- Moves of predicates
829 ;; -------------------------------------------------------------------------
837 ;; -------------------------------------------------------------------------
839 (define_expand "mov<mode>"
840 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand")
841 (match_operand:PRED_ALL 1 "general_operand"))]
844 if (GET_CODE (operands[0]) == MEM)
845 operands[1] = force_reg (<MODE>mode, operands[1]);
847 if (CONSTANT_P (operands[1]))
849 aarch64_expand_mov_immediate (operands[0], operands[1]);
855 (define_insn "*aarch64_sve_mov<mode>"
856 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand" "=Upa, m, Upa, Upa")
857 (match_operand:PRED_ALL 1 "aarch64_mov_operand" "Upa, Upa, m, Dn"))]
859 && (register_operand (operands[0], <MODE>mode)
860 || register_operand (operands[1], <MODE>mode))"
865 * return aarch64_output_sve_mov_immediate (operands[1]);"
868 ;; Match PTRUES Pn.B when both the predicate and flags are useful.
869 (define_insn_and_rewrite "*aarch64_sve_ptruevnx16bi_cc"
870 [(set (reg:CC_NZC CC_REGNUM)
874 (const_int SVE_KNOWN_PTRUE)
875 (match_operator:VNx16BI 1 "aarch64_sve_ptrue_svpattern_immediate"
877 [(match_operand:SI 4 "const_int_operand")
878 (match_operand:VNx16BI 5 "aarch64_simd_imm_zero")]
881 (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
885 return aarch64_output_sve_ptrues (operands[1]);
887 "&& (!CONSTANT_P (operands[2]) || !CONSTANT_P (operands[3]))"
889 operands[2] = operands[3] = CONSTM1_RTX (VNx16BImode);
893 ;; Match PTRUES Pn.[HSD] when both the predicate and flags are useful.
894 (define_insn_and_rewrite "*aarch64_sve_ptrue<mode>_cc"
895 [(set (reg:CC_NZC CC_REGNUM)
899 (const_int SVE_KNOWN_PTRUE)
901 (match_operator:VNx16BI 1 "aarch64_sve_ptrue_svpattern_immediate"
903 [(match_operand:SI 4 "const_int_operand")
904 (match_operand:PRED_HSD 5 "aarch64_simd_imm_zero")]
907 (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
911 return aarch64_output_sve_ptrues (operands[1]);
913 "&& (!CONSTANT_P (operands[2]) || !CONSTANT_P (operands[3]))"
915 operands[2] = CONSTM1_RTX (VNx16BImode);
916 operands[3] = CONSTM1_RTX (<MODE>mode);
920 ;; Match PTRUES Pn.B when only the flags result is useful (which is
921 ;; a way of testing VL).
922 (define_insn_and_rewrite "*aarch64_sve_ptruevnx16bi_ptest"
923 [(set (reg:CC_NZC CC_REGNUM)
927 (const_int SVE_KNOWN_PTRUE)
928 (match_operator:VNx16BI 1 "aarch64_sve_ptrue_svpattern_immediate"
930 [(match_operand:SI 4 "const_int_operand")
931 (match_operand:VNx16BI 5 "aarch64_simd_imm_zero")]
934 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
937 return aarch64_output_sve_ptrues (operands[1]);
939 "&& (!CONSTANT_P (operands[2]) || !CONSTANT_P (operands[3]))"
941 operands[2] = operands[3] = CONSTM1_RTX (VNx16BImode);
945 ;; Match PTRUES Pn.[HWD] when only the flags result is useful (which is
946 ;; a way of testing VL).
947 (define_insn_and_rewrite "*aarch64_sve_ptrue<mode>_ptest"
948 [(set (reg:CC_NZC CC_REGNUM)
952 (const_int SVE_KNOWN_PTRUE)
954 (match_operator:VNx16BI 1 "aarch64_sve_ptrue_svpattern_immediate"
956 [(match_operand:SI 4 "const_int_operand")
957 (match_operand:PRED_HSD 5 "aarch64_simd_imm_zero")]
960 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
963 return aarch64_output_sve_ptrues (operands[1]);
965 "&& (!CONSTANT_P (operands[2]) || !CONSTANT_P (operands[3]))"
967 operands[2] = CONSTM1_RTX (VNx16BImode);
968 operands[3] = CONSTM1_RTX (<MODE>mode);
972 ;; -------------------------------------------------------------------------
973 ;; ---- Moves relating to the FFR
974 ;; -------------------------------------------------------------------------
979 ;; -------------------------------------------------------------------------
981 ;; [W1 in the block comment above about FFR handling]
983 ;; Write to the FFR and start a new FFRT scheduling region.
984 (define_insn "aarch64_wrffr"
985 [(set (reg:VNx16BI FFR_REGNUM)
986 (match_operand:VNx16BI 0 "aarch64_simd_reg_or_minus_one" "Dm, Upa"))
987 (set (reg:VNx16BI FFRT_REGNUM)
988 (unspec:VNx16BI [(match_dup 0)] UNSPEC_WRFFR))]
995 ;; [L2 in the block comment above about FFR handling]
997 ;; Introduce a read from and write to the FFR in the current FFRT region,
998 ;; so that the FFR value is live on entry to the region and so that the FFR
999 ;; value visibly changes within the region. This is used (possibly multiple
1000 ;; times) in an FFRT region that includes LDFF1 or LDNF1 instructions.
1001 (define_insn "aarch64_update_ffr_for_load"
1002 [(set (reg:VNx16BI FFR_REGNUM)
1003 (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)
1004 (reg:VNx16BI FFR_REGNUM)] UNSPEC_UPDATE_FFR))]
1007 [(set_attr "type" "no_insn")]
1010 ;; [R1 in the block comment above about FFR handling]
1012 ;; Notionally copy the FFR to the FFRT, so that the current FFR value
1013 ;; can be read from there by the RDFFR instructions below. This acts
1014 ;; as a scheduling barrier for earlier LDFF1 and LDNF1 instructions and
1015 ;; creates a natural dependency with earlier writes.
1016 (define_insn "aarch64_copy_ffr_to_ffrt"
1017 [(set (reg:VNx16BI FFRT_REGNUM)
1018 (reg:VNx16BI FFR_REGNUM))]
1021 [(set_attr "type" "no_insn")]
1024 ;; [R2 in the block comment above about FFR handling]
1026 ;; Read the FFR via the FFRT.
1027 (define_insn "aarch64_rdffr"
1028 [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
1029 (reg:VNx16BI FFRT_REGNUM))]
1034 ;; Likewise with zero predication.
1035 (define_insn "aarch64_rdffr_z"
1036 [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
1038 (reg:VNx16BI FFRT_REGNUM)
1039 (match_operand:VNx16BI 1 "register_operand" "Upa")))]
1044 ;; Read the FFR to test for a fault, without using the predicate result.
1045 (define_insn "*aarch64_rdffr_z_ptest"
1046 [(set (reg:CC_NZC CC_REGNUM)
1048 [(match_operand:VNx16BI 1 "register_operand" "Upa")
1050 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
1052 (reg:VNx16BI FFRT_REGNUM)
1055 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
1057 "rdffrs\t%0.b, %1/z"
1060 ;; Same for unpredicated RDFFR when tested with a known PTRUE.
1061 (define_insn "*aarch64_rdffr_ptest"
1062 [(set (reg:CC_NZC CC_REGNUM)
1064 [(match_operand:VNx16BI 1 "register_operand" "Upa")
1066 (const_int SVE_KNOWN_PTRUE)
1067 (reg:VNx16BI FFRT_REGNUM)]
1069 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
1071 "rdffrs\t%0.b, %1/z"
1074 ;; Read the FFR with zero predication and test the result.
1075 (define_insn "*aarch64_rdffr_z_cc"
1076 [(set (reg:CC_NZC CC_REGNUM)
1078 [(match_operand:VNx16BI 1 "register_operand" "Upa")
1080 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
1082 (reg:VNx16BI FFRT_REGNUM)
1085 (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
1087 (reg:VNx16BI FFRT_REGNUM)
1090 "rdffrs\t%0.b, %1/z"
1093 ;; Same for unpredicated RDFFR when tested with a known PTRUE.
1094 (define_insn "*aarch64_rdffr_cc"
1095 [(set (reg:CC_NZC CC_REGNUM)
1097 [(match_operand:VNx16BI 1 "register_operand" "Upa")
1099 (const_int SVE_KNOWN_PTRUE)
1100 (reg:VNx16BI FFRT_REGNUM)]
1102 (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
1103 (reg:VNx16BI FFRT_REGNUM))]
1105 "rdffrs\t%0.b, %1/z"
1108 ;; [R3 in the block comment above about FFR handling]
1110 ;; Arbitrarily update the FFRT after a read from the FFR. This acts as
1111 ;; a scheduling barrier for later LDFF1 and LDNF1 instructions.
1112 (define_insn "aarch64_update_ffrt"
1113 [(set (reg:VNx16BI FFRT_REGNUM)
1114 (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)] UNSPEC_UPDATE_FFRT))]
1117 [(set_attr "type" "no_insn")]
1120 ;; =========================================================================
1122 ;; =========================================================================
1124 ;; -------------------------------------------------------------------------
1125 ;; ---- Normal contiguous loads
1126 ;; -------------------------------------------------------------------------
1127 ;; Includes contiguous forms of:
1144 ;; -------------------------------------------------------------------------
1147 (define_insn "maskload<mode><vpred>"
1148 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
1150 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1151 (match_operand:SVE_ALL 1 "memory_operand" "m")]
1154 "ld1<Vesize>\t%0.<Vctype>, %2/z, %1"
1157 ;; Unpredicated LD[234].
1158 (define_expand "vec_load_lanes<mode><vsingle>"
1159 [(set (match_operand:SVE_STRUCT 0 "register_operand")
1162 (match_operand:SVE_STRUCT 1 "memory_operand")]
1166 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
1170 ;; Predicated LD[234].
1171 (define_insn "vec_mask_load_lanes<mode><vsingle>"
1172 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w")
1174 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1175 (match_operand:SVE_STRUCT 1 "memory_operand" "m")]
1178 "ld<vector_count><Vesize>\t%0, %2/z, %1"
1181 ;; -------------------------------------------------------------------------
1182 ;; ---- Extending contiguous loads
1183 ;; -------------------------------------------------------------------------
1184 ;; Includes contiguous forms of:
1191 ;; -------------------------------------------------------------------------
1193 ;; Predicated load and extend, with 8 elements per 128-bit block.
1194 (define_insn_and_rewrite "@aarch64_load_<ANY_EXTEND:optab><SVE_HSDI:mode><SVE_PARTIAL_I:mode>"
1195 [(set (match_operand:SVE_HSDI 0 "register_operand" "=w")
1197 [(match_operand:<SVE_HSDI:VPRED> 3 "general_operand" "UplDnm")
1198 (ANY_EXTEND:SVE_HSDI
1199 (unspec:SVE_PARTIAL_I
1200 [(match_operand:<SVE_PARTIAL_I:VPRED> 2 "register_operand" "Upl")
1201 (match_operand:SVE_PARTIAL_I 1 "memory_operand" "m")]
1204 "TARGET_SVE && (~<SVE_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
1205 "ld1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_HSDI:Vctype>, %2/z, %1"
1206 "&& !CONSTANT_P (operands[3])"
1208 operands[3] = CONSTM1_RTX (<SVE_HSDI:VPRED>mode);
1212 ;; -------------------------------------------------------------------------
1213 ;; ---- First-faulting contiguous loads
1214 ;; -------------------------------------------------------------------------
1215 ;; Includes contiguous forms of:
1224 ;; -------------------------------------------------------------------------
1226 ;; Contiguous non-extending first-faulting or non-faulting loads.
1227 (define_insn "@aarch64_ld<fn>f1<mode>"
1228 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
1230 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1231 (match_operand:SVE_FULL 1 "aarch64_sve_ld<fn>f1_operand" "Ut<fn>")
1232 (reg:VNx16BI FFRT_REGNUM)]
1235 "ld<fn>f1<Vesize>\t%0.<Vetype>, %2/z, %1"
1238 ;; -------------------------------------------------------------------------
1239 ;; ---- First-faulting extending contiguous loads
1240 ;; -------------------------------------------------------------------------
1241 ;; Includes contiguous forms of:
1254 ;; -------------------------------------------------------------------------
1256 ;; Predicated first-faulting or non-faulting load and extend.
1257 (define_insn_and_rewrite "@aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SVE_PARTIAL_I:mode>"
1258 [(set (match_operand:SVE_HSDI 0 "register_operand" "=w")
1260 [(match_operand:<SVE_HSDI:VPRED> 3 "general_operand" "UplDnm")
1261 (ANY_EXTEND:SVE_HSDI
1262 (unspec:SVE_PARTIAL_I
1263 [(match_operand:<SVE_PARTIAL_I:VPRED> 2 "register_operand" "Upl")
1264 (match_operand:SVE_PARTIAL_I 1 "aarch64_sve_ld<fn>f1_operand" "Ut<fn>")
1265 (reg:VNx16BI FFRT_REGNUM)]
1268 "TARGET_SVE && (~<SVE_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
1269 "ld<fn>f1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_HSDI:Vctype>, %2/z, %1"
1270 "&& !CONSTANT_P (operands[3])"
1272 operands[3] = CONSTM1_RTX (<SVE_HSDI:VPRED>mode);
1276 ;; -------------------------------------------------------------------------
1277 ;; ---- Non-temporal contiguous loads
1278 ;; -------------------------------------------------------------------------
1284 ;; -------------------------------------------------------------------------
1286 ;; Predicated contiguous non-temporal load.
1287 (define_insn "@aarch64_ldnt1<mode>"
1288 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
1290 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1291 (match_operand:SVE_FULL 1 "memory_operand" "m")]
1294 "ldnt1<Vesize>\t%0.<Vetype>, %2/z, %1"
1297 ;; -------------------------------------------------------------------------
1298 ;; ---- Normal gather loads
1299 ;; -------------------------------------------------------------------------
1300 ;; Includes gather forms of:
1303 ;; -------------------------------------------------------------------------
1305 ;; Unpredicated gather loads.
1306 (define_expand "gather_load<mode><v_int_container>"
1307 [(set (match_operand:SVE_24 0 "register_operand")
1310 (match_operand:DI 1 "aarch64_sve_gather_offset_<Vesize>")
1311 (match_operand:<V_INT_CONTAINER> 2 "register_operand")
1312 (match_operand:DI 3 "const_int_operand")
1313 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>")
1314 (mem:BLK (scratch))]
1315 UNSPEC_LD1_GATHER))]
1318 operands[5] = aarch64_ptrue_reg (<VPRED>mode);
1322 ;; Predicated gather loads for 32-bit elements. Operand 3 is true for
1323 ;; unsigned extension and false for signed extension.
1324 (define_insn "mask_gather_load<mode><v_int_container>"
1325 [(set (match_operand:SVE_4 0 "register_operand" "=w, w, w, w, w, w")
1327 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
1328 (match_operand:DI 1 "aarch64_sve_gather_offset_<Vesize>" "Z, vgw, rk, rk, rk, rk")
1329 (match_operand:VNx4SI 2 "register_operand" "w, w, w, w, w, w")
1330 (match_operand:DI 3 "const_int_operand" "Ui1, Ui1, Z, Ui1, Z, Ui1")
1331 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
1332 (mem:BLK (scratch))]
1333 UNSPEC_LD1_GATHER))]
1336 ld1<Vesize>\t%0.s, %5/z, [%2.s]
1337 ld1<Vesize>\t%0.s, %5/z, [%2.s, #%1]
1338 ld1<Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw]
1339 ld1<Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw]
1340 ld1<Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw %p4]
1341 ld1<Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw %p4]"
1344 ;; Predicated gather loads for 64-bit elements. The value of operand 3
1345 ;; doesn't matter in this case.
1346 (define_insn "mask_gather_load<mode><v_int_container>"
1347 [(set (match_operand:SVE_2 0 "register_operand" "=w, w, w, w")
1349 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
1350 (match_operand:DI 1 "aarch64_sve_gather_offset_<Vesize>" "Z, vgd, rk, rk")
1351 (match_operand:VNx2DI 2 "register_operand" "w, w, w, w")
1352 (match_operand:DI 3 "const_int_operand")
1353 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>" "Ui1, Ui1, Ui1, i")
1354 (mem:BLK (scratch))]
1355 UNSPEC_LD1_GATHER))]
1358 ld1<Vesize>\t%0.d, %5/z, [%2.d]
1359 ld1<Vesize>\t%0.d, %5/z, [%2.d, #%1]
1360 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d]
1361 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, lsl %p4]"
1364 ;; Likewise, but with the offset being extended from 32 bits.
1365 (define_insn_and_rewrite "*mask_gather_load<mode><v_int_container>_<su>xtw_unpacked"
1366 [(set (match_operand:SVE_2 0 "register_operand" "=w, w")
1368 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1369 (match_operand:DI 1 "register_operand" "rk, rk")
1373 (match_operand:VNx2SI 2 "register_operand" "w, w"))]
1375 (match_operand:DI 3 "const_int_operand")
1376 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
1377 (mem:BLK (scratch))]
1378 UNSPEC_LD1_GATHER))]
1381 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, <su>xtw]
1382 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, <su>xtw %p4]"
1383 "&& !CONSTANT_P (operands[6])"
1385 operands[6] = CONSTM1_RTX (VNx2BImode);
1389 ;; Likewise, but with the offset being truncated to 32 bits and then
1391 (define_insn_and_rewrite "*mask_gather_load<mode><v_int_container>_sxtw"
1392 [(set (match_operand:SVE_2 0 "register_operand" "=w, w")
1394 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1395 (match_operand:DI 1 "register_operand" "rk, rk")
1400 (match_operand:VNx2DI 2 "register_operand" "w, w")))]
1402 (match_operand:DI 3 "const_int_operand")
1403 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
1404 (mem:BLK (scratch))]
1405 UNSPEC_LD1_GATHER))]
1408 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw]
1409 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw %p4]"
1410 "&& !CONSTANT_P (operands[6])"
1412 operands[6] = CONSTM1_RTX (VNx2BImode);
1416 ;; Likewise, but with the offset being truncated to 32 bits and then
1418 (define_insn "*mask_gather_load<mode><v_int_container>_uxtw"
1419 [(set (match_operand:SVE_2 0 "register_operand" "=w, w")
1421 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1422 (match_operand:DI 1 "register_operand" "rk, rk")
1424 (match_operand:VNx2DI 2 "register_operand" "w, w")
1425 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
1426 (match_operand:DI 3 "const_int_operand")
1427 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
1428 (mem:BLK (scratch))]
1429 UNSPEC_LD1_GATHER))]
1432 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw]
1433 ld1<Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw %p4]"
1436 ;; -------------------------------------------------------------------------
1437 ;; ---- Extending gather loads
1438 ;; -------------------------------------------------------------------------
1439 ;; Includes gather forms of:
1446 ;; -------------------------------------------------------------------------
1448 ;; Predicated extending gather loads for 32-bit elements. Operand 3 is
1449 ;; true for unsigned extension and false for signed extension.
1450 (define_insn_and_rewrite "@aarch64_gather_load_<ANY_EXTEND:optab><SVE_4HSI:mode><SVE_4BHI:mode>"
1451 [(set (match_operand:SVE_4HSI 0 "register_operand" "=w, w, w, w, w, w")
1453 [(match_operand:VNx4BI 6 "general_operand" "UplDnm, UplDnm, UplDnm, UplDnm, UplDnm, UplDnm")
1454 (ANY_EXTEND:SVE_4HSI
1456 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
1457 (match_operand:DI 1 "aarch64_sve_gather_offset_<SVE_4BHI:Vesize>" "Z, vg<SVE_4BHI:Vesize>, rk, rk, rk, rk")
1458 (match_operand:VNx4SI 2 "register_operand" "w, w, w, w, w, w")
1459 (match_operand:DI 3 "const_int_operand" "Ui1, Ui1, Z, Ui1, Z, Ui1")
1460 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_4BHI:Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
1461 (mem:BLK (scratch))]
1462 UNSPEC_LD1_GATHER))]
1464 "TARGET_SVE && (~<SVE_4HSI:narrower_mask> & <SVE_4BHI:self_mask>) == 0"
1466 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%2.s]
1467 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%2.s, #%1]
1468 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw]
1469 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw]
1470 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw %p4]
1471 ld1<ANY_EXTEND:s><SVE_4BHI:Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw %p4]"
1472 "&& !CONSTANT_P (operands[6])"
1474 operands[6] = CONSTM1_RTX (VNx4BImode);
1478 ;; Predicated extending gather loads for 64-bit elements. The value of
1479 ;; operand 3 doesn't matter in this case.
1480 (define_insn_and_rewrite "@aarch64_gather_load_<ANY_EXTEND:optab><SVE_2HSDI:mode><SVE_2BHSI:mode>"
1481 [(set (match_operand:SVE_2HSDI 0 "register_operand" "=w, w, w, w")
1483 [(match_operand:VNx2BI 6 "general_operand" "UplDnm, UplDnm, UplDnm, UplDnm")
1484 (ANY_EXTEND:SVE_2HSDI
1486 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
1487 (match_operand:DI 1 "aarch64_sve_gather_offset_<SVE_2BHSI:Vesize>" "Z, vg<SVE_2BHSI:Vesize>, rk, rk")
1488 (match_operand:VNx2DI 2 "register_operand" "w, w, w, w")
1489 (match_operand:DI 3 "const_int_operand")
1490 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_2BHSI:Vesize>" "Ui1, Ui1, Ui1, i")
1491 (mem:BLK (scratch))]
1492 UNSPEC_LD1_GATHER))]
1494 "TARGET_SVE && (~<SVE_2HSDI:narrower_mask> & <SVE_2BHSI:self_mask>) == 0"
1496 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%2.d]
1497 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%2.d, #%1]
1498 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d]
1499 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, lsl %p4]"
1500 "&& !CONSTANT_P (operands[6])"
1502 operands[6] = CONSTM1_RTX (VNx2BImode);
1506 ;; Likewise, but with the offset being extended from 32 bits.
1507 (define_insn_and_rewrite "*aarch64_gather_load_<ANY_EXTEND:optab><SVE_2HSDI:mode><SVE_2BHSI:mode>_<ANY_EXTEND2:su>xtw_unpacked"
1508 [(set (match_operand:SVE_2HSDI 0 "register_operand" "=w, w")
1511 (ANY_EXTEND:SVE_2HSDI
1513 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1514 (match_operand:DI 1 "aarch64_reg_or_zero" "rk, rk")
1518 (match_operand:VNx2SI 2 "register_operand" "w, w"))]
1520 (match_operand:DI 3 "const_int_operand")
1521 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_2BHSI:Vesize>" "Ui1, i")
1522 (mem:BLK (scratch))]
1523 UNSPEC_LD1_GATHER))]
1525 "TARGET_SVE && (~<SVE_2HSDI:narrower_mask> & <SVE_2BHSI:self_mask>) == 0"
1527 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, <ANY_EXTEND2:su>xtw]
1528 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, <ANY_EXTEND2:su>xtw %p4]"
1529 "&& (!CONSTANT_P (operands[6]) || !CONSTANT_P (operands[7]))"
1531 operands[6] = CONSTM1_RTX (VNx2BImode);
1532 operands[7] = CONSTM1_RTX (VNx2BImode);
1536 ;; Likewise, but with the offset being truncated to 32 bits and then
1538 (define_insn_and_rewrite "*aarch64_gather_load_<ANY_EXTEND:optab><SVE_2HSDI:mode><SVE_2BHSI:mode>_sxtw"
1539 [(set (match_operand:SVE_2HSDI 0 "register_operand" "=w, w")
1542 (ANY_EXTEND:SVE_2HSDI
1544 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1545 (match_operand:DI 1 "aarch64_reg_or_zero" "rk, rk")
1550 (match_operand:VNx2DI 2 "register_operand" "w, w")))]
1552 (match_operand:DI 3 "const_int_operand")
1553 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_2BHSI:Vesize>" "Ui1, i")
1554 (mem:BLK (scratch))]
1555 UNSPEC_LD1_GATHER))]
1557 "TARGET_SVE && (~<SVE_2HSDI:narrower_mask> & <SVE_2BHSI:self_mask>) == 0"
1559 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw]
1560 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw %p4]"
1561 "&& (!CONSTANT_P (operands[6]) || !CONSTANT_P (operands[7]))"
1563 operands[6] = CONSTM1_RTX (VNx2BImode);
1564 operands[7] = CONSTM1_RTX (VNx2BImode);
1568 ;; Likewise, but with the offset being truncated to 32 bits and then
1570 (define_insn_and_rewrite "*aarch64_gather_load_<ANY_EXTEND:optab><SVE_2HSDI:mode><SVE_2BHSI:mode>_uxtw"
1571 [(set (match_operand:SVE_2HSDI 0 "register_operand" "=w, w")
1574 (ANY_EXTEND:SVE_2HSDI
1576 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1577 (match_operand:DI 1 "aarch64_reg_or_zero" "rk, rk")
1579 (match_operand:VNx2DI 2 "register_operand" "w, w")
1580 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
1581 (match_operand:DI 3 "const_int_operand")
1582 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_2BHSI:Vesize>" "Ui1, i")
1583 (mem:BLK (scratch))]
1584 UNSPEC_LD1_GATHER))]
1586 "TARGET_SVE && (~<SVE_2HSDI:narrower_mask> & <SVE_2BHSI:self_mask>) == 0"
1588 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw]
1589 ld1<ANY_EXTEND:s><SVE_2BHSI:Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw %p4]"
1590 "&& !CONSTANT_P (operands[7])"
1592 operands[7] = CONSTM1_RTX (VNx2BImode);
1596 ;; -------------------------------------------------------------------------
1597 ;; ---- First-faulting gather loads
1598 ;; -------------------------------------------------------------------------
1599 ;; Includes gather forms of:
1602 ;; -------------------------------------------------------------------------
1604 ;; Predicated first-faulting gather loads for 32-bit elements. Operand
1605 ;; 3 is true for unsigned extension and false for signed extension.
1606 (define_insn "@aarch64_ldff1_gather<mode>"
1607 [(set (match_operand:SVE_FULL_S 0 "register_operand" "=w, w, w, w, w, w")
1609 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
1610 (match_operand:DI 1 "aarch64_sve_gather_offset_w" "Z, vgw, rk, rk, rk, rk")
1611 (match_operand:VNx4SI 2 "register_operand" "w, w, w, w, w, w")
1612 (match_operand:DI 3 "const_int_operand" "i, i, Z, Ui1, Z, Ui1")
1613 (match_operand:DI 4 "aarch64_gather_scale_operand_w" "Ui1, Ui1, Ui1, Ui1, i, i")
1615 (reg:VNx16BI FFRT_REGNUM)]
1616 UNSPEC_LDFF1_GATHER))]
1619 ldff1w\t%0.s, %5/z, [%2.s]
1620 ldff1w\t%0.s, %5/z, [%2.s, #%1]
1621 ldff1w\t%0.s, %5/z, [%1, %2.s, sxtw]
1622 ldff1w\t%0.s, %5/z, [%1, %2.s, uxtw]
1623 ldff1w\t%0.s, %5/z, [%1, %2.s, sxtw %p4]
1624 ldff1w\t%0.s, %5/z, [%1, %2.s, uxtw %p4]"
1627 ;; Predicated first-faulting gather loads for 64-bit elements. The value
1628 ;; of operand 3 doesn't matter in this case.
1629 (define_insn "@aarch64_ldff1_gather<mode>"
1630 [(set (match_operand:SVE_FULL_D 0 "register_operand" "=w, w, w, w")
1632 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
1633 (match_operand:DI 1 "aarch64_sve_gather_offset_d" "Z, vgd, rk, rk")
1634 (match_operand:VNx2DI 2 "register_operand" "w, w, w, w")
1635 (match_operand:DI 3 "const_int_operand")
1636 (match_operand:DI 4 "aarch64_gather_scale_operand_d" "Ui1, Ui1, Ui1, i")
1638 (reg:VNx16BI FFRT_REGNUM)]
1639 UNSPEC_LDFF1_GATHER))]
1642 ldff1d\t%0.d, %5/z, [%2.d]
1643 ldff1d\t%0.d, %5/z, [%2.d, #%1]
1644 ldff1d\t%0.d, %5/z, [%1, %2.d]
1645 ldff1d\t%0.d, %5/z, [%1, %2.d, lsl %p4]"
1648 ;; Likewise, but with the offset being sign-extended from 32 bits.
1649 (define_insn_and_rewrite "*aarch64_ldff1_gather<mode>_sxtw"
1650 [(set (match_operand:SVE_FULL_D 0 "register_operand" "=w, w")
1652 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1653 (match_operand:DI 1 "register_operand" "rk, rk")
1658 (match_operand:VNx2DI 2 "register_operand" "w, w")))]
1660 (match_operand:DI 3 "const_int_operand")
1661 (match_operand:DI 4 "aarch64_gather_scale_operand_d" "Ui1, i")
1663 (reg:VNx16BI FFRT_REGNUM)]
1664 UNSPEC_LDFF1_GATHER))]
1667 ldff1d\t%0.d, %5/z, [%1, %2.d, sxtw]
1668 ldff1d\t%0.d, %5/z, [%1, %2.d, sxtw %p4]"
1669 "&& !CONSTANT_P (operands[6])"
1671 operands[6] = CONSTM1_RTX (VNx2BImode);
1675 ;; Likewise, but with the offset being zero-extended from 32 bits.
1676 (define_insn "*aarch64_ldff1_gather<mode>_uxtw"
1677 [(set (match_operand:SVE_FULL_D 0 "register_operand" "=w, w")
1679 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1680 (match_operand:DI 1 "register_operand" "rk, rk")
1682 (match_operand:VNx2DI 2 "register_operand" "w, w")
1683 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
1684 (match_operand:DI 3 "const_int_operand")
1685 (match_operand:DI 4 "aarch64_gather_scale_operand_d" "Ui1, i")
1687 (reg:VNx16BI FFRT_REGNUM)]
1688 UNSPEC_LDFF1_GATHER))]
1691 ldff1d\t%0.d, %5/z, [%1, %2.d, uxtw]
1692 ldff1d\t%0.d, %5/z, [%1, %2.d, uxtw %p4]"
1695 ;; -------------------------------------------------------------------------
1696 ;; ---- First-faulting extending gather loads
1697 ;; -------------------------------------------------------------------------
1698 ;; Includes gather forms of:
1705 ;; -------------------------------------------------------------------------
1707 ;; Predicated extending first-faulting gather loads for 32-bit elements.
1708 ;; Operand 3 is true for unsigned extension and false for signed extension.
1709 (define_insn_and_rewrite "@aarch64_ldff1_gather_<ANY_EXTEND:optab><VNx4_WIDE:mode><VNx4_NARROW:mode>"
1710 [(set (match_operand:VNx4_WIDE 0 "register_operand" "=w, w, w, w, w, w")
1712 [(match_operand:VNx4BI 6 "general_operand" "UplDnm, UplDnm, UplDnm, UplDnm, UplDnm, UplDnm")
1713 (ANY_EXTEND:VNx4_WIDE
1715 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
1716 (match_operand:DI 1 "aarch64_sve_gather_offset_<VNx4_NARROW:Vesize>" "Z, vg<VNx4_NARROW:Vesize>, rk, rk, rk, rk")
1717 (match_operand:VNx4_WIDE 2 "register_operand" "w, w, w, w, w, w")
1718 (match_operand:DI 3 "const_int_operand" "i, i, Z, Ui1, Z, Ui1")
1719 (match_operand:DI 4 "aarch64_gather_scale_operand_<VNx4_NARROW:Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
1721 (reg:VNx16BI FFRT_REGNUM)]
1722 UNSPEC_LDFF1_GATHER))]
1726 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%2.s]
1727 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%2.s, #%1]
1728 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw]
1729 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw]
1730 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%1, %2.s, sxtw %p4]
1731 ldff1<ANY_EXTEND:s><VNx4_NARROW:Vesize>\t%0.s, %5/z, [%1, %2.s, uxtw %p4]"
1732 "&& !CONSTANT_P (operands[6])"
1734 operands[6] = CONSTM1_RTX (VNx4BImode);
1738 ;; Predicated extending first-faulting gather loads for 64-bit elements.
1739 ;; The value of operand 3 doesn't matter in this case.
1740 (define_insn_and_rewrite "@aarch64_ldff1_gather_<ANY_EXTEND:optab><VNx2_WIDE:mode><VNx2_NARROW:mode>"
1741 [(set (match_operand:VNx2_WIDE 0 "register_operand" "=w, w, w, w")
1743 [(match_operand:VNx2BI 6 "general_operand" "UplDnm, UplDnm, UplDnm, UplDnm")
1744 (ANY_EXTEND:VNx2_WIDE
1746 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
1747 (match_operand:DI 1 "aarch64_sve_gather_offset_<VNx2_NARROW:Vesize>" "Z, vg<VNx2_NARROW:Vesize>, rk, rk")
1748 (match_operand:VNx2_WIDE 2 "register_operand" "w, w, w, w")
1749 (match_operand:DI 3 "const_int_operand")
1750 (match_operand:DI 4 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, Ui1, Ui1, i")
1752 (reg:VNx16BI FFRT_REGNUM)]
1753 UNSPEC_LDFF1_GATHER))]
1757 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%2.d]
1758 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%2.d, #%1]
1759 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d]
1760 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d, lsl %p4]"
1761 "&& !CONSTANT_P (operands[6])"
1763 operands[6] = CONSTM1_RTX (VNx2BImode);
1767 ;; Likewise, but with the offset being sign-extended from 32 bits.
1768 (define_insn_and_rewrite "*aarch64_ldff1_gather_<ANY_EXTEND:optab><VNx2_WIDE:mode><VNx2_NARROW:mode>_sxtw"
1769 [(set (match_operand:VNx2_WIDE 0 "register_operand" "=w, w")
1772 (ANY_EXTEND:VNx2_WIDE
1774 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1775 (match_operand:DI 1 "aarch64_reg_or_zero" "rk, rk")
1780 (match_operand:VNx2DI 2 "register_operand" "w, w")))]
1782 (match_operand:DI 3 "const_int_operand")
1783 (match_operand:DI 4 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, i")
1785 (reg:VNx16BI FFRT_REGNUM)]
1786 UNSPEC_LDFF1_GATHER))]
1790 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw]
1791 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d, sxtw %p4]"
1792 "&& (!CONSTANT_P (operands[6]) || !CONSTANT_P (operands[7]))"
1794 operands[6] = CONSTM1_RTX (VNx2BImode);
1795 operands[7] = CONSTM1_RTX (VNx2BImode);
1799 ;; Likewise, but with the offset being zero-extended from 32 bits.
1800 (define_insn_and_rewrite "*aarch64_ldff1_gather_<ANY_EXTEND:optab><VNx2_WIDE:mode><VNx2_NARROW:mode>_uxtw"
1801 [(set (match_operand:VNx2_WIDE 0 "register_operand" "=w, w")
1804 (ANY_EXTEND:VNx2_WIDE
1806 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
1807 (match_operand:DI 1 "aarch64_reg_or_zero" "rk, rk")
1809 (match_operand:VNx2DI 2 "register_operand" "w, w")
1810 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
1811 (match_operand:DI 3 "const_int_operand")
1812 (match_operand:DI 4 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, i")
1814 (reg:VNx16BI FFRT_REGNUM)]
1815 UNSPEC_LDFF1_GATHER))]
1819 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw]
1820 ldff1<ANY_EXTEND:s><VNx2_NARROW:Vesize>\t%0.d, %5/z, [%1, %2.d, uxtw %p4]"
1821 "&& !CONSTANT_P (operands[7])"
1823 operands[7] = CONSTM1_RTX (VNx2BImode);
1827 ;; =========================================================================
1829 ;; =========================================================================
1831 ;; -------------------------------------------------------------------------
1832 ;; ---- Contiguous prefetches
1833 ;; -------------------------------------------------------------------------
1834 ;; Includes contiguous forms of:
1839 ;; -------------------------------------------------------------------------
1841 ;; Contiguous predicated prefetches. Operand 2 gives the real prefetch
1842 ;; operation (as an svprfop), with operands 3 and 4 providing distilled
1844 (define_insn "@aarch64_sve_prefetch<mode>"
1845 [(prefetch (unspec:DI
1846 [(match_operand:<VPRED> 0 "register_operand" "Upl")
1847 (match_operand:SVE_FULL_I 1 "aarch64_sve_prefetch_operand" "UP<Vesize>")
1848 (match_operand:DI 2 "const_int_operand")]
1849 UNSPEC_SVE_PREFETCH)
1850 (match_operand:DI 3 "const_int_operand")
1851 (match_operand:DI 4 "const_int_operand"))]
1854 operands[1] = gen_rtx_MEM (<MODE>mode, operands[1]);
1855 return aarch64_output_sve_prefetch ("prf<Vesize>", operands[2], "%0, %1");
1859 ;; -------------------------------------------------------------------------
1860 ;; ---- Gather prefetches
1861 ;; -------------------------------------------------------------------------
1862 ;; Includes gather forms of:
1867 ;; -------------------------------------------------------------------------
1869 ;; Predicated gather prefetches for 32-bit bases and offsets. The operands
1871 ;; 0: the governing predicate
1872 ;; 1: the scalar component of the address
1873 ;; 2: the vector component of the address
1874 ;; 3: 1 for zero extension, 0 for sign extension
1875 ;; 4: the scale multiplier
1876 ;; 5: a vector zero that identifies the mode of data being accessed
1877 ;; 6: the prefetch operator (an svprfop)
1878 ;; 7: the normal RTL prefetch rw flag
1879 ;; 8: the normal RTL prefetch locality value
1880 (define_insn "@aarch64_sve_gather_prefetch<SVE_FULL_I:mode><VNx4SI_ONLY:mode>"
1881 [(prefetch (unspec:DI
1882 [(match_operand:VNx4BI 0 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
1883 (match_operand:DI 1 "aarch64_sve_gather_offset_<SVE_FULL_I:Vesize>" "Z, vg<SVE_FULL_I:Vesize>, rk, rk, rk, rk")
1884 (match_operand:VNx4SI_ONLY 2 "register_operand" "w, w, w, w, w, w")
1885 (match_operand:DI 3 "const_int_operand" "i, i, Z, Ui1, Z, Ui1")
1886 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_FULL_I:Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
1887 (match_operand:SVE_FULL_I 5 "aarch64_simd_imm_zero")
1888 (match_operand:DI 6 "const_int_operand")]
1889 UNSPEC_SVE_PREFETCH_GATHER)
1890 (match_operand:DI 7 "const_int_operand")
1891 (match_operand:DI 8 "const_int_operand"))]
1894 static const char *const insns[][2] = {
1895 "prf<SVE_FULL_I:Vesize>", "%0, [%2.s]",
1896 "prf<SVE_FULL_I:Vesize>", "%0, [%2.s, #%1]",
1897 "prfb", "%0, [%1, %2.s, sxtw]",
1898 "prfb", "%0, [%1, %2.s, uxtw]",
1899 "prf<SVE_FULL_I:Vesize>", "%0, [%1, %2.s, sxtw %p4]",
1900 "prf<SVE_FULL_I:Vesize>", "%0, [%1, %2.s, uxtw %p4]"
1902 const char *const *parts = insns[which_alternative];
1903 return aarch64_output_sve_prefetch (parts[0], operands[6], parts[1]);
1907 ;; Predicated gather prefetches for 64-bit elements. The value of operand 3
1908 ;; doesn't matter in this case.
1909 (define_insn "@aarch64_sve_gather_prefetch<SVE_FULL_I:mode><VNx2DI_ONLY:mode>"
1910 [(prefetch (unspec:DI
1911 [(match_operand:VNx2BI 0 "register_operand" "Upl, Upl, Upl, Upl")
1912 (match_operand:DI 1 "aarch64_sve_gather_offset_<SVE_FULL_I:Vesize>" "Z, vg<SVE_FULL_I:Vesize>, rk, rk")
1913 (match_operand:VNx2DI_ONLY 2 "register_operand" "w, w, w, w")
1914 (match_operand:DI 3 "const_int_operand")
1915 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_FULL_I:Vesize>" "Ui1, Ui1, Ui1, i")
1916 (match_operand:SVE_FULL_I 5 "aarch64_simd_imm_zero")
1917 (match_operand:DI 6 "const_int_operand")]
1918 UNSPEC_SVE_PREFETCH_GATHER)
1919 (match_operand:DI 7 "const_int_operand")
1920 (match_operand:DI 8 "const_int_operand"))]
1923 static const char *const insns[][2] = {
1924 "prf<SVE_FULL_I:Vesize>", "%0, [%2.d]",
1925 "prf<SVE_FULL_I:Vesize>", "%0, [%2.d, #%1]",
1926 "prfb", "%0, [%1, %2.d]",
1927 "prf<SVE_FULL_I:Vesize>", "%0, [%1, %2.d, lsl %p4]"
1929 const char *const *parts = insns[which_alternative];
1930 return aarch64_output_sve_prefetch (parts[0], operands[6], parts[1]);
1934 ;; Likewise, but with the offset being sign-extended from 32 bits.
1935 (define_insn_and_rewrite "*aarch64_sve_gather_prefetch<SVE_FULL_I:mode><VNx2DI_ONLY:mode>_sxtw"
1936 [(prefetch (unspec:DI
1937 [(match_operand:VNx2BI 0 "register_operand" "Upl, Upl")
1938 (match_operand:DI 1 "register_operand" "rk, rk")
1943 (match_operand:VNx2DI 2 "register_operand" "w, w")))]
1945 (match_operand:DI 3 "const_int_operand")
1946 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_FULL_I:Vesize>" "Ui1, i")
1947 (match_operand:SVE_FULL_I 5 "aarch64_simd_imm_zero")
1948 (match_operand:DI 6 "const_int_operand")]
1949 UNSPEC_SVE_PREFETCH_GATHER)
1950 (match_operand:DI 7 "const_int_operand")
1951 (match_operand:DI 8 "const_int_operand"))]
1954 static const char *const insns[][2] = {
1955 "prfb", "%0, [%1, %2.d, sxtw]",
1956 "prf<SVE_FULL_I:Vesize>", "%0, [%1, %2.d, sxtw %p4]"
1958 const char *const *parts = insns[which_alternative];
1959 return aarch64_output_sve_prefetch (parts[0], operands[6], parts[1]);
1961 "&& !rtx_equal_p (operands[0], operands[9])"
1963 operands[9] = copy_rtx (operands[0]);
1967 ;; Likewise, but with the offset being zero-extended from 32 bits.
1968 (define_insn "*aarch64_sve_gather_prefetch<SVE_FULL_I:mode><VNx2DI_ONLY:mode>_uxtw"
1969 [(prefetch (unspec:DI
1970 [(match_operand:VNx2BI 0 "register_operand" "Upl, Upl")
1971 (match_operand:DI 1 "register_operand" "rk, rk")
1973 (match_operand:VNx2DI 2 "register_operand" "w, w")
1974 (match_operand:VNx2DI 9 "aarch64_sve_uxtw_immediate"))
1975 (match_operand:DI 3 "const_int_operand")
1976 (match_operand:DI 4 "aarch64_gather_scale_operand_<SVE_FULL_I:Vesize>" "Ui1, i")
1977 (match_operand:SVE_FULL_I 5 "aarch64_simd_imm_zero")
1978 (match_operand:DI 6 "const_int_operand")]
1979 UNSPEC_SVE_PREFETCH_GATHER)
1980 (match_operand:DI 7 "const_int_operand")
1981 (match_operand:DI 8 "const_int_operand"))]
1984 static const char *const insns[][2] = {
1985 "prfb", "%0, [%1, %2.d, uxtw]",
1986 "prf<SVE_FULL_I:Vesize>", "%0, [%1, %2.d, uxtw %p4]"
1988 const char *const *parts = insns[which_alternative];
1989 return aarch64_output_sve_prefetch (parts[0], operands[6], parts[1]);
1993 ;; =========================================================================
1995 ;; =========================================================================
1997 ;; -------------------------------------------------------------------------
1998 ;; ---- Normal contiguous stores
1999 ;; -------------------------------------------------------------------------
2000 ;; Includes contiguous forms of:
2017 ;; -------------------------------------------------------------------------
2020 (define_insn "maskstore<mode><vpred>"
2021 [(set (match_operand:SVE_ALL 0 "memory_operand" "+m")
2023 [(match_operand:<VPRED> 2 "register_operand" "Upl")
2024 (match_operand:SVE_ALL 1 "register_operand" "w")
2028 "st1<Vesize>\t%1.<Vctype>, %2, %0"
2031 ;; Unpredicated ST[234]. This is always a full update, so the dependence
2032 ;; on the old value of the memory location (via (match_dup 0)) is redundant.
2033 ;; There doesn't seem to be any obvious benefit to treating the all-true
2034 ;; case differently though. In particular, it's very unlikely that we'll
2035 ;; only find out during RTL that a store_lanes is dead.
2036 (define_expand "vec_store_lanes<mode><vsingle>"
2037 [(set (match_operand:SVE_STRUCT 0 "memory_operand")
2040 (match_operand:SVE_STRUCT 1 "register_operand")
2045 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
2049 ;; Predicated ST[234].
2050 (define_insn "vec_mask_store_lanes<mode><vsingle>"
2051 [(set (match_operand:SVE_STRUCT 0 "memory_operand" "+m")
2053 [(match_operand:<VPRED> 2 "register_operand" "Upl")
2054 (match_operand:SVE_STRUCT 1 "register_operand" "w")
2058 "st<vector_count><Vesize>\t%1, %2, %0"
2061 ;; -------------------------------------------------------------------------
2062 ;; ---- Truncating contiguous stores
2063 ;; -------------------------------------------------------------------------
2068 ;; -------------------------------------------------------------------------
2070 ;; Predicated truncate and store, with 8 elements per 128-bit block.
2071 (define_insn "@aarch64_store_trunc<VNx8_NARROW:mode><VNx8_WIDE:mode>"
2072 [(set (match_operand:VNx8_NARROW 0 "memory_operand" "+m")
2074 [(match_operand:VNx8BI 2 "register_operand" "Upl")
2075 (truncate:VNx8_NARROW
2076 (match_operand:VNx8_WIDE 1 "register_operand" "w"))
2080 "st1<VNx8_NARROW:Vesize>\t%1.<VNx8_WIDE:Vetype>, %2, %0"
2083 ;; Predicated truncate and store, with 4 elements per 128-bit block.
2084 (define_insn "@aarch64_store_trunc<VNx4_NARROW:mode><VNx4_WIDE:mode>"
2085 [(set (match_operand:VNx4_NARROW 0 "memory_operand" "+m")
2087 [(match_operand:VNx4BI 2 "register_operand" "Upl")
2088 (truncate:VNx4_NARROW
2089 (match_operand:VNx4_WIDE 1 "register_operand" "w"))
2093 "st1<VNx4_NARROW:Vesize>\t%1.<VNx4_WIDE:Vetype>, %2, %0"
2096 ;; Predicated truncate and store, with 2 elements per 128-bit block.
2097 (define_insn "@aarch64_store_trunc<VNx2_NARROW:mode><VNx2_WIDE:mode>"
2098 [(set (match_operand:VNx2_NARROW 0 "memory_operand" "+m")
2100 [(match_operand:VNx2BI 2 "register_operand" "Upl")
2101 (truncate:VNx2_NARROW
2102 (match_operand:VNx2_WIDE 1 "register_operand" "w"))
2106 "st1<VNx2_NARROW:Vesize>\t%1.<VNx2_WIDE:Vetype>, %2, %0"
2109 ;; -------------------------------------------------------------------------
2110 ;; ---- Non-temporal contiguous stores
2111 ;; -------------------------------------------------------------------------
2117 ;; -------------------------------------------------------------------------
2119 (define_insn "@aarch64_stnt1<mode>"
2120 [(set (match_operand:SVE_FULL 0 "memory_operand" "+m")
2122 [(match_operand:<VPRED> 2 "register_operand" "Upl")
2123 (match_operand:SVE_FULL 1 "register_operand" "w")
2127 "stnt1<Vesize>\t%1.<Vetype>, %2, %0"
2130 ;; -------------------------------------------------------------------------
2131 ;; ---- Normal scatter stores
2132 ;; -------------------------------------------------------------------------
2133 ;; Includes scatter forms of:
2136 ;; -------------------------------------------------------------------------
2138 ;; Unpredicated scatter stores.
2139 (define_expand "scatter_store<mode><v_int_container>"
2140 [(set (mem:BLK (scratch))
2143 (match_operand:DI 0 "aarch64_sve_gather_offset_<Vesize>")
2144 (match_operand:<V_INT_CONTAINER> 1 "register_operand")
2145 (match_operand:DI 2 "const_int_operand")
2146 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>")
2147 (match_operand:SVE_24 4 "register_operand")]
2148 UNSPEC_ST1_SCATTER))]
2151 operands[5] = aarch64_ptrue_reg (<VPRED>mode);
2155 ;; Predicated scatter stores for 32-bit elements. Operand 2 is true for
2156 ;; unsigned extension and false for signed extension.
2157 (define_insn "mask_scatter_store<mode><v_int_container>"
2158 [(set (mem:BLK (scratch))
2160 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
2161 (match_operand:DI 0 "aarch64_sve_gather_offset_<Vesize>" "Z, vgw, rk, rk, rk, rk")
2162 (match_operand:VNx4SI 1 "register_operand" "w, w, w, w, w, w")
2163 (match_operand:DI 2 "const_int_operand" "Ui1, Ui1, Z, Ui1, Z, Ui1")
2164 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
2165 (match_operand:SVE_4 4 "register_operand" "w, w, w, w, w, w")]
2166 UNSPEC_ST1_SCATTER))]
2169 st1<Vesize>\t%4.s, %5, [%1.s]
2170 st1<Vesize>\t%4.s, %5, [%1.s, #%0]
2171 st1<Vesize>\t%4.s, %5, [%0, %1.s, sxtw]
2172 st1<Vesize>\t%4.s, %5, [%0, %1.s, uxtw]
2173 st1<Vesize>\t%4.s, %5, [%0, %1.s, sxtw %p3]
2174 st1<Vesize>\t%4.s, %5, [%0, %1.s, uxtw %p3]"
2177 ;; Predicated scatter stores for 64-bit elements. The value of operand 2
2178 ;; doesn't matter in this case.
2179 (define_insn "mask_scatter_store<mode><v_int_container>"
2180 [(set (mem:BLK (scratch))
2182 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
2183 (match_operand:DI 0 "aarch64_sve_gather_offset_<Vesize>" "Z, vgd, rk, rk")
2184 (match_operand:VNx2DI 1 "register_operand" "w, w, w, w")
2185 (match_operand:DI 2 "const_int_operand")
2186 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>" "Ui1, Ui1, Ui1, i")
2187 (match_operand:SVE_2 4 "register_operand" "w, w, w, w")]
2188 UNSPEC_ST1_SCATTER))]
2191 st1<Vesize>\t%4.d, %5, [%1.d]
2192 st1<Vesize>\t%4.d, %5, [%1.d, #%0]
2193 st1<Vesize>\t%4.d, %5, [%0, %1.d]
2194 st1<Vesize>\t%4.d, %5, [%0, %1.d, lsl %p3]"
2197 ;; Likewise, but with the offset being extended from 32 bits.
2198 (define_insn_and_rewrite "*mask_scatter_store<mode><v_int_container>_<su>xtw_unpacked"
2199 [(set (mem:BLK (scratch))
2201 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
2202 (match_operand:DI 0 "register_operand" "rk, rk")
2206 (match_operand:VNx2SI 1 "register_operand" "w, w"))]
2208 (match_operand:DI 2 "const_int_operand")
2209 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
2210 (match_operand:SVE_2 4 "register_operand" "w, w")]
2211 UNSPEC_ST1_SCATTER))]
2214 st1<Vesize>\t%4.d, %5, [%0, %1.d, <su>xtw]
2215 st1<Vesize>\t%4.d, %5, [%0, %1.d, <su>xtw %p3]"
2216 "&& !CONSTANT_P (operands[6])"
2218 operands[6] = CONSTM1_RTX (<VPRED>mode);
2222 ;; Likewise, but with the offset being truncated to 32 bits and then
2224 (define_insn_and_rewrite "*mask_scatter_store<mode><v_int_container>_sxtw"
2225 [(set (mem:BLK (scratch))
2227 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
2228 (match_operand:DI 0 "register_operand" "rk, rk")
2233 (match_operand:VNx2DI 1 "register_operand" "w, w")))]
2235 (match_operand:DI 2 "const_int_operand")
2236 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
2237 (match_operand:SVE_2 4 "register_operand" "w, w")]
2238 UNSPEC_ST1_SCATTER))]
2241 st1<Vesize>\t%4.d, %5, [%0, %1.d, sxtw]
2242 st1<Vesize>\t%4.d, %5, [%0, %1.d, sxtw %p3]"
2243 "&& !CONSTANT_P (operands[6])"
2245 operands[6] = CONSTM1_RTX (<VPRED>mode);
2249 ;; Likewise, but with the offset being truncated to 32 bits and then
2251 (define_insn "*mask_scatter_store<mode><v_int_container>_uxtw"
2252 [(set (mem:BLK (scratch))
2254 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
2255 (match_operand:DI 0 "aarch64_reg_or_zero" "rk, rk")
2257 (match_operand:VNx2DI 1 "register_operand" "w, w")
2258 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
2259 (match_operand:DI 2 "const_int_operand")
2260 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>" "Ui1, i")
2261 (match_operand:SVE_2 4 "register_operand" "w, w")]
2262 UNSPEC_ST1_SCATTER))]
2265 st1<Vesize>\t%4.d, %5, [%0, %1.d, uxtw]
2266 st1<Vesize>\t%4.d, %5, [%0, %1.d, uxtw %p3]"
2269 ;; -------------------------------------------------------------------------
2270 ;; ---- Truncating scatter stores
2271 ;; -------------------------------------------------------------------------
2272 ;; Includes scatter forms of:
2276 ;; -------------------------------------------------------------------------
2278 ;; Predicated truncating scatter stores for 32-bit elements. Operand 2 is
2279 ;; true for unsigned extension and false for signed extension.
2280 (define_insn "@aarch64_scatter_store_trunc<VNx4_NARROW:mode><VNx4_WIDE:mode>"
2281 [(set (mem:BLK (scratch))
2283 [(match_operand:VNx4BI 5 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
2284 (match_operand:DI 0 "aarch64_sve_gather_offset_<VNx4_NARROW:Vesize>" "Z, vg<VNx4_NARROW:Vesize>, rk, rk, rk, rk")
2285 (match_operand:VNx4SI 1 "register_operand" "w, w, w, w, w, w")
2286 (match_operand:DI 2 "const_int_operand" "Ui1, Ui1, Z, Ui1, Z, Ui1")
2287 (match_operand:DI 3 "aarch64_gather_scale_operand_<VNx4_NARROW:Vesize>" "Ui1, Ui1, Ui1, Ui1, i, i")
2288 (truncate:VNx4_NARROW
2289 (match_operand:VNx4_WIDE 4 "register_operand" "w, w, w, w, w, w"))]
2290 UNSPEC_ST1_SCATTER))]
2293 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%1.s]
2294 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%1.s, #%0]
2295 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%0, %1.s, sxtw]
2296 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%0, %1.s, uxtw]
2297 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%0, %1.s, sxtw %p3]
2298 st1<VNx4_NARROW:Vesize>\t%4.s, %5, [%0, %1.s, uxtw %p3]"
2301 ;; Predicated truncating scatter stores for 64-bit elements. The value of
2302 ;; operand 2 doesn't matter in this case.
2303 (define_insn "@aarch64_scatter_store_trunc<VNx2_NARROW:mode><VNx2_WIDE:mode>"
2304 [(set (mem:BLK (scratch))
2306 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl, Upl, Upl")
2307 (match_operand:DI 0 "aarch64_sve_gather_offset_<VNx2_NARROW:Vesize>" "Z, vg<VNx2_NARROW:Vesize>, rk, rk")
2308 (match_operand:VNx2DI 1 "register_operand" "w, w, w, w")
2309 (match_operand:DI 2 "const_int_operand")
2310 (match_operand:DI 3 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, Ui1, Ui1, i")
2311 (truncate:VNx2_NARROW
2312 (match_operand:VNx2_WIDE 4 "register_operand" "w, w, w, w"))]
2313 UNSPEC_ST1_SCATTER))]
2316 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%1.d]
2317 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%1.d, #%0]
2318 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d]
2319 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d, lsl %p3]"
2322 ;; Likewise, but with the offset being sign-extended from 32 bits.
2323 (define_insn_and_rewrite "*aarch64_scatter_store_trunc<VNx2_NARROW:mode><VNx2_WIDE:mode>_sxtw"
2324 [(set (mem:BLK (scratch))
2326 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
2327 (match_operand:DI 0 "register_operand" "rk, rk")
2332 (match_operand:VNx2DI 1 "register_operand" "w, w")))]
2334 (match_operand:DI 2 "const_int_operand")
2335 (match_operand:DI 3 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, i")
2336 (truncate:VNx2_NARROW
2337 (match_operand:VNx2_WIDE 4 "register_operand" "w, w"))]
2338 UNSPEC_ST1_SCATTER))]
2341 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d, sxtw]
2342 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d, sxtw %p3]"
2343 "&& !rtx_equal_p (operands[5], operands[6])"
2345 operands[6] = copy_rtx (operands[5]);
2349 ;; Likewise, but with the offset being zero-extended from 32 bits.
2350 (define_insn "*aarch64_scatter_store_trunc<VNx2_NARROW:mode><VNx2_WIDE:mode>_uxtw"
2351 [(set (mem:BLK (scratch))
2353 [(match_operand:VNx2BI 5 "register_operand" "Upl, Upl")
2354 (match_operand:DI 0 "aarch64_reg_or_zero" "rk, rk")
2356 (match_operand:VNx2DI 1 "register_operand" "w, w")
2357 (match_operand:VNx2DI 6 "aarch64_sve_uxtw_immediate"))
2358 (match_operand:DI 2 "const_int_operand")
2359 (match_operand:DI 3 "aarch64_gather_scale_operand_<VNx2_NARROW:Vesize>" "Ui1, i")
2360 (truncate:VNx2_NARROW
2361 (match_operand:VNx2_WIDE 4 "register_operand" "w, w"))]
2362 UNSPEC_ST1_SCATTER))]
2365 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d, uxtw]
2366 st1<VNx2_NARROW:Vesize>\t%4.d, %5, [%0, %1.d, uxtw %p3]"
2369 ;; =========================================================================
2370 ;; == Vector creation
2371 ;; =========================================================================
2373 ;; -------------------------------------------------------------------------
2374 ;; ---- [INT,FP] Duplicate element
2375 ;; -------------------------------------------------------------------------
2387 ;; -------------------------------------------------------------------------
2389 (define_expand "vec_duplicate<mode>"
2391 [(set (match_operand:SVE_ALL 0 "register_operand")
2392 (vec_duplicate:SVE_ALL
2393 (match_operand:<VEL> 1 "aarch64_sve_dup_operand")))
2394 (clobber (scratch:VNx16BI))])]
2397 if (MEM_P (operands[1]))
2399 rtx ptrue = aarch64_ptrue_reg (<VPRED>mode);
2400 emit_insn (gen_sve_ld1r<mode> (operands[0], ptrue, operands[1],
2401 CONST0_RTX (<MODE>mode)));
2407 ;; Accept memory operands for the benefit of combine, and also in case
2408 ;; the scalar input gets spilled to memory during RA. We want to split
2409 ;; the load at the first opportunity in order to allow the PTRUE to be
2410 ;; optimized with surrounding code.
2411 (define_insn_and_split "*vec_duplicate<mode>_reg"
2412 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w, w")
2413 (vec_duplicate:SVE_ALL
2414 (match_operand:<VEL> 1 "aarch64_sve_dup_operand" "r, w, Uty")))
2415 (clobber (match_scratch:VNx16BI 2 "=X, X, Upl"))]
2418 mov\t%0.<Vetype>, %<vwcore>1
2419 mov\t%0.<Vetype>, %<Vetype>1
2421 "&& MEM_P (operands[1])"
2424 if (GET_CODE (operands[2]) == SCRATCH)
2425 operands[2] = gen_reg_rtx (VNx16BImode);
2426 emit_move_insn (operands[2], CONSTM1_RTX (VNx16BImode));
2427 rtx gp = gen_lowpart (<VPRED>mode, operands[2]);
2428 emit_insn (gen_sve_ld1r<mode> (operands[0], gp, operands[1],
2429 CONST0_RTX (<MODE>mode)));
2432 [(set_attr "length" "4,4,8")]
2435 ;; Duplicate an Advanced SIMD vector to fill an SVE vector (LE version).
2436 (define_insn "@aarch64_vec_duplicate_vq<mode>_le"
2437 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
2438 (vec_duplicate:SVE_FULL
2439 (match_operand:<V128> 1 "register_operand" "w")))]
2440 "TARGET_SVE && !BYTES_BIG_ENDIAN"
2442 operands[1] = gen_rtx_REG (<MODE>mode, REGNO (operands[1]));
2443 return "dup\t%0.q, %1.q[0]";
2447 ;; Duplicate an Advanced SIMD vector to fill an SVE vector (BE version).
2448 ;; The SVE register layout puts memory lane N into (architectural)
2449 ;; register lane N, whereas the Advanced SIMD layout puts the memory
2450 ;; lsb into the register lsb. We therefore have to describe this in rtl
2451 ;; terms as a reverse of the V128 vector followed by a duplicate.
2452 (define_insn "@aarch64_vec_duplicate_vq<mode>_be"
2453 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
2454 (vec_duplicate:SVE_FULL
2456 (match_operand:<V128> 1 "register_operand" "w")
2457 (match_operand 2 "descending_int_parallel"))))]
2460 && known_eq (INTVAL (XVECEXP (operands[2], 0, 0)),
2461 GET_MODE_NUNITS (<V128>mode) - 1)"
2463 operands[1] = gen_rtx_REG (<MODE>mode, REGNO (operands[1]));
2464 return "dup\t%0.q, %1.q[0]";
2468 ;; This is used for vec_duplicate<mode>s from memory, but can also
2469 ;; be used by combine to optimize selects of a a vec_duplicate<mode>
2471 (define_insn "sve_ld1r<mode>"
2472 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
2474 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2475 (vec_duplicate:SVE_ALL
2476 (match_operand:<VEL> 2 "aarch64_sve_ld1r_operand" "Uty"))
2477 (match_operand:SVE_ALL 3 "aarch64_simd_imm_zero")]
2480 "ld1r<Vesize>\t%0.<Vetype>, %1/z, %2"
2483 ;; Load 128 bits from memory under predicate control and duplicate to
2485 (define_insn "@aarch64_sve_ld1rq<mode>"
2486 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
2488 [(match_operand:<VPRED> 2 "register_operand" "Upl")
2489 (match_operand:<V128> 1 "aarch64_sve_ld1rq_operand" "UtQ")]
2493 operands[1] = gen_rtx_MEM (<VEL>mode, XEXP (operands[1], 0));
2494 return "ld1rq<Vesize>\t%0.<Vetype>, %2/z, %1";
2498 ;; -------------------------------------------------------------------------
2499 ;; ---- [INT,FP] Initialize from individual elements
2500 ;; -------------------------------------------------------------------------
2503 ;; -------------------------------------------------------------------------
2505 (define_expand "vec_init<mode><Vel>"
2506 [(match_operand:SVE_FULL 0 "register_operand")
2507 (match_operand 1 "")]
2510 aarch64_sve_expand_vector_init (operands[0], operands[1]);
2515 ;; Shift an SVE vector left and insert a scalar into element 0.
2516 (define_insn "vec_shl_insert_<mode>"
2517 [(set (match_operand:SVE_FULL 0 "register_operand" "=?w, w, ??&w, ?&w")
2519 [(match_operand:SVE_FULL 1 "register_operand" "0, 0, w, w")
2520 (match_operand:<VEL> 2 "aarch64_reg_or_zero" "rZ, w, rZ, w")]
2524 insr\t%0.<Vetype>, %<vwcore>2
2525 insr\t%0.<Vetype>, %<Vetype>2
2526 movprfx\t%0, %1\;insr\t%0.<Vetype>, %<vwcore>2
2527 movprfx\t%0, %1\;insr\t%0.<Vetype>, %<Vetype>2"
2528 [(set_attr "movprfx" "*,*,yes,yes")]
2531 ;; -------------------------------------------------------------------------
2532 ;; ---- [INT] Linear series
2533 ;; -------------------------------------------------------------------------
2536 ;; -------------------------------------------------------------------------
2538 (define_insn "vec_series<mode>"
2539 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w")
2541 (match_operand:<VEL> 1 "aarch64_sve_index_operand" "Usi, r, r")
2542 (match_operand:<VEL> 2 "aarch64_sve_index_operand" "r, Usi, r")))]
2545 index\t%0.<Vctype>, #%1, %<vccore>2
2546 index\t%0.<Vctype>, %<vccore>1, #%2
2547 index\t%0.<Vctype>, %<vccore>1, %<vccore>2"
2550 ;; Optimize {x, x, x, x, ...} + {0, n, 2*n, 3*n, ...} if n is in range
2551 ;; of an INDEX instruction.
2552 (define_insn "*vec_series<mode>_plus"
2553 [(set (match_operand:SVE_I 0 "register_operand" "=w")
2555 (vec_duplicate:SVE_I
2556 (match_operand:<VEL> 1 "register_operand" "r"))
2557 (match_operand:SVE_I 2 "immediate_operand")))]
2558 "TARGET_SVE && aarch64_check_zero_based_sve_index_immediate (operands[2])"
2560 operands[2] = aarch64_check_zero_based_sve_index_immediate (operands[2]);
2561 return "index\t%0.<Vctype>, %<vccore>1, #%2";
2565 ;; -------------------------------------------------------------------------
2566 ;; ---- [PRED] Duplicate element
2567 ;; -------------------------------------------------------------------------
2568 ;; The patterns in this section are synthetic.
2569 ;; -------------------------------------------------------------------------
2571 ;; Implement a predicate broadcast by shifting the low bit of the scalar
2572 ;; input into the top bit and using a WHILELO. An alternative would be to
2573 ;; duplicate the input and do a compare with zero.
2574 (define_expand "vec_duplicate<mode>"
2575 [(set (match_operand:PRED_ALL 0 "register_operand")
2576 (vec_duplicate:PRED_ALL (match_operand:QI 1 "register_operand")))]
2579 rtx tmp = gen_reg_rtx (DImode);
2580 rtx op1 = gen_lowpart (DImode, operands[1]);
2581 emit_insn (gen_ashldi3 (tmp, op1, gen_int_mode (63, DImode)));
2582 emit_insn (gen_while_ultdi<mode> (operands[0], const0_rtx, tmp));
2587 ;; =========================================================================
2588 ;; == Vector decomposition
2589 ;; =========================================================================
2591 ;; -------------------------------------------------------------------------
2592 ;; ---- [INT,FP] Extract index
2593 ;; -------------------------------------------------------------------------
2595 ;; - DUP (Advanced SIMD)
2598 ;; - ST1 (Advanced SIMD)
2599 ;; - UMOV (Advanced SIMD)
2600 ;; -------------------------------------------------------------------------
2602 (define_expand "vec_extract<mode><Vel>"
2603 [(set (match_operand:<VEL> 0 "register_operand")
2605 (match_operand:SVE_FULL 1 "register_operand")
2606 (parallel [(match_operand:SI 2 "nonmemory_operand")])))]
2610 if (poly_int_rtx_p (operands[2], &val)
2611 && known_eq (val, GET_MODE_NUNITS (<MODE>mode) - 1))
2613 /* The last element can be extracted with a LASTB and a false
2615 rtx sel = aarch64_pfalse_reg (<VPRED>mode);
2616 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1]));
2619 if (!CONST_INT_P (operands[2]))
2621 /* Create an index with operand[2] as the base and -1 as the step.
2622 It will then be zero for the element we care about. */
2623 rtx index = gen_lowpart (<VEL_INT>mode, operands[2]);
2624 index = force_reg (<VEL_INT>mode, index);
2625 rtx series = gen_reg_rtx (<V_INT_EQUIV>mode);
2626 emit_insn (gen_vec_series<v_int_equiv> (series, index, constm1_rtx));
2628 /* Get a predicate that is true for only that element. */
2629 rtx zero = CONST0_RTX (<V_INT_EQUIV>mode);
2630 rtx cmp = gen_rtx_EQ (<V_INT_EQUIV>mode, series, zero);
2631 rtx sel = gen_reg_rtx (<VPRED>mode);
2632 emit_insn (gen_vec_cmp<v_int_equiv><vpred> (sel, cmp, series, zero));
2634 /* Select the element using LASTB. */
2635 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1]));
2641 ;; Extract element zero. This is a special case because we want to force
2642 ;; the registers to be the same for the second alternative, and then
2643 ;; split the instruction into nothing after RA.
2644 (define_insn_and_split "*vec_extract<mode><Vel>_0"
2645 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2647 (match_operand:SVE_FULL 1 "register_operand" "w, 0, w")
2648 (parallel [(const_int 0)])))]
2651 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
2652 switch (which_alternative)
2655 return "umov\\t%<vwcore>0, %1.<Vetype>[0]";
2659 return "st1\\t{%1.<Vetype>}[0], %0";
2664 "&& reload_completed
2665 && REG_P (operands[0])
2666 && REGNO (operands[0]) == REGNO (operands[1])"
2669 emit_note (NOTE_INSN_DELETED);
2672 [(set_attr "type" "neon_to_gp_q, untyped, neon_store1_one_lane_q")]
2675 ;; Extract an element from the Advanced SIMD portion of the register.
2676 ;; We don't just reuse the aarch64-simd.md pattern because we don't
2677 ;; want any change in lane number on big-endian targets.
2678 (define_insn "*vec_extract<mode><Vel>_v128"
2679 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2681 (match_operand:SVE_FULL 1 "register_operand" "w, w, w")
2682 (parallel [(match_operand:SI 2 "const_int_operand")])))]
2684 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 1, 15)"
2686 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
2687 switch (which_alternative)
2690 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2692 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2694 return "st1\\t{%1.<Vetype>}[%2], %0";
2699 [(set_attr "type" "neon_to_gp_q, neon_dup_q, neon_store1_one_lane_q")]
2702 ;; Extract an element in the range of DUP. This pattern allows the
2703 ;; source and destination to be different.
2704 (define_insn "*vec_extract<mode><Vel>_dup"
2705 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2707 (match_operand:SVE_FULL 1 "register_operand" "w")
2708 (parallel [(match_operand:SI 2 "const_int_operand")])))]
2710 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 16, 63)"
2712 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
2713 return "dup\t%0.<Vetype>, %1.<Vetype>[%2]";
2717 ;; Extract an element outside the range of DUP. This pattern requires the
2718 ;; source and destination to be the same.
2719 (define_insn "*vec_extract<mode><Vel>_ext"
2720 [(set (match_operand:<VEL> 0 "register_operand" "=w, ?&w")
2722 (match_operand:SVE_FULL 1 "register_operand" "0, w")
2723 (parallel [(match_operand:SI 2 "const_int_operand")])))]
2724 "TARGET_SVE && INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode) >= 64"
2726 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
2727 operands[2] = GEN_INT (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode));
2728 return (which_alternative == 0
2729 ? "ext\t%0.b, %0.b, %0.b, #%2"
2730 : "movprfx\t%0, %1\;ext\t%0.b, %0.b, %1.b, #%2");
2732 [(set_attr "movprfx" "*,yes")]
2735 ;; -------------------------------------------------------------------------
2736 ;; ---- [INT,FP] Extract active element
2737 ;; -------------------------------------------------------------------------
2741 ;; -------------------------------------------------------------------------
2743 ;; Extract the last active element of operand 1 into operand 0.
2744 ;; If no elements are active, extract the last inactive element instead.
2745 (define_insn "@extract_<last_op>_<mode>"
2746 [(set (match_operand:<VEL> 0 "register_operand" "=?r, w")
2748 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2749 (match_operand:SVE_FULL 2 "register_operand" "w, w")]
2753 last<ab>\t%<vwcore>0, %1, %2.<Vetype>
2754 last<ab>\t%<Vetype>0, %1, %2.<Vetype>"
2757 ;; -------------------------------------------------------------------------
2758 ;; ---- [PRED] Extract index
2759 ;; -------------------------------------------------------------------------
2760 ;; The patterns in this section are synthetic.
2761 ;; -------------------------------------------------------------------------
2763 ;; Handle extractions from a predicate by converting to an integer vector
2764 ;; and extracting from there.
2765 (define_expand "vec_extract<vpred><Vel>"
2766 [(match_operand:<VEL> 0 "register_operand")
2767 (match_operand:<VPRED> 1 "register_operand")
2768 (match_operand:SI 2 "nonmemory_operand")
2769 ;; Dummy operand to which we can attach the iterator.
2770 (reg:SVE_FULL_I V0_REGNUM)]
2773 rtx tmp = gen_reg_rtx (<MODE>mode);
2774 emit_insn (gen_vcond_mask_<mode><vpred> (tmp, operands[1],
2775 CONST1_RTX (<MODE>mode),
2776 CONST0_RTX (<MODE>mode)));
2777 emit_insn (gen_vec_extract<mode><Vel> (operands[0], tmp, operands[2]));
2782 ;; =========================================================================
2783 ;; == Unary arithmetic
2784 ;; =========================================================================
2786 ;; -------------------------------------------------------------------------
2787 ;; ---- [INT] General unary arithmetic corresponding to rtx codes
2788 ;; -------------------------------------------------------------------------
2793 ;; - CNT (= popcount)
2796 ;; -------------------------------------------------------------------------
2798 ;; Unpredicated integer unary arithmetic.
2799 (define_expand "<optab><mode>2"
2800 [(set (match_operand:SVE_FULL_I 0 "register_operand")
2803 (SVE_INT_UNARY:SVE_FULL_I
2804 (match_operand:SVE_FULL_I 1 "register_operand"))]
2808 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
2812 ;; Integer unary arithmetic predicated with a PTRUE.
2813 (define_insn "@aarch64_pred_<optab><mode>"
2814 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
2816 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2817 (SVE_INT_UNARY:SVE_FULL_I
2818 (match_operand:SVE_FULL_I 2 "register_operand" "w"))]
2821 "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2824 ;; Predicated integer unary arithmetic with merging.
2825 (define_expand "@cond_<optab><mode>"
2826 [(set (match_operand:SVE_FULL_I 0 "register_operand")
2828 [(match_operand:<VPRED> 1 "register_operand")
2829 (SVE_INT_UNARY:SVE_FULL_I
2830 (match_operand:SVE_FULL_I 2 "register_operand"))
2831 (match_operand:SVE_FULL_I 3 "aarch64_simd_reg_or_zero")]
2836 ;; Predicated integer unary arithmetic, merging with the first input.
2837 (define_insn "*cond_<optab><mode>_2"
2838 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
2840 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2841 (SVE_INT_UNARY:SVE_FULL_I
2842 (match_operand:SVE_FULL_I 2 "register_operand" "0, w"))
2847 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>
2848 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2849 [(set_attr "movprfx" "*,yes")]
2852 ;; Predicated integer unary arithmetic, merging with an independent value.
2854 ;; The earlyclobber isn't needed for the first alternative, but omitting
2855 ;; it would only help the case in which operands 2 and 3 are the same,
2856 ;; which is handled above rather than here. Marking all the alternatives
2857 ;; as earlyclobber helps to make the instruction more regular to the
2858 ;; register allocator.
2859 (define_insn "*cond_<optab><mode>_any"
2860 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, ?&w, ?&w")
2862 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2863 (SVE_INT_UNARY:SVE_FULL_I
2864 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, w"))
2865 (match_operand:SVE_FULL_I 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
2867 "TARGET_SVE && !rtx_equal_p (operands[2], operands[3])"
2869 <sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2870 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2871 movprfx\t%0, %3\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2872 [(set_attr "movprfx" "*,yes,yes")]
2875 ;; -------------------------------------------------------------------------
2876 ;; ---- [INT] General unary arithmetic corresponding to unspecs
2877 ;; -------------------------------------------------------------------------
2883 ;; -------------------------------------------------------------------------
2885 ;; Predicated integer unary operations.
2886 (define_insn "@aarch64_pred_<optab><mode>"
2887 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
2889 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2891 [(match_operand:SVE_FULL_I 2 "register_operand" "w")]
2894 "TARGET_SVE && <elem_bits> >= <min_elem_bits>"
2895 "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2898 ;; Predicated integer unary operations with merging.
2899 (define_insn "@cond_<optab><mode>"
2900 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w, ?&w")
2902 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2904 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w, w")]
2906 (match_operand:SVE_FULL_I 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
2908 "TARGET_SVE && <elem_bits> >= <min_elem_bits>"
2910 <sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2911 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2912 movprfx\t%0, %3\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2913 [(set_attr "movprfx" "*,yes,yes")]
2916 ;; -------------------------------------------------------------------------
2917 ;; ---- [INT] Sign and zero extension
2918 ;; -------------------------------------------------------------------------
2926 ;; -------------------------------------------------------------------------
2928 ;; Unpredicated sign and zero extension from a narrower mode.
2929 (define_expand "<optab><SVE_PARTIAL_I:mode><SVE_HSDI:mode>2"
2930 [(set (match_operand:SVE_HSDI 0 "register_operand")
2933 (ANY_EXTEND:SVE_HSDI
2934 (match_operand:SVE_PARTIAL_I 1 "register_operand"))]
2936 "TARGET_SVE && (~<SVE_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
2938 operands[2] = aarch64_ptrue_reg (<SVE_HSDI:VPRED>mode);
2942 ;; Predicated sign and zero extension from a narrower mode.
2943 (define_insn "*<optab><SVE_PARTIAL_I:mode><SVE_HSDI:mode>2"
2944 [(set (match_operand:SVE_HSDI 0 "register_operand" "=w")
2946 [(match_operand:<SVE_HSDI:VPRED> 1 "register_operand" "Upl")
2947 (ANY_EXTEND:SVE_HSDI
2948 (match_operand:SVE_PARTIAL_I 2 "register_operand" "w"))]
2950 "TARGET_SVE && (~<SVE_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
2951 "<su>xt<SVE_PARTIAL_I:Vesize>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>"
2954 ;; Predicated truncate-and-sign-extend operations.
2955 (define_insn "@aarch64_pred_sxt<SVE_FULL_HSDI:mode><SVE_PARTIAL_I:mode>"
2956 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
2957 (unspec:SVE_FULL_HSDI
2958 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl")
2959 (sign_extend:SVE_FULL_HSDI
2960 (truncate:SVE_PARTIAL_I
2961 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")))]
2964 && (~<SVE_FULL_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
2965 "sxt<SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>"
2968 ;; Predicated truncate-and-sign-extend operations with merging.
2969 (define_insn "@aarch64_cond_sxt<SVE_FULL_HSDI:mode><SVE_PARTIAL_I:mode>"
2970 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w, ?&w")
2971 (unspec:SVE_FULL_HSDI
2972 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl, Upl, Upl")
2973 (sign_extend:SVE_FULL_HSDI
2974 (truncate:SVE_PARTIAL_I
2975 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w, w")))
2976 (match_operand:SVE_FULL_HSDI 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
2979 && (~<SVE_FULL_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
2981 sxt<SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>
2982 movprfx\t%0.<SVE_FULL_HSDI:Vetype>, %1/z, %2.<SVE_FULL_HSDI:Vetype>\;sxt<SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>
2983 movprfx\t%0, %3\;sxt<SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>"
2984 [(set_attr "movprfx" "*,yes,yes")]
2987 ;; Predicated truncate-and-zero-extend operations, merging with the
2990 ;; The canonical form of this operation is an AND of a constant rather
2991 ;; than (zero_extend (truncate ...)).
2992 (define_insn "*cond_uxt<mode>_2"
2993 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
2995 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2997 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
2998 (match_operand:SVE_FULL_I 3 "aarch64_sve_uxt_immediate"))
3003 uxt%e3\t%0.<Vetype>, %1/m, %0.<Vetype>
3004 movprfx\t%0, %2\;uxt%e3\t%0.<Vetype>, %1/m, %2.<Vetype>"
3005 [(set_attr "movprfx" "*,yes")]
3008 ;; Predicated truncate-and-zero-extend operations, merging with an
3009 ;; independent value.
3011 ;; The earlyclobber isn't needed for the first alternative, but omitting
3012 ;; it would only help the case in which operands 2 and 4 are the same,
3013 ;; which is handled above rather than here. Marking all the alternatives
3014 ;; as early-clobber helps to make the instruction more regular to the
3015 ;; register allocator.
3016 (define_insn "*cond_uxt<mode>_any"
3017 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, ?&w, ?&w")
3019 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
3021 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, w")
3022 (match_operand:SVE_FULL_I 3 "aarch64_sve_uxt_immediate"))
3023 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "0, Dz, w")]
3025 "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
3027 uxt%e3\t%0.<Vetype>, %1/m, %2.<Vetype>
3028 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;uxt%e3\t%0.<Vetype>, %1/m, %2.<Vetype>
3029 movprfx\t%0, %4\;uxt%e3\t%0.<Vetype>, %1/m, %2.<Vetype>"
3030 [(set_attr "movprfx" "*,yes,yes")]
3033 ;; -------------------------------------------------------------------------
3034 ;; ---- [INT] Truncation
3035 ;; -------------------------------------------------------------------------
3036 ;; The patterns in this section are synthetic.
3037 ;; -------------------------------------------------------------------------
3039 ;; Truncate to a partial SVE vector from either a full vector or a
3040 ;; wider partial vector. This is a no-op, because we can just ignore
3041 ;; the unused upper bits of the source.
3042 (define_insn_and_split "trunc<SVE_HSDI:mode><SVE_PARTIAL_I:mode>2"
3043 [(set (match_operand:SVE_PARTIAL_I 0 "register_operand" "=w")
3044 (truncate:SVE_PARTIAL_I
3045 (match_operand:SVE_HSDI 1 "register_operand" "w")))]
3046 "TARGET_SVE && (~<SVE_HSDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
3048 "&& reload_completed"
3049 [(set (match_dup 0) (match_dup 1))]
3051 operands[1] = aarch64_replace_reg_mode (operands[1],
3052 <SVE_PARTIAL_I:MODE>mode);
3056 ;; -------------------------------------------------------------------------
3057 ;; ---- [INT] Logical inverse
3058 ;; -------------------------------------------------------------------------
3061 ;; -------------------------------------------------------------------------
3063 ;; Predicated logical inverse.
3064 (define_expand "@aarch64_pred_cnot<mode>"
3065 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3068 [(match_operand:<VPRED> 1 "register_operand")
3069 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
3071 (match_operand:SVE_FULL_I 3 "register_operand")
3079 operands[4] = CONST0_RTX (<MODE>mode);
3080 operands[5] = CONST1_RTX (<MODE>mode);
3084 (define_insn "*cnot<mode>"
3085 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
3088 [(match_operand:<VPRED> 1 "register_operand" "Upl")
3089 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
3091 (match_operand:SVE_FULL_I 2 "register_operand" "w")
3092 (match_operand:SVE_FULL_I 3 "aarch64_simd_imm_zero"))]
3094 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_one")
3098 "cnot\t%0.<Vetype>, %1/m, %2.<Vetype>"
3101 ;; Predicated logical inverse with merging.
3102 (define_expand "@cond_cnot<mode>"
3103 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3105 [(match_operand:<VPRED> 1 "register_operand")
3109 (const_int SVE_KNOWN_PTRUE)
3111 (match_operand:SVE_FULL_I 2 "register_operand")
3117 (match_operand:SVE_FULL_I 3 "aarch64_simd_reg_or_zero")]
3121 operands[4] = CONSTM1_RTX (<VPRED>mode);
3122 operands[5] = CONST0_RTX (<MODE>mode);
3123 operands[6] = CONST1_RTX (<MODE>mode);
3127 ;; Predicated logical inverse, merging with the first input.
3128 (define_insn_and_rewrite "*cond_cnot<mode>_2"
3129 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3131 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3132 ;; Logical inverse of operand 2 (as above).
3136 (const_int SVE_KNOWN_PTRUE)
3138 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
3139 (match_operand:SVE_FULL_I 3 "aarch64_simd_imm_zero"))]
3141 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_one")
3148 cnot\t%0.<Vetype>, %1/m, %0.<Vetype>
3149 movprfx\t%0, %2\;cnot\t%0.<Vetype>, %1/m, %2.<Vetype>"
3150 "&& !CONSTANT_P (operands[5])"
3152 operands[5] = CONSTM1_RTX (<VPRED>mode);
3154 [(set_attr "movprfx" "*,yes")]
3157 ;; Predicated logical inverse, merging with an independent value.
3159 ;; The earlyclobber isn't needed for the first alternative, but omitting
3160 ;; it would only help the case in which operands 2 and 6 are the same,
3161 ;; which is handled above rather than here. Marking all the alternatives
3162 ;; as earlyclobber helps to make the instruction more regular to the
3163 ;; register allocator.
3164 (define_insn_and_rewrite "*cond_cnot<mode>_any"
3165 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, ?&w, ?&w")
3167 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
3168 ;; Logical inverse of operand 2 (as above).
3172 (const_int SVE_KNOWN_PTRUE)
3174 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, w")
3175 (match_operand:SVE_FULL_I 3 "aarch64_simd_imm_zero"))]
3177 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_one")
3180 (match_operand:SVE_FULL_I 6 "aarch64_simd_reg_or_zero" "0, Dz, w")]
3182 "TARGET_SVE && !rtx_equal_p (operands[2], operands[6])"
3184 cnot\t%0.<Vetype>, %1/m, %2.<Vetype>
3185 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;cnot\t%0.<Vetype>, %1/m, %2.<Vetype>
3186 movprfx\t%0, %6\;cnot\t%0.<Vetype>, %1/m, %2.<Vetype>"
3187 "&& !CONSTANT_P (operands[5])"
3189 operands[5] = CONSTM1_RTX (<VPRED>mode);
3191 [(set_attr "movprfx" "*,yes,yes")]
3194 ;; -------------------------------------------------------------------------
3195 ;; ---- [FP<-INT] General unary arithmetic that maps to unspecs
3196 ;; -------------------------------------------------------------------------
3199 ;; -------------------------------------------------------------------------
3201 ;; Unpredicated unary operations that take an integer and return a float.
3202 (define_insn "@aarch64_sve_<optab><mode>"
3203 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
3205 [(match_operand:<V_INT_EQUIV> 1 "register_operand" "w")]
3208 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>"
3211 ;; -------------------------------------------------------------------------
3212 ;; ---- [FP] General unary arithmetic corresponding to unspecs
3213 ;; -------------------------------------------------------------------------
3228 ;; -------------------------------------------------------------------------
3230 ;; Unpredicated floating-point unary operations.
3231 (define_insn "@aarch64_sve_<optab><mode>"
3232 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
3234 [(match_operand:SVE_FULL_F 1 "register_operand" "w")]
3237 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>"
3240 ;; Unpredicated floating-point unary operations.
3241 (define_expand "<optab><mode>2"
3242 [(set (match_operand:SVE_FULL_F 0 "register_operand")
3245 (const_int SVE_RELAXED_GP)
3246 (match_operand:SVE_FULL_F 1 "register_operand")]
3247 SVE_COND_FP_UNARY))]
3250 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
3254 ;; Predicated floating-point unary operations.
3255 (define_insn "@aarch64_pred_<optab><mode>"
3256 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
3258 [(match_operand:<VPRED> 1 "register_operand" "Upl")
3259 (match_operand:SI 3 "aarch64_sve_gp_strictness")
3260 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
3261 SVE_COND_FP_UNARY))]
3263 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
3266 ;; Predicated floating-point unary arithmetic with merging.
3267 (define_expand "@cond_<optab><mode>"
3268 [(set (match_operand:SVE_FULL_F 0 "register_operand")
3270 [(match_operand:<VPRED> 1 "register_operand")
3273 (const_int SVE_STRICT_GP)
3274 (match_operand:SVE_FULL_F 2 "register_operand")]
3276 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]
3281 ;; Predicated floating-point unary arithmetic, merging with the first input.
3282 (define_insn_and_rewrite "*cond_<optab><mode>_2"
3283 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
3285 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3288 (match_operand:SI 4 "aarch64_sve_gp_strictness")
3289 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")]
3293 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[3], operands[1])"
3295 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>
3296 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
3297 "&& !rtx_equal_p (operands[1], operands[3])"
3299 operands[3] = copy_rtx (operands[1]);
3301 [(set_attr "movprfx" "*,yes")]
3304 ;; Predicated floating-point unary arithmetic, merging with an independent
3307 ;; The earlyclobber isn't needed for the first alternative, but omitting
3308 ;; it would only help the case in which operands 2 and 3 are the same,
3309 ;; which is handled above rather than here. Marking all the alternatives
3310 ;; as earlyclobber helps to make the instruction more regular to the
3311 ;; register allocator.
3312 (define_insn_and_rewrite "*cond_<optab><mode>_any"
3313 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, ?&w, ?&w")
3315 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
3318 (match_operand:SI 5 "aarch64_sve_gp_strictness")
3319 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")]
3321 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
3324 && !rtx_equal_p (operands[2], operands[3])
3325 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
3327 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
3328 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
3329 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
3330 "&& !rtx_equal_p (operands[1], operands[4])"
3332 operands[4] = copy_rtx (operands[1]);
3334 [(set_attr "movprfx" "*,yes,yes")]
3337 ;; -------------------------------------------------------------------------
3338 ;; ---- [PRED] Inverse
3339 ;; -------------------------------------------------------------------------
3342 ;; -------------------------------------------------------------------------
3344 ;; Unpredicated predicate inverse.
3345 (define_expand "one_cmpl<mode>2"
3346 [(set (match_operand:PRED_ALL 0 "register_operand")
3348 (not:PRED_ALL (match_operand:PRED_ALL 1 "register_operand"))
3352 operands[2] = aarch64_ptrue_reg (<MODE>mode);
3356 ;; Predicated predicate inverse.
3357 (define_insn "*one_cmpl<mode>3"
3358 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
3360 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
3361 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
3363 "not\t%0.b, %1/z, %2.b"
3366 ;; =========================================================================
3367 ;; == Binary arithmetic
3368 ;; =========================================================================
3370 ;; -------------------------------------------------------------------------
3371 ;; ---- [INT] General binary arithmetic corresponding to rtx codes
3372 ;; -------------------------------------------------------------------------
3374 ;; - ADD (merging form only)
3375 ;; - AND (merging form only)
3376 ;; - ASR (merging form only)
3377 ;; - EOR (merging form only)
3378 ;; - LSL (merging form only)
3379 ;; - LSR (merging form only)
3381 ;; - ORR (merging form only)
3384 ;; - SUB (merging form only)
3387 ;; -------------------------------------------------------------------------
3389 ;; Unpredicated integer binary operations that have an immediate form.
3390 (define_expand "<optab><mode>3"
3391 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3394 (SVE_INT_BINARY_IMM:SVE_FULL_I
3395 (match_operand:SVE_FULL_I 1 "register_operand")
3396 (match_operand:SVE_FULL_I 2 "aarch64_sve_<sve_imm_con>_operand"))]
3400 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
3404 ;; Integer binary operations that have an immediate form, predicated
3405 ;; with a PTRUE. We don't actually need the predicate for the first
3406 ;; and third alternatives, but using Upa or X isn't likely to gain much
3407 ;; and would make the instruction seem less uniform to the register
3409 (define_insn_and_split "@aarch64_pred_<optab><mode>"
3410 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w, ?&w")
3412 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
3413 (SVE_INT_BINARY_IMM:SVE_FULL_I
3414 (match_operand:SVE_FULL_I 2 "register_operand" "%0, 0, w, w")
3415 (match_operand:SVE_FULL_I 3 "aarch64_sve_<sve_imm_con>_operand" "<sve_imm_con>, w, <sve_imm_con>, w"))]
3420 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3422 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3423 ; Split the unpredicated form after reload, so that we don't have
3424 ; the unnecessary PTRUE.
3425 "&& reload_completed
3426 && !register_operand (operands[3], <MODE>mode)"
3428 (SVE_INT_BINARY_IMM:SVE_FULL_I (match_dup 2) (match_dup 3)))]
3430 [(set_attr "movprfx" "*,*,yes,yes")]
3433 ;; Unpredicated binary operations with a constant (post-RA only).
3434 ;; These are generated by splitting a predicated instruction whose
3435 ;; predicate is unused.
3436 (define_insn "*post_ra_<optab><mode>3"
3437 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3438 (SVE_INT_BINARY_IMM:SVE_FULL_I
3439 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")
3440 (match_operand:SVE_FULL_I 2 "aarch64_sve_<sve_imm_con>_immediate")))]
3441 "TARGET_SVE && reload_completed"
3443 <sve_int_op>\t%0.<Vetype>, %0.<Vetype>, #%<sve_imm_prefix>2
3444 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %0.<Vetype>, #%<sve_imm_prefix>2"
3445 [(set_attr "movprfx" "*,yes")]
3448 ;; Predicated integer operations with merging.
3449 (define_expand "@cond_<optab><mode>"
3450 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3452 [(match_operand:<VPRED> 1 "register_operand")
3453 (SVE_INT_BINARY:SVE_FULL_I
3454 (match_operand:SVE_FULL_I 2 "register_operand")
3455 (match_operand:SVE_FULL_I 3 "<sve_pred_int_rhs2_operand>"))
3456 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
3461 ;; Predicated integer operations, merging with the first input.
3462 (define_insn "*cond_<optab><mode>_2"
3463 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3465 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3466 (SVE_INT_BINARY:SVE_FULL_I
3467 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
3468 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
3473 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3474 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3475 [(set_attr "movprfx" "*,yes")]
3478 ;; Predicated integer operations, merging with the second input.
3479 (define_insn "*cond_<optab><mode>_3"
3480 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3482 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3483 (SVE_INT_BINARY:SVE_FULL_I
3484 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
3485 (match_operand:SVE_FULL_I 3 "register_operand" "0, w"))
3490 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
3491 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
3492 [(set_attr "movprfx" "*,yes")]
3495 ;; Predicated integer operations, merging with an independent value.
3496 (define_insn_and_rewrite "*cond_<optab><mode>_any"
3497 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, ?&w")
3499 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
3500 (SVE_INT_BINARY:SVE_FULL_I
3501 (match_operand:SVE_FULL_I 2 "register_operand" "0, w, w, w, w")
3502 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w, w, w"))
3503 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
3506 && !rtx_equal_p (operands[2], operands[4])
3507 && !rtx_equal_p (operands[3], operands[4])"
3509 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3510 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
3511 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3512 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3514 "&& reload_completed
3515 && register_operand (operands[4], <MODE>mode)
3516 && !rtx_equal_p (operands[0], operands[4])"
3518 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
3519 operands[4], operands[1]));
3520 operands[4] = operands[2] = operands[0];
3522 [(set_attr "movprfx" "yes")]
3525 ;; -------------------------------------------------------------------------
3526 ;; ---- [INT] Addition
3527 ;; -------------------------------------------------------------------------
3539 ;; -------------------------------------------------------------------------
3541 (define_insn "add<mode>3"
3542 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w, ?w, ?w, w")
3544 (match_operand:SVE_I 1 "register_operand" "%0, 0, 0, w, w, w")
3545 (match_operand:SVE_I 2 "aarch64_sve_add_operand" "vsa, vsn, vsi, vsa, vsn, w")))]
3548 add\t%0.<Vetype>, %0.<Vetype>, #%D2
3549 sub\t%0.<Vetype>, %0.<Vetype>, #%N2
3550 * return aarch64_output_sve_vector_inc_dec (\"%0.<Vetype>\", operands[2]);
3551 movprfx\t%0, %1\;add\t%0.<Vetype>, %0.<Vetype>, #%D2
3552 movprfx\t%0, %1\;sub\t%0.<Vetype>, %0.<Vetype>, #%N2
3553 add\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
3554 [(set_attr "movprfx" "*,*,*,yes,yes,*")]
3557 ;; Merging forms are handled through SVE_INT_BINARY.
3559 ;; -------------------------------------------------------------------------
3560 ;; ---- [INT] Subtraction
3561 ;; -------------------------------------------------------------------------
3565 ;; -------------------------------------------------------------------------
3567 (define_insn "sub<mode>3"
3568 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w")
3570 (match_operand:SVE_FULL_I 1 "aarch64_sve_arith_operand" "w, vsa, vsa")
3571 (match_operand:SVE_FULL_I 2 "register_operand" "w, 0, w")))]
3574 sub\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>
3575 subr\t%0.<Vetype>, %0.<Vetype>, #%D1
3576 movprfx\t%0, %2\;subr\t%0.<Vetype>, %0.<Vetype>, #%D1"
3577 [(set_attr "movprfx" "*,*,yes")]
3580 ;; Merging forms are handled through SVE_INT_BINARY.
3582 ;; -------------------------------------------------------------------------
3583 ;; ---- [INT] Take address
3584 ;; -------------------------------------------------------------------------
3587 ;; -------------------------------------------------------------------------
3589 ;; An unshifted and unscaled ADR. This is functionally equivalent to an ADD,
3590 ;; but the svadrb intrinsics should preserve the user's choice.
3591 (define_insn "@aarch64_adr<mode>"
3592 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w")
3593 (unspec:SVE_FULL_SDI
3594 [(match_operand:SVE_FULL_SDI 1 "register_operand" "w")
3595 (match_operand:SVE_FULL_SDI 2 "register_operand" "w")]
3598 "adr\t%0.<Vetype>, [%1.<Vetype>, %2.<Vetype>]"
3601 ;; Same, but with the offset being sign-extended from the low 32 bits.
3602 (define_insn_and_rewrite "*aarch64_adr_sxtw"
3603 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
3605 [(match_operand:VNx2DI 1 "register_operand" "w")
3610 (match_operand:VNx2DI 2 "register_operand" "w")))]
3614 "adr\t%0.d, [%1.d, %2.d, sxtw]"
3615 "&& !CONSTANT_P (operands[3])"
3617 operands[3] = CONSTM1_RTX (VNx2BImode);
3621 ;; Same, but with the offset being zero-extended from the low 32 bits.
3622 (define_insn "*aarch64_adr_uxtw_unspec"
3623 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
3625 [(match_operand:VNx2DI 1 "register_operand" "w")
3627 (match_operand:VNx2DI 2 "register_operand" "w")
3628 (match_operand:VNx2DI 3 "aarch64_sve_uxtw_immediate"))]
3631 "adr\t%0.d, [%1.d, %2.d, uxtw]"
3634 ;; Same, matching as a PLUS rather than unspec.
3635 (define_insn "*aarch64_adr_uxtw_and"
3636 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
3639 (match_operand:VNx2DI 2 "register_operand" "w")
3640 (match_operand:VNx2DI 3 "aarch64_sve_uxtw_immediate"))
3641 (match_operand:VNx2DI 1 "register_operand" "w")))]
3643 "adr\t%0.d, [%1.d, %2.d, uxtw]"
3646 ;; ADR with a nonzero shift.
3647 (define_expand "@aarch64_adr<mode>_shift"
3648 [(set (match_operand:SVE_FULL_SDI 0 "register_operand")
3650 (unspec:SVE_FULL_SDI
3652 (ashift:SVE_FULL_SDI
3653 (match_operand:SVE_FULL_SDI 2 "register_operand")
3654 (match_operand:SVE_FULL_SDI 3 "const_1_to_3_operand"))]
3656 (match_operand:SVE_FULL_SDI 1 "register_operand")))]
3659 operands[4] = CONSTM1_RTX (<VPRED>mode);
3663 (define_insn_and_rewrite "*aarch64_adr<mode>_shift"
3664 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w")
3666 (unspec:SVE_FULL_SDI
3668 (ashift:SVE_FULL_SDI
3669 (match_operand:SVE_FULL_SDI 2 "register_operand" "w")
3670 (match_operand:SVE_FULL_SDI 3 "const_1_to_3_operand"))]
3672 (match_operand:SVE_FULL_SDI 1 "register_operand" "w")))]
3674 "adr\t%0.<Vetype>, [%1.<Vetype>, %2.<Vetype>, lsl %3]"
3675 "&& !CONSTANT_P (operands[4])"
3677 operands[4] = CONSTM1_RTX (<VPRED>mode);
3681 ;; Same, but with the index being sign-extended from the low 32 bits.
3682 (define_insn_and_rewrite "*aarch64_adr_shift_sxtw"
3683 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
3692 (match_operand:VNx2DI 2 "register_operand" "w")))]
3694 (match_operand:VNx2DI 3 "const_1_to_3_operand"))]
3696 (match_operand:VNx2DI 1 "register_operand" "w")))]
3698 "adr\t%0.d, [%1.d, %2.d, sxtw %3]"
3699 "&& (!CONSTANT_P (operands[4]) || !CONSTANT_P (operands[5]))"
3701 operands[5] = operands[4] = CONSTM1_RTX (VNx2BImode);
3705 ;; Same, but with the index being zero-extended from the low 32 bits.
3706 (define_insn_and_rewrite "*aarch64_adr_shift_uxtw"
3707 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
3713 (match_operand:VNx2DI 2 "register_operand" "w")
3714 (match_operand:VNx2DI 4 "aarch64_sve_uxtw_immediate"))
3715 (match_operand:VNx2DI 3 "const_1_to_3_operand"))]
3717 (match_operand:VNx2DI 1 "register_operand" "w")))]
3719 "adr\t%0.d, [%1.d, %2.d, uxtw %3]"
3720 "&& !CONSTANT_P (operands[5])"
3722 operands[5] = CONSTM1_RTX (VNx2BImode);
3726 ;; -------------------------------------------------------------------------
3727 ;; ---- [INT] Absolute difference
3728 ;; -------------------------------------------------------------------------
3732 ;; -------------------------------------------------------------------------
3734 ;; Unpredicated integer absolute difference.
3735 (define_expand "<su>abd<mode>_3"
3736 [(use (match_operand:SVE_FULL_I 0 "register_operand"))
3738 (match_operand:SVE_FULL_I 1 "register_operand")
3739 (match_operand:SVE_FULL_I 2 "register_operand"))]
3742 rtx pred = aarch64_ptrue_reg (<VPRED>mode);
3743 emit_insn (gen_aarch64_pred_<su>abd<mode> (operands[0], pred, operands[1],
3749 ;; Predicated integer absolute difference.
3750 (define_insn "@aarch64_pred_<su>abd<mode>"
3751 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3753 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3756 (match_operand:SVE_FULL_I 2 "register_operand" "%0, w")
3757 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
3758 (<max_opp>:SVE_FULL_I
3764 <su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3765 movprfx\t%0, %2\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3766 [(set_attr "movprfx" "*,yes")]
3769 (define_expand "@aarch64_cond_<su>abd<mode>"
3770 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3772 [(match_operand:<VPRED> 1 "register_operand")
3777 (match_operand:SVE_FULL_I 2 "register_operand")
3778 (match_operand:SVE_FULL_I 3 "register_operand"))]
3782 (<max_opp>:SVE_FULL_I
3786 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
3790 if (rtx_equal_p (operands[3], operands[4]))
3791 std::swap (operands[2], operands[3]);
3794 ;; Predicated integer absolute difference, merging with the first input.
3795 (define_insn_and_rewrite "*aarch64_cond_<su>abd<mode>_2"
3796 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3798 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3803 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
3804 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))]
3808 (<max_opp>:SVE_FULL_I
3816 <su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3817 movprfx\t%0, %2\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3818 "&& (!CONSTANT_P (operands[4]) || !CONSTANT_P (operands[5]))"
3820 operands[4] = operands[5] = CONSTM1_RTX (<VPRED>mode);
3822 [(set_attr "movprfx" "*,yes")]
3825 ;; Predicated integer absolute difference, merging with an independent value.
3826 (define_insn_and_rewrite "*aarch64_cond_<su>abd<mode>_any"
3827 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, ?&w")
3829 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
3834 (match_operand:SVE_FULL_I 2 "register_operand" "0, w, w, w, w")
3835 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w, w, w"))]
3839 (<max_opp>:SVE_FULL_I
3843 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
3846 && !rtx_equal_p (operands[2], operands[4])
3847 && !rtx_equal_p (operands[3], operands[4])"
3849 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3850 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
3851 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3852 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<su>abd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3856 if (!CONSTANT_P (operands[5]) || !CONSTANT_P (operands[6]))
3857 operands[5] = operands[6] = CONSTM1_RTX (<VPRED>mode);
3858 else if (reload_completed
3859 && register_operand (operands[4], <MODE>mode)
3860 && !rtx_equal_p (operands[0], operands[4]))
3862 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
3863 operands[4], operands[1]));
3864 operands[4] = operands[2] = operands[0];
3869 [(set_attr "movprfx" "yes")]
3872 ;; -------------------------------------------------------------------------
3873 ;; ---- [INT] Saturating addition and subtraction
3874 ;; -------------------------------------------------------------------------
3879 ;; -------------------------------------------------------------------------
3881 ;; Unpredicated saturating signed addition and subtraction.
3882 (define_insn "@aarch64_<su_optab><optab><mode>"
3883 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w, ?&w, w")
3884 (SBINQOPS:SVE_FULL_I
3885 (match_operand:SVE_FULL_I 1 "register_operand" "0, 0, w, w, w")
3886 (match_operand:SVE_FULL_I 2 "aarch64_sve_sqadd_operand" "vsQ, vsS, vsQ, vsS, w")))]
3889 <binqops_op>\t%0.<Vetype>, %0.<Vetype>, #%D2
3890 <binqops_op_rev>\t%0.<Vetype>, %0.<Vetype>, #%N2
3891 movprfx\t%0, %1\;<binqops_op>\t%0.<Vetype>, %0.<Vetype>, #%D2
3892 movprfx\t%0, %1\;<binqops_op_rev>\t%0.<Vetype>, %0.<Vetype>, #%N2
3893 <binqops_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
3894 [(set_attr "movprfx" "*,*,yes,yes,*")]
3897 ;; Unpredicated saturating unsigned addition and subtraction.
3898 (define_insn "@aarch64_<su_optab><optab><mode>"
3899 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w, w")
3900 (UBINQOPS:SVE_FULL_I
3901 (match_operand:SVE_FULL_I 1 "register_operand" "0, w, w")
3902 (match_operand:SVE_FULL_I 2 "aarch64_sve_arith_operand" "vsa, vsa, w")))]
3905 <binqops_op>\t%0.<Vetype>, %0.<Vetype>, #%D2
3906 movprfx\t%0, %1\;<binqops_op>\t%0.<Vetype>, %0.<Vetype>, #%D2
3907 <binqops_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
3908 [(set_attr "movprfx" "*,yes,*")]
3911 ;; -------------------------------------------------------------------------
3912 ;; ---- [INT] Highpart multiplication
3913 ;; -------------------------------------------------------------------------
3917 ;; -------------------------------------------------------------------------
3919 ;; Unpredicated highpart multiplication.
3920 (define_expand "<su>mul<mode>3_highpart"
3921 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3925 [(match_operand:SVE_FULL_I 1 "register_operand")
3926 (match_operand:SVE_FULL_I 2 "register_operand")]
3931 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
3935 ;; Predicated highpart multiplication.
3936 (define_insn "@aarch64_pred_<optab><mode>"
3937 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3939 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3941 [(match_operand:SVE_FULL_I 2 "register_operand" "%0, w")
3942 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
3947 <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3948 movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3949 [(set_attr "movprfx" "*,yes")]
3952 ;; Predicated highpart multiplications with merging.
3953 (define_expand "@cond_<optab><mode>"
3954 [(set (match_operand:SVE_FULL_I 0 "register_operand")
3956 [(match_operand:<VPRED> 1 "register_operand")
3958 [(match_operand:SVE_FULL_I 2 "register_operand")
3959 (match_operand:SVE_FULL_I 3 "register_operand")]
3961 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
3965 /* Only target code is aware of these operations, so we don't need
3966 to handle the fully-general case. */
3967 gcc_assert (rtx_equal_p (operands[2], operands[4])
3968 || CONSTANT_P (operands[4]));
3971 ;; Predicated highpart multiplications, merging with the first input.
3972 (define_insn "*cond_<optab><mode>_2"
3973 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
3975 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3977 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
3978 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
3984 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
3985 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
3986 [(set_attr "movprfx" "*,yes")])
3988 ;; Predicated highpart multiplications, merging with zero.
3989 (define_insn "*cond_<optab><mode>_z"
3990 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w")
3992 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3994 [(match_operand:SVE_FULL_I 2 "register_operand" "%0, w")
3995 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
3997 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_zero")]
4001 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4002 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4003 [(set_attr "movprfx" "yes")])
4005 ;; -------------------------------------------------------------------------
4006 ;; ---- [INT] Division
4007 ;; -------------------------------------------------------------------------
4013 ;; -------------------------------------------------------------------------
4015 ;; Unpredicated integer division.
4016 (define_expand "<optab><mode>3"
4017 [(set (match_operand:SVE_FULL_SDI 0 "register_operand")
4018 (unspec:SVE_FULL_SDI
4020 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4021 (match_operand:SVE_FULL_SDI 1 "register_operand")
4022 (match_operand:SVE_FULL_SDI 2 "register_operand"))]
4026 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
4030 ;; Integer division predicated with a PTRUE.
4031 (define_insn "@aarch64_pred_<optab><mode>"
4032 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, w, ?&w")
4033 (unspec:SVE_FULL_SDI
4034 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
4035 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4036 (match_operand:SVE_FULL_SDI 2 "register_operand" "0, w, w")
4037 (match_operand:SVE_FULL_SDI 3 "register_operand" "w, 0, w"))]
4041 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4042 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4043 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4044 [(set_attr "movprfx" "*,*,yes")]
4047 ;; Predicated integer division with merging.
4048 (define_expand "@cond_<optab><mode>"
4049 [(set (match_operand:SVE_FULL_SDI 0 "register_operand")
4050 (unspec:SVE_FULL_SDI
4051 [(match_operand:<VPRED> 1 "register_operand")
4052 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4053 (match_operand:SVE_FULL_SDI 2 "register_operand")
4054 (match_operand:SVE_FULL_SDI 3 "register_operand"))
4055 (match_operand:SVE_FULL_SDI 4 "aarch64_simd_reg_or_zero")]
4060 ;; Predicated integer division, merging with the first input.
4061 (define_insn "*cond_<optab><mode>_2"
4062 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
4063 (unspec:SVE_FULL_SDI
4064 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4065 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4066 (match_operand:SVE_FULL_SDI 2 "register_operand" "0, w")
4067 (match_operand:SVE_FULL_SDI 3 "register_operand" "w, w"))
4072 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4073 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4074 [(set_attr "movprfx" "*,yes")]
4077 ;; Predicated integer division, merging with the second input.
4078 (define_insn "*cond_<optab><mode>_3"
4079 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
4080 (unspec:SVE_FULL_SDI
4081 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4082 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4083 (match_operand:SVE_FULL_SDI 2 "register_operand" "w, w")
4084 (match_operand:SVE_FULL_SDI 3 "register_operand" "0, w"))
4089 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4090 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
4091 [(set_attr "movprfx" "*,yes")]
4094 ;; Predicated integer division, merging with an independent value.
4095 (define_insn_and_rewrite "*cond_<optab><mode>_any"
4096 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=&w, &w, &w, &w, ?&w")
4097 (unspec:SVE_FULL_SDI
4098 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
4099 (SVE_INT_BINARY_SD:SVE_FULL_SDI
4100 (match_operand:SVE_FULL_SDI 2 "register_operand" "0, w, w, w, w")
4101 (match_operand:SVE_FULL_SDI 3 "register_operand" "w, 0, w, w, w"))
4102 (match_operand:SVE_FULL_SDI 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
4105 && !rtx_equal_p (operands[2], operands[4])
4106 && !rtx_equal_p (operands[3], operands[4])"
4108 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4109 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4110 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4111 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4113 "&& reload_completed
4114 && register_operand (operands[4], <MODE>mode)
4115 && !rtx_equal_p (operands[0], operands[4])"
4117 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4118 operands[4], operands[1]));
4119 operands[4] = operands[2] = operands[0];
4121 [(set_attr "movprfx" "yes")]
4124 ;; -------------------------------------------------------------------------
4125 ;; ---- [INT] Binary logical operations
4126 ;; -------------------------------------------------------------------------
4131 ;; -------------------------------------------------------------------------
4133 ;; Unpredicated integer binary logical operations.
4134 (define_insn "<optab><mode>3"
4135 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?w, w")
4137 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w, w")
4138 (match_operand:SVE_FULL_I 2 "aarch64_sve_logical_operand" "vsl, vsl, w")))]
4141 <logical>\t%0.<Vetype>, %0.<Vetype>, #%C2
4142 movprfx\t%0, %1\;<logical>\t%0.<Vetype>, %0.<Vetype>, #%C2
4143 <logical>\t%0.d, %1.d, %2.d"
4144 [(set_attr "movprfx" "*,yes,*")]
4147 ;; Merging forms are handled through SVE_INT_BINARY.
4149 ;; -------------------------------------------------------------------------
4150 ;; ---- [INT] Binary logical operations (inverted second input)
4151 ;; -------------------------------------------------------------------------
4154 ;; -------------------------------------------------------------------------
4156 ;; Unpredicated BIC.
4157 (define_expand "@aarch64_bic<mode>"
4158 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4162 (not:SVE_FULL_I (match_operand:SVE_FULL_I 2 "register_operand"))]
4164 (match_operand:SVE_FULL_I 1 "register_operand")))]
4167 operands[3] = CONSTM1_RTX (<VPRED>mode);
4172 (define_insn_and_rewrite "*bic<mode>3"
4173 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
4178 (match_operand:SVE_FULL_I 2 "register_operand" "w"))]
4180 (match_operand:SVE_FULL_I 1 "register_operand" "w")))]
4182 "bic\t%0.d, %1.d, %2.d"
4183 "&& !CONSTANT_P (operands[3])"
4185 operands[3] = CONSTM1_RTX (<VPRED>mode);
4189 ;; Predicated BIC with merging.
4190 (define_expand "@cond_bic<mode>"
4191 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4193 [(match_operand:<VPRED> 1 "register_operand")
4195 (not:SVE_FULL_I (match_operand:SVE_FULL_I 3 "register_operand"))
4196 (match_operand:SVE_FULL_I 2 "register_operand"))
4197 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
4202 ;; Predicated integer BIC, merging with the first input.
4203 (define_insn "*cond_bic<mode>_2"
4204 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
4206 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4209 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
4210 (match_operand:SVE_FULL_I 2 "register_operand" "0, w"))
4215 bic\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4216 movprfx\t%0, %2\;bic\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4217 [(set_attr "movprfx" "*,yes")]
4220 ;; Predicated integer BIC, merging with an independent value.
4221 (define_insn_and_rewrite "*cond_bic<mode>_any"
4222 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, ?&w")
4224 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
4227 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, w, w"))
4228 (match_operand:SVE_FULL_I 2 "register_operand" "0, w, w, w"))
4229 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, 0, w")]
4231 "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
4233 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;bic\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4234 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;bic\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4235 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;bic\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4237 "&& reload_completed
4238 && register_operand (operands[4], <MODE>mode)
4239 && !rtx_equal_p (operands[0], operands[4])"
4241 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4242 operands[4], operands[1]));
4243 operands[4] = operands[2] = operands[0];
4245 [(set_attr "movprfx" "yes")]
4248 ;; -------------------------------------------------------------------------
4249 ;; ---- [INT] Shifts (rounding towards -Inf)
4250 ;; -------------------------------------------------------------------------
4258 ;; -------------------------------------------------------------------------
4260 ;; Unpredicated shift by a scalar, which expands into one of the vector
4262 (define_expand "<ASHIFT:optab><mode>3"
4263 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4265 (match_operand:SVE_FULL_I 1 "register_operand")
4266 (match_operand:<VEL> 2 "general_operand")))]
4270 if (CONST_INT_P (operands[2]))
4272 amount = gen_const_vec_duplicate (<MODE>mode, operands[2]);
4273 if (!aarch64_sve_<lr>shift_operand (operands[2], <MODE>mode))
4274 amount = force_reg (<MODE>mode, amount);
4278 amount = gen_reg_rtx (<MODE>mode);
4279 emit_insn (gen_vec_duplicate<mode> (amount,
4280 convert_to_mode (<VEL>mode,
4283 emit_insn (gen_v<optab><mode>3 (operands[0], operands[1], amount));
4288 ;; Unpredicated shift by a vector.
4289 (define_expand "v<optab><mode>3"
4290 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4294 (match_operand:SVE_FULL_I 1 "register_operand")
4295 (match_operand:SVE_FULL_I 2 "aarch64_sve_<lr>shift_operand"))]
4299 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
4303 ;; Shift by a vector, predicated with a PTRUE. We don't actually need
4304 ;; the predicate for the first alternative, but using Upa or X isn't
4305 ;; likely to gain much and would make the instruction seem less uniform
4306 ;; to the register allocator.
4307 (define_insn_and_split "@aarch64_pred_<optab><mode>"
4308 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, w, ?&w")
4310 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
4312 (match_operand:SVE_FULL_I 2 "register_operand" "w, 0, w, w")
4313 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, 0, w"))]
4318 <shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4319 <shift>r\t%0.<Vetype>, %1/m, %3.<Vetype>, %2.<Vetype>
4320 movprfx\t%0, %2\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4321 "&& reload_completed
4322 && !register_operand (operands[3], <MODE>mode)"
4323 [(set (match_dup 0) (ASHIFT:SVE_FULL_I (match_dup 2) (match_dup 3)))]
4325 [(set_attr "movprfx" "*,*,*,yes")]
4328 ;; Unpredicated shift operations by a constant (post-RA only).
4329 ;; These are generated by splitting a predicated instruction whose
4330 ;; predicate is unused.
4331 (define_insn "*post_ra_v<optab><mode>3"
4332 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
4334 (match_operand:SVE_FULL_I 1 "register_operand" "w")
4335 (match_operand:SVE_FULL_I 2 "aarch64_simd_<lr>shift_imm")))]
4336 "TARGET_SVE && reload_completed"
4337 "<shift>\t%0.<Vetype>, %1.<Vetype>, #%2"
4340 ;; Predicated integer shift, merging with the first input.
4341 (define_insn "*cond_<optab><mode>_2_const"
4342 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
4344 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4346 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
4347 (match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm"))
4352 <shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4353 movprfx\t%0, %2\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
4354 [(set_attr "movprfx" "*,yes")]
4357 ;; Predicated integer shift, merging with an independent value.
4358 (define_insn_and_rewrite "*cond_<optab><mode>_any_const"
4359 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, &w, ?&w")
4361 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
4363 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, w")
4364 (match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm"))
4365 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, 0, w")]
4367 "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
4369 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4370 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4372 "&& reload_completed
4373 && register_operand (operands[4], <MODE>mode)
4374 && !rtx_equal_p (operands[0], operands[4])"
4376 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4377 operands[4], operands[1]));
4378 operands[4] = operands[2] = operands[0];
4380 [(set_attr "movprfx" "yes")]
4383 ;; Unpredicated shifts of narrow elements by 64-bit amounts.
4384 (define_insn "@aarch64_sve_<sve_int_op><mode>"
4385 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand" "=w")
4386 (unspec:SVE_FULL_BHSI
4387 [(match_operand:SVE_FULL_BHSI 1 "register_operand" "w")
4388 (match_operand:VNx2DI 2 "register_operand" "w")]
4391 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.d"
4394 ;; Merging predicated shifts of narrow elements by 64-bit amounts.
4395 (define_expand "@cond_<sve_int_op><mode>"
4396 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand")
4397 (unspec:SVE_FULL_BHSI
4398 [(match_operand:<VPRED> 1 "register_operand")
4399 (unspec:SVE_FULL_BHSI
4400 [(match_operand:SVE_FULL_BHSI 2 "register_operand")
4401 (match_operand:VNx2DI 3 "register_operand")]
4403 (match_operand:SVE_FULL_BHSI 4 "aarch64_simd_reg_or_zero")]
4408 ;; Predicated shifts of narrow elements by 64-bit amounts, merging with
4410 (define_insn "*cond_<sve_int_op><mode>_m"
4411 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand" "=w, ?&w")
4412 (unspec:SVE_FULL_BHSI
4413 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4414 (unspec:SVE_FULL_BHSI
4415 [(match_operand:SVE_FULL_BHSI 2 "register_operand" "0, w")
4416 (match_operand:VNx2DI 3 "register_operand" "w, w")]
4422 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.d
4423 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.d"
4424 [(set_attr "movprfx" "*, yes")])
4426 ;; Predicated shifts of narrow elements by 64-bit amounts, merging with zero.
4427 (define_insn "*cond_<sve_int_op><mode>_z"
4428 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand" "=&w, &w")
4429 (unspec:SVE_FULL_BHSI
4430 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4431 (unspec:SVE_FULL_BHSI
4432 [(match_operand:SVE_FULL_BHSI 2 "register_operand" "0, w")
4433 (match_operand:VNx2DI 3 "register_operand" "w, w")]
4435 (match_operand:SVE_FULL_BHSI 4 "aarch64_simd_imm_zero")]
4439 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.d
4440 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.d"
4441 [(set_attr "movprfx" "yes")])
4443 ;; -------------------------------------------------------------------------
4444 ;; ---- [INT] Shifts (rounding towards 0)
4445 ;; -------------------------------------------------------------------------
4448 ;; -------------------------------------------------------------------------
4450 ;; Unpredicated ASRD.
4451 (define_expand "sdiv_pow2<mode>3"
4452 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4456 [(match_operand:SVE_FULL_I 1 "register_operand")
4457 (match_operand 2 "aarch64_simd_rshift_imm")]
4463 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
4467 ;; Predicated ASRD with merging.
4468 (define_expand "@cond_asrd<mode>"
4469 [(set (match_operand:SVE_FULL_I 0 "register_operand")
4471 [(match_operand:<VPRED> 1 "register_operand")
4473 [(match_operand:SVE_FULL_I 2 "register_operand")
4474 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")]
4476 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
4481 ;; Predicated ASRD, merging with the first input.
4482 (define_insn "*cond_asrd<mode>_2"
4483 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
4485 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4487 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
4488 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")]
4494 asrd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4495 movprfx\t%0, %2\;asrd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
4496 [(set_attr "movprfx" "*,yes")])
4498 ;; Predicated ASRD, merging with zero.
4499 (define_insn "*cond_asrd<mode>_z"
4500 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
4502 [(match_operand:<VPRED> 1 "register_operand" "Upl")
4504 [(match_operand:SVE_FULL_I 2 "register_operand" "w")
4505 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")]
4507 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_zero")]
4510 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;asrd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
4511 [(set_attr "movprfx" "yes")])
4513 ;; -------------------------------------------------------------------------
4514 ;; ---- [FP<-INT] General binary arithmetic corresponding to unspecs
4515 ;; -------------------------------------------------------------------------
4520 ;; -------------------------------------------------------------------------
4522 ;; Unpredicated floating-point binary operations that take an integer as
4523 ;; their second operand.
4524 (define_insn "@aarch64_sve_<optab><mode>"
4525 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
4527 [(match_operand:SVE_FULL_F 1 "register_operand" "w")
4528 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")]
4529 SVE_FP_BINARY_INT))]
4531 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
4534 ;; Predicated floating-point binary operations that take an integer
4535 ;; as their second operand.
4536 (define_insn "@aarch64_pred_<optab><mode>"
4537 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
4539 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4540 (match_operand:SI 4 "aarch64_sve_gp_strictness")
4541 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
4542 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w, w")]
4543 SVE_COND_FP_BINARY_INT))]
4546 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4547 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4548 [(set_attr "movprfx" "*,yes")]
4551 ;; Predicated floating-point binary operations with merging, taking an
4552 ;; integer as their second operand.
4553 (define_expand "@cond_<optab><mode>"
4554 [(set (match_operand:SVE_FULL_F 0 "register_operand")
4556 [(match_operand:<VPRED> 1 "register_operand")
4559 (const_int SVE_STRICT_GP)
4560 (match_operand:SVE_FULL_F 2 "register_operand")
4561 (match_operand:<V_INT_EQUIV> 3 "register_operand")]
4562 SVE_COND_FP_BINARY_INT)
4563 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
4568 ;; Predicated floating-point binary operations that take an integer as their
4569 ;; second operand, with inactive lanes coming from the first operand.
4570 (define_insn_and_rewrite "*cond_<optab><mode>_2"
4571 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
4573 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4576 (match_operand:SI 5 "aarch64_sve_gp_strictness")
4577 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
4578 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w, w")]
4579 SVE_COND_FP_BINARY_INT)
4582 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
4584 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4585 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4586 "&& !rtx_equal_p (operands[1], operands[4])"
4588 operands[4] = copy_rtx (operands[1]);
4590 [(set_attr "movprfx" "*,yes")]
4593 ;; Predicated floating-point binary operations that take an integer as
4594 ;; their second operand, with the values of inactive lanes being distinct
4595 ;; from the other inputs.
4596 (define_insn_and_rewrite "*cond_<optab><mode>_any"
4597 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, ?&w")
4599 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
4602 (match_operand:SI 6 "aarch64_sve_gp_strictness")
4603 (match_operand:SVE_FULL_F 2 "register_operand" "0, w, w, w")
4604 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w, w, w, w")]
4605 SVE_COND_FP_BINARY_INT)
4606 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, Dz, 0, w")]
4609 && !rtx_equal_p (operands[2], operands[4])
4610 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
4612 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4613 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4614 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4618 if (reload_completed
4619 && register_operand (operands[4], <MODE>mode)
4620 && !rtx_equal_p (operands[0], operands[4]))
4622 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4623 operands[4], operands[1]));
4624 operands[4] = operands[2] = operands[0];
4626 else if (!rtx_equal_p (operands[1], operands[5]))
4627 operands[5] = copy_rtx (operands[1]);
4631 [(set_attr "movprfx" "yes")]
4634 ;; -------------------------------------------------------------------------
4635 ;; ---- [FP] General binary arithmetic corresponding to rtx codes
4636 ;; -------------------------------------------------------------------------
4637 ;; Includes post-RA forms of:
4641 ;; -------------------------------------------------------------------------
4643 ;; Unpredicated floating-point binary operations (post-RA only).
4644 ;; These are generated by splitting a predicated instruction whose
4645 ;; predicate is unused.
4646 (define_insn "*post_ra_<sve_fp_op><mode>3"
4647 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
4648 (SVE_UNPRED_FP_BINARY:SVE_FULL_F
4649 (match_operand:SVE_FULL_F 1 "register_operand" "w")
4650 (match_operand:SVE_FULL_F 2 "register_operand" "w")))]
4651 "TARGET_SVE && reload_completed"
4652 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>")
4654 ;; -------------------------------------------------------------------------
4655 ;; ---- [FP] General binary arithmetic corresponding to unspecs
4656 ;; -------------------------------------------------------------------------
4657 ;; Includes merging forms of:
4658 ;; - FADD (constant forms handled in the "Addition" section)
4662 ;; - FMAXNM (including #0.0 and #1.0)
4664 ;; - FMINNM (including #0.0 and #1.0)
4665 ;; - FMUL (including #0.5 and #2.0)
4669 ;; - FSUB (constant forms handled in the "Addition" section)
4670 ;; - FSUBR (constant forms handled in the "Subtraction" section)
4671 ;; -------------------------------------------------------------------------
4673 ;; Unpredicated floating-point binary operations.
4674 (define_insn "@aarch64_sve_<optab><mode>"
4675 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
4677 [(match_operand:SVE_FULL_F 1 "register_operand" "w")
4678 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
4681 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
4684 ;; Unpredicated floating-point binary operations that need to be predicated
4686 (define_expand "<optab><mode>3"
4687 [(set (match_operand:SVE_FULL_F 0 "register_operand")
4690 (const_int SVE_RELAXED_GP)
4691 (match_operand:SVE_FULL_F 1 "<sve_pred_fp_rhs1_operand>")
4692 (match_operand:SVE_FULL_F 2 "<sve_pred_fp_rhs2_operand>")]
4693 SVE_COND_FP_BINARY))]
4696 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
4700 ;; Predicated floating-point binary operations that have no immediate forms.
4701 (define_insn "@aarch64_pred_<optab><mode>"
4702 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?&w")
4704 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
4705 (match_operand:SI 4 "aarch64_sve_gp_strictness")
4706 (match_operand:SVE_FULL_F 2 "register_operand" "0, w, w")
4707 (match_operand:SVE_FULL_F 3 "register_operand" "w, 0, w")]
4708 SVE_COND_FP_BINARY_REG))]
4711 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4712 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4713 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4714 [(set_attr "movprfx" "*,*,yes")]
4717 ;; Predicated floating-point operations with merging.
4718 (define_expand "@cond_<optab><mode>"
4719 [(set (match_operand:SVE_FULL_F 0 "register_operand")
4721 [(match_operand:<VPRED> 1 "register_operand")
4724 (const_int SVE_STRICT_GP)
4725 (match_operand:SVE_FULL_F 2 "<sve_pred_fp_rhs1_operand>")
4726 (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_operand>")]
4728 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
4733 ;; Predicated floating-point operations, merging with the first input.
4734 (define_insn_and_rewrite "*cond_<optab><mode>_2"
4735 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
4737 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4740 (match_operand:SI 5 "aarch64_sve_gp_strictness")
4741 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
4742 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
4746 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
4748 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4749 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4750 "&& !rtx_equal_p (operands[1], operands[4])"
4752 operands[4] = copy_rtx (operands[1]);
4754 [(set_attr "movprfx" "*,yes")]
4757 ;; Same for operations that take a 1-bit constant.
4758 (define_insn_and_rewrite "*cond_<optab><mode>_2_const"
4759 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?w")
4761 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4764 (match_operand:SI 5 "aarch64_sve_gp_strictness")
4765 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
4766 (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
4767 SVE_COND_FP_BINARY_I1)
4770 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
4772 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4773 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
4774 "&& !rtx_equal_p (operands[1], operands[4])"
4776 operands[4] = copy_rtx (operands[1]);
4778 [(set_attr "movprfx" "*,yes")]
4781 ;; Predicated floating-point operations, merging with the second input.
4782 (define_insn_and_rewrite "*cond_<optab><mode>_3"
4783 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
4785 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
4788 (match_operand:SI 5 "aarch64_sve_gp_strictness")
4789 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
4790 (match_operand:SVE_FULL_F 3 "register_operand" "0, w")]
4794 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
4796 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4797 movprfx\t%0, %3\;<sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
4798 "&& !rtx_equal_p (operands[1], operands[4])"
4800 operands[4] = copy_rtx (operands[1]);
4802 [(set_attr "movprfx" "*,yes")]
4805 ;; Predicated floating-point operations, merging with an independent value.
4806 (define_insn_and_rewrite "*cond_<optab><mode>_any"
4807 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, &w, ?&w")
4809 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
4812 (match_operand:SI 6 "aarch64_sve_gp_strictness")
4813 (match_operand:SVE_FULL_F 2 "register_operand" "0, w, w, w, w")
4814 (match_operand:SVE_FULL_F 3 "register_operand" "w, 0, w, w, w")]
4816 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
4819 && !rtx_equal_p (operands[2], operands[4])
4820 && !rtx_equal_p (operands[3], operands[4])
4821 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
4823 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4824 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
4825 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4826 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4830 if (reload_completed
4831 && register_operand (operands[4], <MODE>mode)
4832 && !rtx_equal_p (operands[0], operands[4]))
4834 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4835 operands[4], operands[1]));
4836 operands[4] = operands[2] = operands[0];
4838 else if (!rtx_equal_p (operands[1], operands[5]))
4839 operands[5] = copy_rtx (operands[1]);
4843 [(set_attr "movprfx" "yes")]
4846 ;; Same for operations that take a 1-bit constant.
4847 (define_insn_and_rewrite "*cond_<optab><mode>_any_const"
4848 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?w")
4850 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
4853 (match_operand:SI 6 "aarch64_sve_gp_strictness")
4854 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")
4855 (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
4856 SVE_COND_FP_BINARY_I1)
4857 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, 0, w")]
4860 && !rtx_equal_p (operands[2], operands[4])
4861 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
4863 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4864 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4868 if (reload_completed
4869 && register_operand (operands[4], <MODE>mode)
4870 && !rtx_equal_p (operands[0], operands[4]))
4872 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4873 operands[4], operands[1]));
4874 operands[4] = operands[2] = operands[0];
4876 else if (!rtx_equal_p (operands[1], operands[5]))
4877 operands[5] = copy_rtx (operands[1]);
4881 [(set_attr "movprfx" "yes")]
4884 ;; -------------------------------------------------------------------------
4885 ;; ---- [FP] Addition
4886 ;; -------------------------------------------------------------------------
4890 ;; -------------------------------------------------------------------------
4892 ;; Predicated floating-point addition.
4893 (define_insn_and_split "@aarch64_pred_<optab><mode>"
4894 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, w, w, ?&w, ?&w, ?&w")
4896 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl, Upl")
4897 (match_operand:SI 4 "aarch64_sve_gp_strictness" "i, i, Z, Ui1, i, i, Ui1")
4898 (match_operand:SVE_FULL_F 2 "register_operand" "%0, 0, w, 0, w, w, w")
4899 (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_operand" "vsA, vsN, w, w, vsA, vsN, w")]
4903 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4904 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
4906 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
4907 movprfx\t%0, %2\;fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4908 movprfx\t%0, %2\;fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
4909 movprfx\t%0, %2\;fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
4910 ; Split the unpredicated form after reload, so that we don't have
4911 ; the unnecessary PTRUE.
4912 "&& reload_completed
4913 && register_operand (operands[3], <MODE>mode)
4914 && INTVAL (operands[4]) == SVE_RELAXED_GP"
4915 [(set (match_dup 0) (plus:SVE_FULL_F (match_dup 2) (match_dup 3)))]
4917 [(set_attr "movprfx" "*,*,*,*,yes,yes,yes")]
4920 ;; Predicated floating-point addition of a constant, merging with the
4922 (define_insn_and_rewrite "*cond_add<mode>_2_const"
4923 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?w, ?w")
4925 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
4928 (match_operand:SI 5 "aarch64_sve_gp_strictness")
4929 (match_operand:SVE_FULL_F 2 "register_operand" "0, 0, w, w")
4930 (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate" "vsA, vsN, vsA, vsN")]
4934 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
4936 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4937 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
4938 movprfx\t%0, %2\;fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4939 movprfx\t%0, %2\;fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3"
4940 "&& !rtx_equal_p (operands[1], operands[4])"
4942 operands[4] = copy_rtx (operands[1]);
4944 [(set_attr "movprfx" "*,*,yes,yes")]
4947 ;; Predicated floating-point addition of a constant, merging with an
4948 ;; independent value.
4949 (define_insn_and_rewrite "*cond_add<mode>_any_const"
4950 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, w, w, ?w, ?w")
4952 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
4955 (match_operand:SI 6 "aarch64_sve_gp_strictness")
4956 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w, w, w, w")
4957 (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate" "vsA, vsN, vsA, vsN, vsA, vsN")]
4959 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, Dz, 0, 0, w, w")]
4962 && !rtx_equal_p (operands[2], operands[4])
4963 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
4965 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4966 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
4967 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
4968 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
4973 if (reload_completed
4974 && register_operand (operands[4], <MODE>mode)
4975 && !rtx_equal_p (operands[0], operands[4]))
4977 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
4978 operands[4], operands[1]));
4979 operands[4] = operands[2] = operands[0];
4981 else if (!rtx_equal_p (operands[1], operands[5]))
4982 operands[5] = copy_rtx (operands[1]);
4986 [(set_attr "movprfx" "yes")]
4989 ;; Register merging forms are handled through SVE_COND_FP_BINARY.
4991 ;; -------------------------------------------------------------------------
4992 ;; ---- [FP] Complex addition
4993 ;; -------------------------------------------------------------------------
4996 ;; -------------------------------------------------------------------------
4998 ;; Predicated FCADD.
4999 (define_insn "@aarch64_pred_<optab><mode>"
5000 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
5002 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5003 (match_operand:SI 4 "aarch64_sve_gp_strictness")
5004 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
5005 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
5009 fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>
5010 movprfx\t%0, %2\;fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>"
5011 [(set_attr "movprfx" "*,yes")]
5014 ;; Predicated FCADD with merging.
5015 (define_expand "@cond_<optab><mode>"
5016 [(set (match_operand:SVE_FULL_F 0 "register_operand")
5018 [(match_operand:<VPRED> 1 "register_operand")
5021 (const_int SVE_STRICT_GP)
5022 (match_operand:SVE_FULL_F 2 "register_operand")
5023 (match_operand:SVE_FULL_F 3 "register_operand")]
5025 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
5030 ;; Predicated FCADD, merging with the first input.
5031 (define_insn_and_rewrite "*cond_<optab><mode>_2"
5032 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
5034 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5037 (match_operand:SI 5 "aarch64_sve_gp_strictness")
5038 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
5039 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
5043 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
5045 fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>
5046 movprfx\t%0, %2\;fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>"
5047 "&& !rtx_equal_p (operands[1], operands[4])"
5049 operands[4] = copy_rtx (operands[1]);
5051 [(set_attr "movprfx" "*,yes")]
5054 ;; Predicated FCADD, merging with an independent value.
5055 (define_insn_and_rewrite "*cond_<optab><mode>_any"
5056 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, ?&w")
5058 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
5061 (match_operand:SI 6 "aarch64_sve_gp_strictness")
5062 (match_operand:SVE_FULL_F 2 "register_operand" "w, 0, w, w")
5063 (match_operand:SVE_FULL_F 3 "register_operand" "w, w, w, w")]
5065 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, Dz, 0, w")]
5068 && !rtx_equal_p (operands[2], operands[4])
5069 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
5071 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>
5072 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>
5073 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;fcadd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>, #<rot>
5077 if (reload_completed
5078 && register_operand (operands[4], <MODE>mode)
5079 && !rtx_equal_p (operands[0], operands[4]))
5081 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
5082 operands[4], operands[1]));
5083 operands[4] = operands[2] = operands[0];
5085 else if (!rtx_equal_p (operands[1], operands[5]))
5086 operands[5] = copy_rtx (operands[1]);
5090 [(set_attr "movprfx" "yes")]
5093 ;; -------------------------------------------------------------------------
5094 ;; ---- [FP] Subtraction
5095 ;; -------------------------------------------------------------------------
5099 ;; -------------------------------------------------------------------------
5101 ;; Predicated floating-point subtraction.
5102 (define_insn_and_split "@aarch64_pred_<optab><mode>"
5103 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, w, w, ?&w, ?&w")
5105 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
5106 (match_operand:SI 4 "aarch64_sve_gp_strictness" "i, Z, Ui1, Ui1, i, Ui1")
5107 (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_operand" "vsA, w, 0, w, vsA, w")
5108 (match_operand:SVE_FULL_F 3 "register_operand" "0, w, w, 0, w, w")]
5112 fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
5114 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5115 fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
5116 movprfx\t%0, %3\;fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
5117 movprfx\t%0, %2\;fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
5118 ; Split the unpredicated form after reload, so that we don't have
5119 ; the unnecessary PTRUE.
5120 "&& reload_completed
5121 && register_operand (operands[2], <MODE>mode)
5122 && INTVAL (operands[4]) == SVE_RELAXED_GP"
5123 [(set (match_dup 0) (minus:SVE_FULL_F (match_dup 2) (match_dup 3)))]
5125 [(set_attr "movprfx" "*,*,*,*,yes,yes")]
5128 ;; Predicated floating-point subtraction from a constant, merging with the
5130 (define_insn_and_rewrite "*cond_sub<mode>_3_const"
5131 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?w")
5133 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5136 (match_operand:SI 5 "aarch64_sve_gp_strictness")
5137 (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
5138 (match_operand:SVE_FULL_F 3 "register_operand" "0, w")]
5142 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
5144 fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
5145 movprfx\t%0, %3\;fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2"
5146 "&& !rtx_equal_p (operands[1], operands[4])"
5148 operands[4] = copy_rtx (operands[1]);
5150 [(set_attr "movprfx" "*,yes")]
5153 ;; Predicated floating-point subtraction from a constant, merging with an
5154 ;; independent value.
5155 (define_insn_and_rewrite "*cond_sub<mode>_any_const"
5156 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?w")
5158 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
5161 (match_operand:SI 6 "aarch64_sve_gp_strictness")
5162 (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
5163 (match_operand:SVE_FULL_F 3 "register_operand" "w, w, w")]
5165 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, 0, w")]
5168 && !rtx_equal_p (operands[3], operands[4])
5169 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
5171 movprfx\t%0.<Vetype>, %1/z, %3.<Vetype>\;fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
5172 movprfx\t%0.<Vetype>, %1/m, %3.<Vetype>\;fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
5176 if (reload_completed
5177 && register_operand (operands[4], <MODE>mode)
5178 && !rtx_equal_p (operands[0], operands[4]))
5180 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[3],
5181 operands[4], operands[1]));
5182 operands[4] = operands[3] = operands[0];
5184 else if (!rtx_equal_p (operands[1], operands[5]))
5185 operands[5] = copy_rtx (operands[1]);
5189 [(set_attr "movprfx" "yes")]
5192 ;; Register merging forms are handled through SVE_COND_FP_BINARY.
5194 ;; -------------------------------------------------------------------------
5195 ;; ---- [FP] Absolute difference
5196 ;; -------------------------------------------------------------------------
5199 ;; -------------------------------------------------------------------------
5201 ;; Predicated floating-point absolute difference.
5202 (define_expand "@aarch64_pred_abd<mode>"
5203 [(set (match_operand:SVE_FULL_F 0 "register_operand")
5205 [(match_operand:<VPRED> 1 "register_operand")
5206 (match_operand:SI 4 "aarch64_sve_gp_strictness")
5210 (match_operand:SVE_FULL_F 2 "register_operand")
5211 (match_operand:SVE_FULL_F 3 "register_operand")]
5217 ;; Predicated floating-point absolute difference.
5218 (define_insn_and_rewrite "*aarch64_pred_abd<mode>"
5219 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
5221 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5222 (match_operand:SI 4 "aarch64_sve_gp_strictness")
5225 (match_operand:SI 6 "aarch64_sve_gp_strictness")
5226 (match_operand:SVE_FULL_F 2 "register_operand" "%0, w")
5227 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
5230 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
5232 fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5233 movprfx\t%0, %2\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
5234 "&& !rtx_equal_p (operands[1], operands[5])"
5236 operands[5] = copy_rtx (operands[1]);
5238 [(set_attr "movprfx" "*,yes")]
5241 (define_expand "@aarch64_cond_abd<mode>"
5242 [(set (match_operand:SVE_FULL_F 0 "register_operand")
5244 [(match_operand:<VPRED> 1 "register_operand")
5247 (const_int SVE_STRICT_GP)
5250 (const_int SVE_STRICT_GP)
5251 (match_operand:SVE_FULL_F 2 "register_operand")
5252 (match_operand:SVE_FULL_F 3 "register_operand")]
5255 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
5259 if (rtx_equal_p (operands[3], operands[4]))
5260 std::swap (operands[2], operands[3]);
5263 ;; Predicated floating-point absolute difference, merging with the first
5265 (define_insn_and_rewrite "*aarch64_cond_abd<mode>_2"
5266 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
5268 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5271 (match_operand:SI 5 "aarch64_sve_gp_strictness")
5274 (match_operand:SI 7 "aarch64_sve_gp_strictness")
5275 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
5276 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
5282 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])
5283 && aarch64_sve_pred_dominates_p (&operands[6], operands[1])"
5285 fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5286 movprfx\t%0, %2\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
5287 "&& (!rtx_equal_p (operands[1], operands[4])
5288 || !rtx_equal_p (operands[1], operands[6]))"
5290 operands[4] = copy_rtx (operands[1]);
5291 operands[6] = copy_rtx (operands[1]);
5293 [(set_attr "movprfx" "*,yes")]
5296 ;; Predicated floating-point absolute difference, merging with the second
5298 (define_insn_and_rewrite "*aarch64_cond_abd<mode>_3"
5299 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
5301 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5304 (match_operand:SI 5 "aarch64_sve_gp_strictness")
5307 (match_operand:SI 7 "aarch64_sve_gp_strictness")
5308 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
5309 (match_operand:SVE_FULL_F 3 "register_operand" "0, w")]
5315 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])
5316 && aarch64_sve_pred_dominates_p (&operands[6], operands[1])"
5318 fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
5319 movprfx\t%0, %3\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
5320 "&& (!rtx_equal_p (operands[1], operands[4])
5321 || !rtx_equal_p (operands[1], operands[6]))"
5323 operands[4] = copy_rtx (operands[1]);
5324 operands[6] = copy_rtx (operands[1]);
5326 [(set_attr "movprfx" "*,yes")]
5329 ;; Predicated floating-point absolute difference, merging with an
5330 ;; independent value.
5331 (define_insn_and_rewrite "*aarch64_cond_abd<mode>_any"
5332 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, &w, ?&w")
5334 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
5337 (match_operand:SI 6 "aarch64_sve_gp_strictness")
5340 (match_operand:SI 8 "aarch64_sve_gp_strictness")
5341 (match_operand:SVE_FULL_F 2 "register_operand" "0, w, w, w, w")
5342 (match_operand:SVE_FULL_F 3 "register_operand" "w, 0, w, w, w")]
5345 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
5348 && !rtx_equal_p (operands[2], operands[4])
5349 && !rtx_equal_p (operands[3], operands[4])
5350 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])
5351 && aarch64_sve_pred_dominates_p (&operands[7], operands[1])"
5353 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5354 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
5355 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5356 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;fabd\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5360 if (reload_completed
5361 && register_operand (operands[4], <MODE>mode)
5362 && !rtx_equal_p (operands[0], operands[4]))
5364 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[3],
5365 operands[4], operands[1]));
5366 operands[4] = operands[3] = operands[0];
5368 else if (!rtx_equal_p (operands[1], operands[5])
5369 || !rtx_equal_p (operands[1], operands[7]))
5371 operands[5] = copy_rtx (operands[1]);
5372 operands[7] = copy_rtx (operands[1]);
5377 [(set_attr "movprfx" "yes")]
5380 ;; -------------------------------------------------------------------------
5381 ;; ---- [FP] Multiplication
5382 ;; -------------------------------------------------------------------------
5385 ;; -------------------------------------------------------------------------
5387 ;; Predicated floating-point multiplication.
5388 (define_insn_and_split "@aarch64_pred_<optab><mode>"
5389 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, w, ?&w, ?&w")
5391 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
5392 (match_operand:SI 4 "aarch64_sve_gp_strictness" "i, Z, Ui1, i, Ui1")
5393 (match_operand:SVE_FULL_F 2 "register_operand" "%0, w, 0, w, w")
5394 (match_operand:SVE_FULL_F 3 "aarch64_sve_float_mul_operand" "vsM, w, w, vsM, w")]
5398 fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
5400 fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5401 movprfx\t%0, %2\;fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
5402 movprfx\t%0, %2\;fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
5403 ; Split the unpredicated form after reload, so that we don't have
5404 ; the unnecessary PTRUE.
5405 "&& reload_completed
5406 && register_operand (operands[3], <MODE>mode)
5407 && INTVAL (operands[4]) == SVE_RELAXED_GP"
5408 [(set (match_dup 0) (mult:SVE_FULL_F (match_dup 2) (match_dup 3)))]
5410 [(set_attr "movprfx" "*,*,*,yes,yes")]
5413 ;; Merging forms are handled through SVE_COND_FP_BINARY and
5414 ;; SVE_COND_FP_BINARY_I1.
5416 ;; Unpredicated multiplication by selected lanes.
5417 (define_insn "@aarch64_mul_lane_<mode>"
5418 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
5421 [(match_operand:SVE_FULL_F 2 "register_operand" "<sve_lane_con>")
5422 (match_operand:SI 3 "const_int_operand")]
5423 UNSPEC_SVE_LANE_SELECT)
5424 (match_operand:SVE_FULL_F 1 "register_operand" "w")))]
5426 "fmul\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]"
5429 ;; -------------------------------------------------------------------------
5430 ;; ---- [FP] Binary logical operations
5431 ;; -------------------------------------------------------------------------
5436 ;; -------------------------------------------------------------------------
5438 ;; Binary logical operations on floating-point modes. We avoid subregs
5439 ;; by providing this, but we need to use UNSPECs since rtx logical ops
5440 ;; aren't defined for floating-point modes.
5441 (define_insn "*<optab><mode>3"
5442 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
5444 [(match_operand:SVE_FULL_F 1 "register_operand" "w")
5445 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
5448 "<logicalf_op>\t%0.d, %1.d, %2.d"
5451 ;; -------------------------------------------------------------------------
5452 ;; ---- [FP] Sign copying
5453 ;; -------------------------------------------------------------------------
5454 ;; The patterns in this section are synthetic.
5455 ;; -------------------------------------------------------------------------
5457 (define_expand "copysign<mode>3"
5458 [(match_operand:SVE_FULL_F 0 "register_operand")
5459 (match_operand:SVE_FULL_F 1 "register_operand")
5460 (match_operand:SVE_FULL_F 2 "register_operand")]
5463 rtx sign = gen_reg_rtx (<V_INT_EQUIV>mode);
5464 rtx mant = gen_reg_rtx (<V_INT_EQUIV>mode);
5465 rtx int_res = gen_reg_rtx (<V_INT_EQUIV>mode);
5466 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
5468 rtx arg1 = lowpart_subreg (<V_INT_EQUIV>mode, operands[1], <MODE>mode);
5469 rtx arg2 = lowpart_subreg (<V_INT_EQUIV>mode, operands[2], <MODE>mode);
5471 emit_insn (gen_and<v_int_equiv>3
5473 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
5476 emit_insn (gen_and<v_int_equiv>3
5478 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
5481 emit_insn (gen_ior<v_int_equiv>3 (int_res, sign, mant));
5482 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, int_res));
5487 (define_expand "xorsign<mode>3"
5488 [(match_operand:SVE_FULL_F 0 "register_operand")
5489 (match_operand:SVE_FULL_F 1 "register_operand")
5490 (match_operand:SVE_FULL_F 2 "register_operand")]
5493 rtx sign = gen_reg_rtx (<V_INT_EQUIV>mode);
5494 rtx int_res = gen_reg_rtx (<V_INT_EQUIV>mode);
5495 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
5497 rtx arg1 = lowpart_subreg (<V_INT_EQUIV>mode, operands[1], <MODE>mode);
5498 rtx arg2 = lowpart_subreg (<V_INT_EQUIV>mode, operands[2], <MODE>mode);
5500 emit_insn (gen_and<v_int_equiv>3
5502 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
5505 emit_insn (gen_xor<v_int_equiv>3 (int_res, arg1, sign));
5506 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, int_res));
5511 ;; -------------------------------------------------------------------------
5512 ;; ---- [FP] Maximum and minimum
5513 ;; -------------------------------------------------------------------------
5519 ;; -------------------------------------------------------------------------
5521 ;; Unpredicated fmax/fmin (the libm functions). The optabs for the
5522 ;; smin/smax rtx codes are handled in the generic section above.
5523 (define_expand "<maxmin_uns><mode>3"
5524 [(set (match_operand:SVE_FULL_F 0 "register_operand")
5527 (const_int SVE_RELAXED_GP)
5528 (match_operand:SVE_FULL_F 1 "register_operand")
5529 (match_operand:SVE_FULL_F 2 "aarch64_sve_float_maxmin_operand")]
5530 SVE_COND_FP_MAXMIN_PUBLIC))]
5533 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
5537 ;; Predicated floating-point maximum/minimum.
5538 (define_insn "@aarch64_pred_<optab><mode>"
5539 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?&w, ?&w")
5541 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
5542 (match_operand:SI 4 "aarch64_sve_gp_strictness")
5543 (match_operand:SVE_FULL_F 2 "register_operand" "%0, 0, w, w")
5544 (match_operand:SVE_FULL_F 3 "aarch64_sve_float_maxmin_operand" "vsB, w, vsB, w")]
5545 SVE_COND_FP_MAXMIN))]
5548 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
5549 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
5550 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
5551 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
5552 [(set_attr "movprfx" "*,*,yes,yes")]
5555 ;; Merging forms are handled through SVE_COND_FP_BINARY and
5556 ;; SVE_COND_FP_BINARY_I1.
5558 ;; -------------------------------------------------------------------------
5559 ;; ---- [PRED] Binary logical operations
5560 ;; -------------------------------------------------------------------------
5568 ;; -------------------------------------------------------------------------
5570 ;; Predicate AND. We can reuse one of the inputs as the GP.
5571 ;; Doubling the second operand is the preferred implementation
5572 ;; of the MOV alias, so we use that instead of %1/z, %1, %2.
5573 (define_insn "and<mode>3"
5574 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5575 (and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand" "Upa")
5576 (match_operand:PRED_ALL 2 "register_operand" "Upa")))]
5578 "and\t%0.b, %1/z, %2.b, %2.b"
5581 ;; Unpredicated predicate EOR and ORR.
5582 (define_expand "<optab><mode>3"
5583 [(set (match_operand:PRED_ALL 0 "register_operand")
5585 (LOGICAL_OR:PRED_ALL
5586 (match_operand:PRED_ALL 1 "register_operand")
5587 (match_operand:PRED_ALL 2 "register_operand"))
5591 operands[3] = aarch64_ptrue_reg (<MODE>mode);
5595 ;; Predicated predicate AND, EOR and ORR.
5596 (define_insn "@aarch64_pred_<optab><mode>_z"
5597 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5600 (match_operand:PRED_ALL 2 "register_operand" "Upa")
5601 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5602 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
5604 "<logical>\t%0.b, %1/z, %2.b, %3.b"
5607 ;; Perform a logical operation on operands 2 and 3, using operand 1 as
5608 ;; the GP. Store the result in operand 0 and set the flags in the same
5609 ;; way as for PTEST.
5610 (define_insn "*<optab><mode>3_cc"
5611 [(set (reg:CC_NZC CC_REGNUM)
5613 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5615 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5618 (match_operand:PRED_ALL 2 "register_operand" "Upa")
5619 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5622 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5623 (and:PRED_ALL (LOGICAL:PRED_ALL (match_dup 2) (match_dup 3))
5626 "<logical>s\t%0.b, %1/z, %2.b, %3.b"
5629 ;; Same with just the flags result.
5630 (define_insn "*<optab><mode>3_ptest"
5631 [(set (reg:CC_NZC CC_REGNUM)
5633 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5635 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5638 (match_operand:PRED_ALL 2 "register_operand" "Upa")
5639 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5642 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
5644 "<logical>s\t%0.b, %1/z, %2.b, %3.b"
5647 ;; -------------------------------------------------------------------------
5648 ;; ---- [PRED] Binary logical operations (inverted second input)
5649 ;; -------------------------------------------------------------------------
5653 ;; -------------------------------------------------------------------------
5655 ;; Predicated predicate BIC and ORN.
5656 (define_insn "aarch64_pred_<nlogical><mode>_z"
5657 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5660 (not:PRED_ALL (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5661 (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5662 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
5664 "<nlogical>\t%0.b, %1/z, %2.b, %3.b"
5667 ;; Same, but set the flags as a side-effect.
5668 (define_insn "*<nlogical><mode>3_cc"
5669 [(set (reg:CC_NZC CC_REGNUM)
5671 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5673 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5677 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5678 (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5681 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5682 (and:PRED_ALL (NLOGICAL:PRED_ALL
5683 (not:PRED_ALL (match_dup 3))
5687 "<nlogical>s\t%0.b, %1/z, %2.b, %3.b"
5690 ;; Same with just the flags result.
5691 (define_insn "*<nlogical><mode>3_ptest"
5692 [(set (reg:CC_NZC CC_REGNUM)
5694 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5696 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5700 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
5701 (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5704 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
5706 "<nlogical>s\t%0.b, %1/z, %2.b, %3.b"
5709 ;; -------------------------------------------------------------------------
5710 ;; ---- [PRED] Binary logical operations (inverted result)
5711 ;; -------------------------------------------------------------------------
5715 ;; -------------------------------------------------------------------------
5717 ;; Predicated predicate NAND and NOR.
5718 (define_insn "aarch64_pred_<logical_nn><mode>_z"
5719 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5722 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5723 (not:PRED_ALL (match_operand:PRED_ALL 3 "register_operand" "Upa")))
5724 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
5726 "<logical_nn>\t%0.b, %1/z, %2.b, %3.b"
5729 ;; Same, but set the flags as a side-effect.
5730 (define_insn "*<logical_nn><mode>3_cc"
5731 [(set (reg:CC_NZC CC_REGNUM)
5733 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5735 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5739 (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5741 (match_operand:PRED_ALL 3 "register_operand" "Upa")))
5744 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
5745 (and:PRED_ALL (NLOGICAL:PRED_ALL
5746 (not:PRED_ALL (match_dup 2))
5747 (not:PRED_ALL (match_dup 3)))
5750 "<logical_nn>s\t%0.b, %1/z, %2.b, %3.b"
5753 ;; Same with just the flags result.
5754 (define_insn "*<logical_nn><mode>3_ptest"
5755 [(set (reg:CC_NZC CC_REGNUM)
5757 [(match_operand:VNx16BI 1 "register_operand" "Upa")
5759 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
5763 (match_operand:PRED_ALL 2 "register_operand" "Upa"))
5765 (match_operand:PRED_ALL 3 "register_operand" "Upa")))
5768 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
5770 "<logical_nn>s\t%0.b, %1/z, %2.b, %3.b"
5773 ;; =========================================================================
5774 ;; == Ternary arithmetic
5775 ;; =========================================================================
5777 ;; -------------------------------------------------------------------------
5778 ;; ---- [INT] MLA and MAD
5779 ;; -------------------------------------------------------------------------
5783 ;; -------------------------------------------------------------------------
5785 ;; Unpredicated integer addition of product.
5786 (define_expand "fma<mode>4"
5787 [(set (match_operand:SVE_FULL_I 0 "register_operand")
5792 (match_operand:SVE_FULL_I 1 "register_operand")
5793 (match_operand:SVE_FULL_I 2 "nonmemory_operand"))]
5795 (match_operand:SVE_FULL_I 3 "register_operand")))]
5798 if (aarch64_prepare_sve_int_fma (operands, PLUS))
5800 operands[4] = aarch64_ptrue_reg (<VPRED>mode);
5804 ;; Predicated integer addition of product.
5805 (define_insn "@aarch64_pred_fma<mode>"
5806 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w")
5809 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
5811 (match_operand:SVE_FULL_I 2 "register_operand" "%0, w, w")
5812 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, w"))]
5814 (match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w")))]
5817 mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
5818 mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5819 movprfx\t%0, %4\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
5820 [(set_attr "movprfx" "*,*,yes")]
5823 ;; Predicated integer addition of product with merging.
5824 (define_expand "cond_fma<mode>"
5825 [(set (match_operand:SVE_FULL_I 0 "register_operand")
5827 [(match_operand:<VPRED> 1 "register_operand")
5830 (match_operand:SVE_FULL_I 2 "register_operand")
5831 (match_operand:SVE_FULL_I 3 "general_operand"))
5832 (match_operand:SVE_FULL_I 4 "register_operand"))
5833 (match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero")]
5837 if (aarch64_prepare_sve_cond_int_fma (operands, PLUS))
5839 /* Swap the multiplication operands if the fallback value is the
5840 second of the two. */
5841 if (rtx_equal_p (operands[3], operands[5]))
5842 std::swap (operands[2], operands[3]);
5846 ;; Predicated integer addition of product, merging with the first input.
5847 (define_insn "*cond_fma<mode>_2"
5848 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
5850 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5853 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
5854 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
5855 (match_operand:SVE_FULL_I 4 "register_operand" "w, w"))
5860 mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
5861 movprfx\t%0, %2\;mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
5862 [(set_attr "movprfx" "*,yes")]
5865 ;; Predicated integer addition of product, merging with the third input.
5866 (define_insn "*cond_fma<mode>_4"
5867 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
5869 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5872 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
5873 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
5874 (match_operand:SVE_FULL_I 4 "register_operand" "0, w"))
5879 mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5880 movprfx\t%0, %4\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
5881 [(set_attr "movprfx" "*,yes")]
5884 ;; Predicated integer addition of product, merging with an independent value.
5885 (define_insn_and_rewrite "*cond_fma<mode>_any"
5886 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, &w, ?&w")
5888 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
5891 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, 0, w, w, w")
5892 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, w, 0, w, w"))
5893 (match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w, w, w, w"))
5894 (match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, 0, w")]
5897 && !rtx_equal_p (operands[2], operands[5])
5898 && !rtx_equal_p (operands[3], operands[5])
5899 && !rtx_equal_p (operands[4], operands[5])"
5901 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5902 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5903 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
5904 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;mad\t%0.<Vetype>, %1/m, %2.<Vetype>, %4.<Vetype>
5905 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5907 "&& reload_completed
5908 && register_operand (operands[5], <MODE>mode)
5909 && !rtx_equal_p (operands[0], operands[5])"
5911 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[4],
5912 operands[5], operands[1]));
5913 operands[5] = operands[4] = operands[0];
5915 [(set_attr "movprfx" "yes")]
5918 ;; -------------------------------------------------------------------------
5919 ;; ---- [INT] MLS and MSB
5920 ;; -------------------------------------------------------------------------
5924 ;; -------------------------------------------------------------------------
5926 ;; Unpredicated integer subtraction of product.
5927 (define_expand "fnma<mode>4"
5928 [(set (match_operand:SVE_FULL_I 0 "register_operand")
5930 (match_operand:SVE_FULL_I 3 "register_operand")
5934 (match_operand:SVE_FULL_I 1 "register_operand")
5935 (match_operand:SVE_FULL_I 2 "general_operand"))]
5939 if (aarch64_prepare_sve_int_fma (operands, MINUS))
5941 operands[4] = aarch64_ptrue_reg (<VPRED>mode);
5945 ;; Predicated integer subtraction of product.
5946 (define_insn "@aarch64_pred_fnma<mode>"
5947 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w")
5949 (match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w")
5951 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
5953 (match_operand:SVE_FULL_I 2 "register_operand" "%0, w, w")
5954 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, w"))]
5958 msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
5959 mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
5960 movprfx\t%0, %4\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
5961 [(set_attr "movprfx" "*,*,yes")]
5964 ;; Predicated integer subtraction of product with merging.
5965 (define_expand "cond_fnma<mode>"
5966 [(set (match_operand:SVE_FULL_I 0 "register_operand")
5968 [(match_operand:<VPRED> 1 "register_operand")
5970 (match_operand:SVE_FULL_I 4 "register_operand")
5972 (match_operand:SVE_FULL_I 2 "register_operand")
5973 (match_operand:SVE_FULL_I 3 "general_operand")))
5974 (match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero")]
5978 if (aarch64_prepare_sve_cond_int_fma (operands, MINUS))
5980 /* Swap the multiplication operands if the fallback value is the
5981 second of the two. */
5982 if (rtx_equal_p (operands[3], operands[5]))
5983 std::swap (operands[2], operands[3]);
5987 ;; Predicated integer subtraction of product, merging with the first input.
5988 (define_insn "*cond_fnma<mode>_2"
5989 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
5991 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
5993 (match_operand:SVE_FULL_I 4 "register_operand" "w, w")
5995 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
5996 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")))
6001 msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
6002 movprfx\t%0, %2\;msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
6003 [(set_attr "movprfx" "*,yes")]
6006 ;; Predicated integer subtraction of product, merging with the third input.
6007 (define_insn "*cond_fnma<mode>_4"
6008 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
6010 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6012 (match_operand:SVE_FULL_I 4 "register_operand" "0, w")
6014 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
6015 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")))
6020 mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6021 movprfx\t%0, %4\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
6022 [(set_attr "movprfx" "*,yes")]
6025 ;; Predicated integer subtraction of product, merging with an
6026 ;; independent value.
6027 (define_insn_and_rewrite "*cond_fnma<mode>_any"
6028 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, &w, ?&w")
6030 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
6032 (match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w, w, w, w")
6034 (match_operand:SVE_FULL_I 2 "register_operand" "w, w, 0, w, w, w")
6035 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, w, 0, w, w")))
6036 (match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, 0, w")]
6039 && !rtx_equal_p (operands[2], operands[5])
6040 && !rtx_equal_p (operands[3], operands[5])
6041 && !rtx_equal_p (operands[4], operands[5])"
6043 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6044 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6045 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
6046 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;msb\t%0.<Vetype>, %1/m, %2.<Vetype>, %4.<Vetype>
6047 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6049 "&& reload_completed
6050 && register_operand (operands[5], <MODE>mode)
6051 && !rtx_equal_p (operands[0], operands[5])"
6053 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[4],
6054 operands[5], operands[1]));
6055 operands[5] = operands[4] = operands[0];
6057 [(set_attr "movprfx" "yes")]
6060 ;; -------------------------------------------------------------------------
6061 ;; ---- [INT] Dot product
6062 ;; -------------------------------------------------------------------------
6066 ;; -------------------------------------------------------------------------
6068 ;; Four-element integer dot-product with accumulation.
6069 (define_insn "<sur>dot_prod<vsi2qi>"
6070 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
6072 (unspec:SVE_FULL_SDI
6073 [(match_operand:<VSI2QI> 1 "register_operand" "w, w")
6074 (match_operand:<VSI2QI> 2 "register_operand" "w, w")]
6076 (match_operand:SVE_FULL_SDI 3 "register_operand" "0, w")))]
6079 <sur>dot\\t%0.<Vetype>, %1.<Vetype_fourth>, %2.<Vetype_fourth>
6080 movprfx\t%0, %3\;<sur>dot\\t%0.<Vetype>, %1.<Vetype_fourth>, %2.<Vetype_fourth>"
6081 [(set_attr "movprfx" "*,yes")]
6084 ;; Four-element integer dot-product by selected lanes with accumulation.
6085 (define_insn "@aarch64_<sur>dot_prod_lane<vsi2qi>"
6086 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
6088 (unspec:SVE_FULL_SDI
6089 [(match_operand:<VSI2QI> 1 "register_operand" "w, w")
6091 [(match_operand:<VSI2QI> 2 "register_operand" "<sve_lane_con>, <sve_lane_con>")
6092 (match_operand:SI 3 "const_int_operand")]
6093 UNSPEC_SVE_LANE_SELECT)]
6095 (match_operand:SVE_FULL_SDI 4 "register_operand" "0, w")))]
6098 <sur>dot\\t%0.<Vetype>, %1.<Vetype_fourth>, %2.<Vetype_fourth>[%3]
6099 movprfx\t%0, %4\;<sur>dot\\t%0.<Vetype>, %1.<Vetype_fourth>, %2.<Vetype_fourth>[%3]"
6100 [(set_attr "movprfx" "*,yes")]
6103 ;; -------------------------------------------------------------------------
6104 ;; ---- [INT] Sum of absolute differences
6105 ;; -------------------------------------------------------------------------
6106 ;; The patterns in this section are synthetic.
6107 ;; -------------------------------------------------------------------------
6109 ;; Emit a sequence to produce a sum-of-absolute-differences of the inputs in
6110 ;; operands 1 and 2. The sequence also has to perform a widening reduction of
6111 ;; the difference into a vector and accumulate that into operand 3 before
6112 ;; copying that into the result operand 0.
6113 ;; Perform that with a sequence of:
6115 ;; [SU]ABD diff.b, p0/m, op1.b, op2.b
6116 ;; MOVPRFX op0, op3 // If necessary
6117 ;; UDOT op0.s, diff.b, ones.b
6118 (define_expand "<sur>sad<vsi2qi>"
6119 [(use (match_operand:SVE_FULL_SDI 0 "register_operand"))
6120 (unspec:<VSI2QI> [(use (match_operand:<VSI2QI> 1 "register_operand"))
6121 (use (match_operand:<VSI2QI> 2 "register_operand"))] ABAL)
6122 (use (match_operand:SVE_FULL_SDI 3 "register_operand"))]
6125 rtx ones = force_reg (<VSI2QI>mode, CONST1_RTX (<VSI2QI>mode));
6126 rtx diff = gen_reg_rtx (<VSI2QI>mode);
6127 emit_insn (gen_<sur>abd<vsi2qi>_3 (diff, operands[1], operands[2]));
6128 emit_insn (gen_udot_prod<vsi2qi> (operands[0], diff, ones, operands[3]));
6133 ;; -------------------------------------------------------------------------
6134 ;; ---- [FP] General ternary arithmetic corresponding to unspecs
6135 ;; -------------------------------------------------------------------------
6136 ;; Includes merging patterns for:
6145 ;; -------------------------------------------------------------------------
6147 ;; Unpredicated floating-point ternary operations.
6148 (define_expand "<optab><mode>4"
6149 [(set (match_operand:SVE_FULL_F 0 "register_operand")
6152 (const_int SVE_RELAXED_GP)
6153 (match_operand:SVE_FULL_F 1 "register_operand")
6154 (match_operand:SVE_FULL_F 2 "register_operand")
6155 (match_operand:SVE_FULL_F 3 "register_operand")]
6156 SVE_COND_FP_TERNARY))]
6159 operands[4] = aarch64_ptrue_reg (<VPRED>mode);
6163 ;; Predicated floating-point ternary operations.
6164 (define_insn "@aarch64_pred_<optab><mode>"
6165 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, w, ?&w")
6167 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
6168 (match_operand:SI 5 "aarch64_sve_gp_strictness")
6169 (match_operand:SVE_FULL_F 2 "register_operand" "%w, 0, w")
6170 (match_operand:SVE_FULL_F 3 "register_operand" "w, w, w")
6171 (match_operand:SVE_FULL_F 4 "register_operand" "0, w, w")]
6172 SVE_COND_FP_TERNARY))]
6175 <sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6176 <sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
6177 movprfx\t%0, %4\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
6178 [(set_attr "movprfx" "*,*,yes")]
6181 ;; Predicated floating-point ternary operations with merging.
6182 (define_expand "@cond_<optab><mode>"
6183 [(set (match_operand:SVE_FULL_F 0 "register_operand")
6185 [(match_operand:<VPRED> 1 "register_operand")
6188 (const_int SVE_STRICT_GP)
6189 (match_operand:SVE_FULL_F 2 "register_operand")
6190 (match_operand:SVE_FULL_F 3 "register_operand")
6191 (match_operand:SVE_FULL_F 4 "register_operand")]
6192 SVE_COND_FP_TERNARY)
6193 (match_operand:SVE_FULL_F 5 "aarch64_simd_reg_or_zero")]
6197 /* Swap the multiplication operands if the fallback value is the
6198 second of the two. */
6199 if (rtx_equal_p (operands[3], operands[5]))
6200 std::swap (operands[2], operands[3]);
6203 ;; Predicated floating-point ternary operations, merging with the
6205 (define_insn_and_rewrite "*cond_<optab><mode>_2"
6206 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6208 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6211 (match_operand:SI 6 "aarch64_sve_gp_strictness")
6212 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
6213 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")
6214 (match_operand:SVE_FULL_F 4 "register_operand" "w, w")]
6215 SVE_COND_FP_TERNARY)
6218 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
6220 <sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
6221 movprfx\t%0, %2\;<sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
6222 "&& !rtx_equal_p (operands[1], operands[5])"
6224 operands[5] = copy_rtx (operands[1]);
6226 [(set_attr "movprfx" "*,yes")]
6229 ;; Predicated floating-point ternary operations, merging with the
6231 (define_insn_and_rewrite "*cond_<optab><mode>_4"
6232 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6234 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6237 (match_operand:SI 6 "aarch64_sve_gp_strictness")
6238 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
6239 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")
6240 (match_operand:SVE_FULL_F 4 "register_operand" "0, w")]
6241 SVE_COND_FP_TERNARY)
6244 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
6246 <sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6247 movprfx\t%0, %4\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
6248 "&& !rtx_equal_p (operands[1], operands[5])"
6250 operands[5] = copy_rtx (operands[1]);
6252 [(set_attr "movprfx" "*,yes")]
6255 ;; Predicated floating-point ternary operations, merging with an
6256 ;; independent value.
6257 (define_insn_and_rewrite "*cond_<optab><mode>_any"
6258 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, &w, &w, ?&w")
6260 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
6263 (match_operand:SI 7 "aarch64_sve_gp_strictness")
6264 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, 0, w, w, w")
6265 (match_operand:SVE_FULL_F 3 "register_operand" "w, w, w, 0, w, w")
6266 (match_operand:SVE_FULL_F 4 "register_operand" "w, 0, w, w, w, w")]
6267 SVE_COND_FP_TERNARY)
6268 (match_operand:SVE_FULL_F 5 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, 0, w")]
6271 && !rtx_equal_p (operands[2], operands[5])
6272 && !rtx_equal_p (operands[3], operands[5])
6273 && !rtx_equal_p (operands[4], operands[5])
6274 && aarch64_sve_pred_dominates_p (&operands[6], operands[1])"
6276 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6277 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6278 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
6279 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_fmad_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %4.<Vetype>
6280 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
6284 if (reload_completed
6285 && register_operand (operands[5], <MODE>mode)
6286 && !rtx_equal_p (operands[0], operands[5]))
6288 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[4],
6289 operands[5], operands[1]));
6290 operands[5] = operands[4] = operands[0];
6292 else if (!rtx_equal_p (operands[1], operands[6]))
6293 operands[6] = copy_rtx (operands[1]);
6297 [(set_attr "movprfx" "yes")]
6300 ;; Unpredicated FMLA and FMLS by selected lanes. It doesn't seem worth using
6301 ;; (fma ...) since target-independent code won't understand the indexing.
6302 (define_insn "@aarch64_<optab>_lane_<mode>"
6303 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6305 [(match_operand:SVE_FULL_F 1 "register_operand" "w, w")
6307 [(match_operand:SVE_FULL_F 2 "register_operand" "<sve_lane_con>, <sve_lane_con>")
6308 (match_operand:SI 3 "const_int_operand")]
6309 UNSPEC_SVE_LANE_SELECT)
6310 (match_operand:SVE_FULL_F 4 "register_operand" "0, w")]
6311 SVE_FP_TERNARY_LANE))]
6314 <sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]
6315 movprfx\t%0, %4\;<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]"
6316 [(set_attr "movprfx" "*,yes")]
6319 ;; -------------------------------------------------------------------------
6320 ;; ---- [FP] Complex multiply-add
6321 ;; -------------------------------------------------------------------------
6322 ;; Includes merging patterns for:
6324 ;; -------------------------------------------------------------------------
6326 ;; Predicated FCMLA.
6327 (define_insn "@aarch64_pred_<optab><mode>"
6328 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6330 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6331 (match_operand:SI 5 "aarch64_sve_gp_strictness")
6332 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
6333 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")
6334 (match_operand:SVE_FULL_F 4 "register_operand" "0, w")]
6338 fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>
6339 movprfx\t%0, %4\;fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>"
6340 [(set_attr "movprfx" "*,yes")]
6343 ;; Predicated FCMLA with merging.
6344 (define_expand "@cond_<optab><mode>"
6345 [(set (match_operand:SVE_FULL_F 0 "register_operand")
6347 [(match_operand:<VPRED> 1 "register_operand")
6350 (const_int SVE_STRICT_GP)
6351 (match_operand:SVE_FULL_F 2 "register_operand")
6352 (match_operand:SVE_FULL_F 3 "register_operand")
6353 (match_operand:SVE_FULL_F 4 "register_operand")]
6355 (match_operand:SVE_FULL_F 5 "aarch64_simd_reg_or_zero")]
6360 ;; Predicated FCMLA, merging with the third input.
6361 (define_insn_and_rewrite "*cond_<optab><mode>_4"
6362 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6364 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6367 (match_operand:SI 6 "aarch64_sve_gp_strictness")
6368 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
6369 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")
6370 (match_operand:SVE_FULL_F 4 "register_operand" "0, w")]
6374 "TARGET_SVE && aarch64_sve_pred_dominates_p (&operands[5], operands[1])"
6376 fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>
6377 movprfx\t%0, %4\;fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>"
6378 "&& !rtx_equal_p (operands[1], operands[5])"
6380 operands[5] = copy_rtx (operands[1]);
6382 [(set_attr "movprfx" "*,yes")]
6385 ;; Predicated FCMLA, merging with an independent value.
6386 (define_insn_and_rewrite "*cond_<optab><mode>_any"
6387 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, &w, ?&w")
6389 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
6392 (match_operand:SI 7 "aarch64_sve_gp_strictness")
6393 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w, w")
6394 (match_operand:SVE_FULL_F 3 "register_operand" "w, w, w, w")
6395 (match_operand:SVE_FULL_F 4 "register_operand" "w, 0, w, w")]
6397 (match_operand:SVE_FULL_F 5 "aarch64_simd_reg_or_zero" "Dz, Dz, 0, w")]
6400 && !rtx_equal_p (operands[4], operands[5])
6401 && aarch64_sve_pred_dominates_p (&operands[6], operands[1])"
6403 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>
6404 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>
6405 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;fcmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>, #<rot>
6409 if (reload_completed
6410 && register_operand (operands[5], <MODE>mode)
6411 && !rtx_equal_p (operands[0], operands[5]))
6413 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[4],
6414 operands[5], operands[1]));
6415 operands[5] = operands[4] = operands[0];
6417 else if (!rtx_equal_p (operands[1], operands[6]))
6418 operands[6] = copy_rtx (operands[1]);
6422 [(set_attr "movprfx" "yes")]
6425 ;; Unpredicated FCMLA with indexing.
6426 (define_insn "@aarch64_<optab>_lane_<mode>"
6427 [(set (match_operand:SVE_FULL_HSF 0 "register_operand" "=w, ?&w")
6428 (unspec:SVE_FULL_HSF
6429 [(match_operand:SVE_FULL_HSF 1 "register_operand" "w, w")
6430 (unspec:SVE_FULL_HSF
6431 [(match_operand:SVE_FULL_HSF 2 "register_operand" "<sve_lane_pair_con>, <sve_lane_pair_con>")
6432 (match_operand:SI 3 "const_int_operand")]
6433 UNSPEC_SVE_LANE_SELECT)
6434 (match_operand:SVE_FULL_HSF 4 "register_operand" "0, w")]
6438 fcmla\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3], #<rot>
6439 movprfx\t%0, %4\;fcmla\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3], #<rot>"
6440 [(set_attr "movprfx" "*,yes")]
6443 ;; -------------------------------------------------------------------------
6444 ;; ---- [FP] Trigonometric multiply-add
6445 ;; -------------------------------------------------------------------------
6448 ;; -------------------------------------------------------------------------
6450 (define_insn "@aarch64_sve_tmad<mode>"
6451 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
6453 [(match_operand:SVE_FULL_F 1 "register_operand" "0, w")
6454 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
6455 (match_operand:DI 3 "const_int_operand")]
6459 ftmad\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3
6460 movprfx\t%0, %1\;ftmad\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3"
6461 [(set_attr "movprfx" "*,yes")]
6464 ;; =========================================================================
6465 ;; == Comparisons and selects
6466 ;; =========================================================================
6468 ;; -------------------------------------------------------------------------
6469 ;; ---- [INT,FP] Select based on predicates
6470 ;; -------------------------------------------------------------------------
6471 ;; Includes merging patterns for:
6475 ;; -------------------------------------------------------------------------
6477 ;; vcond_mask operand order: true, false, mask
6478 ;; UNSPEC_SEL operand order: mask, true, false (as for VEC_COND_EXPR)
6479 ;; SEL operand order: mask, true, false
6480 (define_expand "@vcond_mask_<mode><vpred>"
6481 [(set (match_operand:SVE_FULL 0 "register_operand")
6483 [(match_operand:<VPRED> 3 "register_operand")
6484 (match_operand:SVE_FULL 1 "aarch64_sve_reg_or_dup_imm")
6485 (match_operand:SVE_FULL 2 "aarch64_simd_reg_or_zero")]
6489 if (register_operand (operands[1], <MODE>mode))
6490 operands[2] = force_reg (<MODE>mode, operands[2]);
6496 ;; - a duplicated immediate and a register
6497 ;; - a duplicated immediate and zero
6498 (define_insn "*vcond_mask_<mode><vpred>"
6499 [(set (match_operand:SVE_FULL 0 "register_operand" "=w, w, w, w, ?w, ?&w, ?&w")
6501 [(match_operand:<VPRED> 3 "register_operand" "Upa, Upa, Upa, Upa, Upl, Upl, Upl")
6502 (match_operand:SVE_FULL 1 "aarch64_sve_reg_or_dup_imm" "w, vss, vss, Ufc, Ufc, vss, Ufc")
6503 (match_operand:SVE_FULL 2 "aarch64_simd_reg_or_zero" "w, 0, Dz, 0, Dz, w, w")]
6506 && (!register_operand (operands[1], <MODE>mode)
6507 || register_operand (operands[2], <MODE>mode))"
6509 sel\t%0.<Vetype>, %3, %1.<Vetype>, %2.<Vetype>
6510 mov\t%0.<Vetype>, %3/m, #%I1
6511 mov\t%0.<Vetype>, %3/z, #%I1
6512 fmov\t%0.<Vetype>, %3/m, #%1
6513 movprfx\t%0.<Vetype>, %3/z, %0.<Vetype>\;fmov\t%0.<Vetype>, %3/m, #%1
6514 movprfx\t%0, %2\;mov\t%0.<Vetype>, %3/m, #%I1
6515 movprfx\t%0, %2\;fmov\t%0.<Vetype>, %3/m, #%1"
6516 [(set_attr "movprfx" "*,*,*,*,yes,yes,yes")]
6519 ;; Optimize selects between a duplicated scalar variable and another vector,
6520 ;; the latter of which can be a zero constant or a variable. Treat duplicates
6521 ;; of GPRs as being more expensive than duplicates of FPRs, since they
6522 ;; involve a cross-file move.
6523 (define_insn "@aarch64_sel_dup<mode>"
6524 [(set (match_operand:SVE_FULL 0 "register_operand" "=?w, w, ??w, ?&w, ??&w, ?&w")
6526 [(match_operand:<VPRED> 3 "register_operand" "Upa, Upa, Upl, Upl, Upl, Upl")
6527 (vec_duplicate:SVE_FULL
6528 (match_operand:<VEL> 1 "register_operand" "r, w, r, w, r, w"))
6529 (match_operand:SVE_FULL 2 "aarch64_simd_reg_or_zero" "0, 0, Dz, Dz, w, w")]
6533 mov\t%0.<Vetype>, %3/m, %<vwcore>1
6534 mov\t%0.<Vetype>, %3/m, %<Vetype>1
6535 movprfx\t%0.<Vetype>, %3/z, %0.<Vetype>\;mov\t%0.<Vetype>, %3/m, %<vwcore>1
6536 movprfx\t%0.<Vetype>, %3/z, %0.<Vetype>\;mov\t%0.<Vetype>, %3/m, %<Vetype>1
6537 movprfx\t%0, %2\;mov\t%0.<Vetype>, %3/m, %<vwcore>1
6538 movprfx\t%0, %2\;mov\t%0.<Vetype>, %3/m, %<Vetype>1"
6539 [(set_attr "movprfx" "*,*,yes,yes,yes,yes")]
6542 ;; -------------------------------------------------------------------------
6543 ;; ---- [INT,FP] Compare and select
6544 ;; -------------------------------------------------------------------------
6545 ;; The patterns in this section are synthetic.
6546 ;; -------------------------------------------------------------------------
6548 ;; Integer (signed) vcond. Don't enforce an immediate range here, since it
6549 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
6550 (define_expand "vcond<mode><v_int_equiv>"
6551 [(set (match_operand:SVE_FULL 0 "register_operand")
6552 (if_then_else:SVE_FULL
6553 (match_operator 3 "comparison_operator"
6554 [(match_operand:<V_INT_EQUIV> 4 "register_operand")
6555 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")])
6556 (match_operand:SVE_FULL 1 "nonmemory_operand")
6557 (match_operand:SVE_FULL 2 "nonmemory_operand")))]
6560 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands);
6565 ;; Integer vcondu. Don't enforce an immediate range here, since it
6566 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
6567 (define_expand "vcondu<mode><v_int_equiv>"
6568 [(set (match_operand:SVE_FULL 0 "register_operand")
6569 (if_then_else:SVE_FULL
6570 (match_operator 3 "comparison_operator"
6571 [(match_operand:<V_INT_EQUIV> 4 "register_operand")
6572 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")])
6573 (match_operand:SVE_FULL 1 "nonmemory_operand")
6574 (match_operand:SVE_FULL 2 "nonmemory_operand")))]
6577 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands);
6582 ;; Floating-point vcond. All comparisons except FCMUO allow a zero operand;
6583 ;; aarch64_expand_sve_vcond handles the case of an FCMUO with zero.
6584 (define_expand "vcond<mode><v_fp_equiv>"
6585 [(set (match_operand:SVE_FULL_HSD 0 "register_operand")
6586 (if_then_else:SVE_FULL_HSD
6587 (match_operator 3 "comparison_operator"
6588 [(match_operand:<V_FP_EQUIV> 4 "register_operand")
6589 (match_operand:<V_FP_EQUIV> 5 "aarch64_simd_reg_or_zero")])
6590 (match_operand:SVE_FULL_HSD 1 "nonmemory_operand")
6591 (match_operand:SVE_FULL_HSD 2 "nonmemory_operand")))]
6594 aarch64_expand_sve_vcond (<MODE>mode, <V_FP_EQUIV>mode, operands);
6599 ;; -------------------------------------------------------------------------
6600 ;; ---- [INT] Comparisons
6601 ;; -------------------------------------------------------------------------
6613 ;; -------------------------------------------------------------------------
6615 ;; Signed integer comparisons. Don't enforce an immediate range here, since
6616 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
6618 (define_expand "vec_cmp<mode><vpred>"
6620 [(set (match_operand:<VPRED> 0 "register_operand")
6621 (match_operator:<VPRED> 1 "comparison_operator"
6622 [(match_operand:SVE_FULL_I 2 "register_operand")
6623 (match_operand:SVE_FULL_I 3 "nonmemory_operand")]))
6624 (clobber (reg:CC_NZC CC_REGNUM))])]
6627 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
6628 operands[2], operands[3]);
6633 ;; Unsigned integer comparisons. Don't enforce an immediate range here, since
6634 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
6636 (define_expand "vec_cmpu<mode><vpred>"
6638 [(set (match_operand:<VPRED> 0 "register_operand")
6639 (match_operator:<VPRED> 1 "comparison_operator"
6640 [(match_operand:SVE_FULL_I 2 "register_operand")
6641 (match_operand:SVE_FULL_I 3 "nonmemory_operand")]))
6642 (clobber (reg:CC_NZC CC_REGNUM))])]
6645 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
6646 operands[2], operands[3]);
6651 ;; Predicated integer comparisons.
6652 (define_insn "@aarch64_pred_cmp<cmp_op><mode>"
6653 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
6655 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6656 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
6657 (SVE_INT_CMP:<VPRED>
6658 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")
6659 (match_operand:SVE_FULL_I 4 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
6661 (clobber (reg:CC_NZC CC_REGNUM))]
6664 cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, #%4
6665 cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
6668 ;; Predicated integer comparisons in which both the flag and predicate
6669 ;; results are interesting.
6670 (define_insn_and_rewrite "*cmp<cmp_op><mode>_cc"
6671 [(set (reg:CC_NZC CC_REGNUM)
6673 [(match_operand:VNx16BI 1 "register_operand" "Upl, Upl")
6675 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
6678 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
6679 (SVE_INT_CMP:<VPRED>
6680 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
6681 (match_operand:SVE_FULL_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
6684 (set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
6688 (SVE_INT_CMP:<VPRED>
6693 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
6695 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
6696 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
6697 "&& !rtx_equal_p (operands[4], operands[6])"
6699 operands[6] = copy_rtx (operands[4]);
6700 operands[7] = operands[5];
6704 ;; Predicated integer comparisons in which only the flags result is
6706 (define_insn_and_rewrite "*cmp<cmp_op><mode>_ptest"
6707 [(set (reg:CC_NZC CC_REGNUM)
6709 [(match_operand:VNx16BI 1 "register_operand" "Upl, Upl")
6711 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
6714 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
6715 (SVE_INT_CMP:<VPRED>
6716 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
6717 (match_operand:SVE_FULL_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
6720 (clobber (match_scratch:<VPRED> 0 "=Upa, Upa"))]
6722 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
6724 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
6725 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
6726 "&& !rtx_equal_p (operands[4], operands[6])"
6728 operands[6] = copy_rtx (operands[4]);
6729 operands[7] = operands[5];
6733 ;; Predicated integer comparisons, formed by combining a PTRUE-predicated
6734 ;; comparison with an AND. Split the instruction into its preferred form
6735 ;; at the earliest opportunity, in order to get rid of the redundant
6737 (define_insn_and_split "*cmp<cmp_op><mode>_and"
6738 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
6742 (const_int SVE_KNOWN_PTRUE)
6743 (SVE_INT_CMP:<VPRED>
6744 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
6745 (match_operand:SVE_FULL_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
6747 (match_operand:<VPRED> 1 "register_operand" "Upl, Upl")))
6748 (clobber (reg:CC_NZC CC_REGNUM))]
6756 (const_int SVE_MAYBE_NOT_PTRUE)
6757 (SVE_INT_CMP:<VPRED>
6761 (clobber (reg:CC_NZC CC_REGNUM))])]
6764 ;; Predicated integer wide comparisons.
6765 (define_insn "@aarch64_pred_cmp<cmp_op><mode>_wide"
6766 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
6768 [(match_operand:VNx16BI 1 "register_operand" "Upl")
6769 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
6771 [(match_operand:SVE_FULL_BHSI 3 "register_operand" "w")
6772 (match_operand:VNx2DI 4 "register_operand" "w")]
6773 SVE_COND_INT_CMP_WIDE)]
6775 (clobber (reg:CC_NZC CC_REGNUM))]
6777 "cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.d"
6780 ;; Predicated integer wide comparisons in which both the flag and
6781 ;; predicate results are interesting.
6782 (define_insn "*aarch64_pred_cmp<cmp_op><mode>_wide_cc"
6783 [(set (reg:CC_NZC CC_REGNUM)
6785 [(match_operand:VNx16BI 1 "register_operand" "Upl")
6787 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
6789 [(match_operand:VNx16BI 6 "register_operand" "Upl")
6790 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
6792 [(match_operand:SVE_FULL_BHSI 2 "register_operand" "w")
6793 (match_operand:VNx2DI 3 "register_operand" "w")]
6794 SVE_COND_INT_CMP_WIDE)]
6797 (set (match_operand:<VPRED> 0 "register_operand" "=Upa")
6804 SVE_COND_INT_CMP_WIDE)]
6807 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
6808 "cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.d"
6811 ;; Predicated integer wide comparisons in which only the flags result
6813 (define_insn "*aarch64_pred_cmp<cmp_op><mode>_wide_ptest"
6814 [(set (reg:CC_NZC CC_REGNUM)
6816 [(match_operand:VNx16BI 1 "register_operand" "Upl")
6818 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
6820 [(match_operand:VNx16BI 6 "register_operand" "Upl")
6821 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
6823 [(match_operand:SVE_FULL_BHSI 2 "register_operand" "w")
6824 (match_operand:VNx2DI 3 "register_operand" "w")]
6825 SVE_COND_INT_CMP_WIDE)]
6828 (clobber (match_scratch:<VPRED> 0 "=Upa"))]
6830 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
6831 "cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.d"
6834 ;; -------------------------------------------------------------------------
6835 ;; ---- [INT] While tests
6836 ;; -------------------------------------------------------------------------
6842 ;; -------------------------------------------------------------------------
6844 ;; Set element I of the result if (cmp (plus operand1 J) operand2) is
6845 ;; true for all J in [0, I].
6846 (define_insn "@while_<while_optab_cmp><GPI:mode><PRED_ALL:mode>"
6847 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
6848 (unspec:PRED_ALL [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
6849 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
6851 (clobber (reg:CC_NZC CC_REGNUM))]
6853 "while<cmp_op>\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
6856 ;; The WHILE instructions set the flags in the same way as a PTEST with
6857 ;; a PTRUE GP. Handle the case in which both results are useful. The GP
6858 ;; operands to the PTEST aren't needed, so we allow them to be anything.
6859 (define_insn_and_rewrite "*while_<while_optab_cmp><GPI:mode><PRED_ALL:mode>_cc"
6860 [(set (reg:CC_NZC CC_REGNUM)
6864 (const_int SVE_KNOWN_PTRUE)
6866 [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
6867 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
6870 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
6871 (unspec:PRED_ALL [(match_dup 1)
6875 "while<cmp_op>\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
6876 ;; Force the compiler to drop the unused predicate operand, so that we
6877 ;; don't have an unnecessary PTRUE.
6878 "&& (!CONSTANT_P (operands[3]) || !CONSTANT_P (operands[4]))"
6880 operands[3] = CONSTM1_RTX (VNx16BImode);
6881 operands[4] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
6885 ;; Same, but handle the case in which only the flags result is useful.
6886 (define_insn_and_rewrite "*while_<while_optab_cmp><GPI:mode><PRED_ALL:mode>_ptest"
6887 [(set (reg:CC_NZC CC_REGNUM)
6891 (const_int SVE_KNOWN_PTRUE)
6893 [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
6894 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
6897 (clobber (match_scratch:PRED_ALL 0 "=Upa"))]
6899 "while<cmp_op>\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
6900 ;; Force the compiler to drop the unused predicate operand, so that we
6901 ;; don't have an unnecessary PTRUE.
6902 "&& (!CONSTANT_P (operands[3]) || !CONSTANT_P (operands[4]))"
6904 operands[3] = CONSTM1_RTX (VNx16BImode);
6905 operands[4] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
6909 ;; -------------------------------------------------------------------------
6910 ;; ---- [FP] Direct comparisons
6911 ;; -------------------------------------------------------------------------
6920 ;; -------------------------------------------------------------------------
6922 ;; Floating-point comparisons. All comparisons except FCMUO allow a zero
6923 ;; operand; aarch64_expand_sve_vec_cmp_float handles the case of an FCMUO
6925 (define_expand "vec_cmp<mode><vpred>"
6926 [(set (match_operand:<VPRED> 0 "register_operand")
6927 (match_operator:<VPRED> 1 "comparison_operator"
6928 [(match_operand:SVE_FULL_F 2 "register_operand")
6929 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]))]
6932 aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
6933 operands[2], operands[3], false);
6938 ;; Predicated floating-point comparisons.
6939 (define_insn "@aarch64_pred_fcm<cmp_op><mode>"
6940 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
6942 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
6943 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
6944 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")
6945 (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero" "Dz, w")]
6946 SVE_COND_FP_CMP_I0))]
6949 fcm<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, #0.0
6950 fcm<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
6953 ;; Same for unordered comparisons.
6954 (define_insn "@aarch64_pred_fcmuo<mode>"
6955 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
6957 [(match_operand:<VPRED> 1 "register_operand" "Upl")
6958 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
6959 (match_operand:SVE_FULL_F 3 "register_operand" "w")
6960 (match_operand:SVE_FULL_F 4 "register_operand" "w")]
6961 UNSPEC_COND_FCMUO))]
6963 "fcmuo\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
6966 ;; Floating-point comparisons predicated on a PTRUE, with the results ANDed
6967 ;; with another predicate P. This does not have the same trapping behavior
6968 ;; as predicating the comparison itself on P, but it's a legitimate fold,
6969 ;; since we can drop any potentially-trapping operations whose results
6972 ;; Split the instruction into its preferred form (below) at the earliest
6973 ;; opportunity, in order to get rid of the redundant operand 1.
6974 (define_insn_and_split "*fcm<cmp_op><mode>_and_combine"
6975 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
6978 [(match_operand:<VPRED> 1)
6979 (const_int SVE_KNOWN_PTRUE)
6980 (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
6981 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
6983 (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))]
6990 (const_int SVE_MAYBE_NOT_PTRUE)
6993 SVE_COND_FP_CMP_I0))]
6996 ;; Same for unordered comparisons.
6997 (define_insn_and_split "*fcmuo<mode>_and_combine"
6998 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
7001 [(match_operand:<VPRED> 1)
7002 (const_int SVE_KNOWN_PTRUE)
7003 (match_operand:SVE_FULL_F 2 "register_operand" "w")
7004 (match_operand:SVE_FULL_F 3 "register_operand" "w")]
7006 (match_operand:<VPRED> 4 "register_operand" "Upl")))]
7013 (const_int SVE_MAYBE_NOT_PTRUE)
7016 UNSPEC_COND_FCMUO))]
7019 ;; -------------------------------------------------------------------------
7020 ;; ---- [FP] Absolute comparisons
7021 ;; -------------------------------------------------------------------------
7027 ;; -------------------------------------------------------------------------
7029 ;; Predicated floating-point absolute comparisons.
7030 (define_expand "@aarch64_pred_fac<cmp_op><mode>"
7031 [(set (match_operand:<VPRED> 0 "register_operand")
7033 [(match_operand:<VPRED> 1 "register_operand")
7034 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
7038 (match_operand:SVE_FULL_F 3 "register_operand")]
7043 (match_operand:SVE_FULL_F 4 "register_operand")]
7045 SVE_COND_FP_ABS_CMP))]
7049 (define_insn_and_rewrite "*aarch64_pred_fac<cmp_op><mode>"
7050 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
7052 [(match_operand:<VPRED> 1 "register_operand" "Upl")
7053 (match_operand:SI 4 "aarch64_sve_ptrue_flag")
7056 (match_operand:SI 6 "aarch64_sve_gp_strictness")
7057 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
7061 (match_operand:SI 8 "aarch64_sve_gp_strictness")
7062 (match_operand:SVE_FULL_F 3 "register_operand" "w")]
7064 SVE_COND_FP_ABS_CMP))]
7066 && aarch64_sve_pred_dominates_p (&operands[5], operands[1])
7067 && aarch64_sve_pred_dominates_p (&operands[7], operands[1])"
7068 "fac<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
7069 "&& (!rtx_equal_p (operands[1], operands[5])
7070 || !rtx_equal_p (operands[1], operands[7]))"
7072 operands[5] = copy_rtx (operands[1]);
7073 operands[7] = copy_rtx (operands[1]);
7077 ;; -------------------------------------------------------------------------
7078 ;; ---- [PRED] Select
7079 ;; -------------------------------------------------------------------------
7082 ;; -------------------------------------------------------------------------
7084 (define_insn "@vcond_mask_<mode><mode>"
7085 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
7088 (match_operand:PRED_ALL 3 "register_operand" "Upa")
7089 (match_operand:PRED_ALL 1 "register_operand" "Upa"))
7092 (match_operand:PRED_ALL 2 "register_operand" "Upa"))))]
7094 "sel\t%0.b, %3, %1.b, %2.b"
7097 ;; -------------------------------------------------------------------------
7098 ;; ---- [PRED] Test bits
7099 ;; -------------------------------------------------------------------------
7102 ;; -------------------------------------------------------------------------
7104 ;; Branch based on predicate equality or inequality.
7105 (define_expand "cbranch<mode>4"
7108 (match_operator 0 "aarch64_equality_operator"
7109 [(match_operand:PRED_ALL 1 "register_operand")
7110 (match_operand:PRED_ALL 2 "aarch64_simd_reg_or_zero")])
7111 (label_ref (match_operand 3 ""))
7115 rtx ptrue = force_reg (VNx16BImode, aarch64_ptrue_all (<data_bytes>));
7116 rtx cast_ptrue = gen_lowpart (<MODE>mode, ptrue);
7117 rtx ptrue_flag = gen_int_mode (SVE_KNOWN_PTRUE, SImode);
7119 if (operands[2] == CONST0_RTX (<MODE>mode))
7123 pred = gen_reg_rtx (<MODE>mode);
7124 emit_insn (gen_aarch64_pred_xor<mode>_z (pred, cast_ptrue, operands[1],
7127 emit_insn (gen_aarch64_ptest<mode> (ptrue, cast_ptrue, ptrue_flag, pred));
7128 operands[1] = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
7129 operands[2] = const0_rtx;
7133 ;; See "Description of UNSPEC_PTEST" above for details.
7134 (define_insn "aarch64_ptest<mode>"
7135 [(set (reg:CC_NZC CC_REGNUM)
7136 (unspec:CC_NZC [(match_operand:VNx16BI 0 "register_operand" "Upa")
7138 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
7139 (match_operand:PRED_ALL 3 "register_operand" "Upa")]
7145 ;; =========================================================================
7147 ;; =========================================================================
7149 ;; -------------------------------------------------------------------------
7150 ;; ---- [INT,FP] Conditional reductions
7151 ;; -------------------------------------------------------------------------
7155 ;; -------------------------------------------------------------------------
7157 ;; Set operand 0 to the last active element in operand 3, or to tied
7158 ;; operand 1 if no elements are active.
7159 (define_insn "@fold_extract_<last_op>_<mode>"
7160 [(set (match_operand:<VEL> 0 "register_operand" "=?r, w")
7162 [(match_operand:<VEL> 1 "register_operand" "0, 0")
7163 (match_operand:<VPRED> 2 "register_operand" "Upl, Upl")
7164 (match_operand:SVE_FULL 3 "register_operand" "w, w")]
7168 clast<ab>\t%<vwcore>0, %2, %<vwcore>0, %3.<Vetype>
7169 clast<ab>\t%<Vetype>0, %2, %<Vetype>0, %3.<Vetype>"
7172 (define_insn "@aarch64_fold_extract_vector_<last_op>_<mode>"
7173 [(set (match_operand:SVE_FULL 0 "register_operand" "=w, ?&w")
7175 [(match_operand:SVE_FULL 1 "register_operand" "0, w")
7176 (match_operand:<VPRED> 2 "register_operand" "Upl, Upl")
7177 (match_operand:SVE_FULL 3 "register_operand" "w, w")]
7181 clast<ab>\t%0.<Vetype>, %2, %0.<Vetype>, %3.<Vetype>
7182 movprfx\t%0, %1\;clast<ab>\t%0.<Vetype>, %2, %0.<Vetype>, %3.<Vetype>"
7185 ;; -------------------------------------------------------------------------
7186 ;; ---- [INT] Tree reductions
7187 ;; -------------------------------------------------------------------------
7198 ;; -------------------------------------------------------------------------
7200 ;; Unpredicated integer add reduction.
7201 (define_expand "reduc_plus_scal_<mode>"
7202 [(match_operand:<VEL> 0 "register_operand")
7203 (match_operand:SVE_FULL_I 1 "register_operand")]
7206 rtx pred = aarch64_ptrue_reg (<VPRED>mode);
7207 rtx tmp = <VEL>mode == DImode ? operands[0] : gen_reg_rtx (DImode);
7208 emit_insn (gen_aarch64_pred_reduc_uadd_<mode> (tmp, pred, operands[1]));
7209 if (tmp != operands[0])
7210 emit_move_insn (operands[0], gen_lowpart (<VEL>mode, tmp));
7215 ;; Predicated integer add reduction. The result is always 64-bits.
7216 (define_insn "@aarch64_pred_reduc_<optab>_<mode>"
7217 [(set (match_operand:DI 0 "register_operand" "=w")
7218 (unspec:DI [(match_operand:<VPRED> 1 "register_operand" "Upl")
7219 (match_operand:SVE_FULL_I 2 "register_operand" "w")]
7221 "TARGET_SVE && <max_elem_bits> >= <elem_bits>"
7222 "<su>addv\t%d0, %1, %2.<Vetype>"
7225 ;; Unpredicated integer reductions.
7226 (define_expand "reduc_<optab>_scal_<mode>"
7227 [(set (match_operand:<VEL> 0 "register_operand")
7228 (unspec:<VEL> [(match_dup 2)
7229 (match_operand:SVE_FULL_I 1 "register_operand")]
7230 SVE_INT_REDUCTION))]
7233 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
7237 ;; Predicated integer reductions.
7238 (define_insn "@aarch64_pred_reduc_<optab>_<mode>"
7239 [(set (match_operand:<VEL> 0 "register_operand" "=w")
7240 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
7241 (match_operand:SVE_FULL_I 2 "register_operand" "w")]
7242 SVE_INT_REDUCTION))]
7244 "<sve_int_op>\t%<Vetype>0, %1, %2.<Vetype>"
7247 ;; -------------------------------------------------------------------------
7248 ;; ---- [FP] Tree reductions
7249 ;; -------------------------------------------------------------------------
7256 ;; -------------------------------------------------------------------------
7258 ;; Unpredicated floating-point tree reductions.
7259 (define_expand "reduc_<optab>_scal_<mode>"
7260 [(set (match_operand:<VEL> 0 "register_operand")
7261 (unspec:<VEL> [(match_dup 2)
7262 (match_operand:SVE_FULL_F 1 "register_operand")]
7266 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
7270 ;; Predicated floating-point tree reductions.
7271 (define_insn "@aarch64_pred_reduc_<optab>_<mode>"
7272 [(set (match_operand:<VEL> 0 "register_operand" "=w")
7273 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
7274 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
7277 "<sve_fp_op>\t%<Vetype>0, %1, %2.<Vetype>"
7280 ;; -------------------------------------------------------------------------
7281 ;; ---- [FP] Left-to-right reductions
7282 ;; -------------------------------------------------------------------------
7285 ;; -------------------------------------------------------------------------
7287 ;; Unpredicated in-order FP reductions.
7288 (define_expand "fold_left_plus_<mode>"
7289 [(set (match_operand:<VEL> 0 "register_operand")
7290 (unspec:<VEL> [(match_dup 3)
7291 (match_operand:<VEL> 1 "register_operand")
7292 (match_operand:SVE_FULL_F 2 "register_operand")]
7296 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
7300 ;; Predicated in-order FP reductions.
7301 (define_insn "mask_fold_left_plus_<mode>"
7302 [(set (match_operand:<VEL> 0 "register_operand" "=w")
7303 (unspec:<VEL> [(match_operand:<VPRED> 3 "register_operand" "Upl")
7304 (match_operand:<VEL> 1 "register_operand" "0")
7305 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
7308 "fadda\t%<Vetype>0, %3, %<Vetype>0, %2.<Vetype>"
7311 ;; =========================================================================
7313 ;; =========================================================================
7315 ;; -------------------------------------------------------------------------
7316 ;; ---- [INT,FP] General permutes
7317 ;; -------------------------------------------------------------------------
7320 ;; -------------------------------------------------------------------------
7322 (define_expand "vec_perm<mode>"
7323 [(match_operand:SVE_FULL 0 "register_operand")
7324 (match_operand:SVE_FULL 1 "register_operand")
7325 (match_operand:SVE_FULL 2 "register_operand")
7326 (match_operand:<V_INT_EQUIV> 3 "aarch64_sve_vec_perm_operand")]
7327 "TARGET_SVE && GET_MODE_NUNITS (<MODE>mode).is_constant ()"
7329 aarch64_expand_sve_vec_perm (operands[0], operands[1],
7330 operands[2], operands[3]);
7335 (define_insn "@aarch64_sve_tbl<mode>"
7336 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
7338 [(match_operand:SVE_FULL 1 "register_operand" "w")
7339 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")]
7342 "tbl\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
7345 ;; -------------------------------------------------------------------------
7346 ;; ---- [INT,FP] Special-purpose unary permutes
7347 ;; -------------------------------------------------------------------------
7352 ;; -------------------------------------------------------------------------
7354 ;; Compact active elements and pad with zeros.
7355 (define_insn "@aarch64_sve_compact<mode>"
7356 [(set (match_operand:SVE_FULL_SD 0 "register_operand" "=w")
7358 [(match_operand:<VPRED> 1 "register_operand" "Upl")
7359 (match_operand:SVE_FULL_SD 2 "register_operand" "w")]
7360 UNSPEC_SVE_COMPACT))]
7362 "compact\t%0.<Vetype>, %1, %2.<Vetype>"
7365 ;; Duplicate one element of a vector.
7366 (define_insn "@aarch64_sve_dup_lane<mode>"
7367 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
7368 (vec_duplicate:SVE_FULL
7370 (match_operand:SVE_FULL 1 "register_operand" "w")
7371 (parallel [(match_operand:SI 2 "const_int_operand")]))))]
7373 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 0, 63)"
7374 "dup\t%0.<Vetype>, %1.<Vetype>[%2]"
7377 ;; Use DUP.Q to duplicate a 128-bit segment of a register.
7379 ;; The vec_select:<V128> sets memory lane number N of the V128 to lane
7380 ;; number op2 + N of op1. (We don't need to distinguish between memory
7381 ;; and architectural register lane numbering for op1 or op0, since the
7382 ;; two numbering schemes are the same for SVE.)
7384 ;; The vec_duplicate:SVE_FULL then copies memory lane number N of the
7385 ;; V128 (and thus lane number op2 + N of op1) to lane numbers N + I * STEP
7386 ;; of op0. We therefore get the correct result for both endiannesses.
7388 ;; The wrinkle is that for big-endian V128 registers, memory lane numbering
7389 ;; is in the opposite order to architectural register lane numbering.
7390 ;; Thus if we were to do this operation via a V128 temporary register,
7391 ;; the vec_select and vec_duplicate would both involve a reverse operation
7392 ;; for big-endian targets. In this fused pattern the two reverses cancel
7394 (define_insn "@aarch64_sve_dupq_lane<mode>"
7395 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
7396 (vec_duplicate:SVE_FULL
7398 (match_operand:SVE_FULL 1 "register_operand" "w")
7399 (match_operand 2 "ascending_int_parallel"))))]
7401 && (INTVAL (XVECEXP (operands[2], 0, 0))
7402 * GET_MODE_SIZE (<VEL>mode)) % 16 == 0
7403 && IN_RANGE (INTVAL (XVECEXP (operands[2], 0, 0))
7404 * GET_MODE_SIZE (<VEL>mode), 0, 63)"
7406 unsigned int byte = (INTVAL (XVECEXP (operands[2], 0, 0))
7407 * GET_MODE_SIZE (<VEL>mode));
7408 operands[2] = gen_int_mode (byte / 16, DImode);
7409 return "dup\t%0.q, %1.q[%2]";
7413 ;; Reverse the order of elements within a full vector.
7414 (define_insn "@aarch64_sve_rev<mode>"
7415 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
7417 [(match_operand:SVE_FULL 1 "register_operand" "w")]
7420 "rev\t%0.<Vetype>, %1.<Vetype>")
7422 ;; -------------------------------------------------------------------------
7423 ;; ---- [INT,FP] Special-purpose binary permutes
7424 ;; -------------------------------------------------------------------------
7433 ;; -------------------------------------------------------------------------
7435 ;; Like EXT, but start at the first active element.
7436 (define_insn "@aarch64_sve_splice<mode>"
7437 [(set (match_operand:SVE_FULL 0 "register_operand" "=w, ?&w")
7439 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
7440 (match_operand:SVE_FULL 2 "register_operand" "0, w")
7441 (match_operand:SVE_FULL 3 "register_operand" "w, w")]
7442 UNSPEC_SVE_SPLICE))]
7445 splice\t%0.<Vetype>, %1, %0.<Vetype>, %3.<Vetype>
7446 movprfx\t%0, %2\;splice\t%0.<Vetype>, %1, %0.<Vetype>, %3.<Vetype>"
7447 [(set_attr "movprfx" "*, yes")]
7450 ;; Permutes that take half the elements from one vector and half the
7451 ;; elements from the other.
7452 (define_insn "@aarch64_sve_<perm_insn><mode>"
7453 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
7455 [(match_operand:SVE_FULL 1 "register_operand" "w")
7456 (match_operand:SVE_FULL 2 "register_operand" "w")]
7459 "<perm_insn>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
7462 ;; Concatenate two vectors and extract a subvector. Note that the
7463 ;; immediate (third) operand is the lane index not the byte index.
7464 (define_insn "@aarch64_sve_ext<mode>"
7465 [(set (match_operand:SVE_FULL 0 "register_operand" "=w, ?&w")
7467 [(match_operand:SVE_FULL 1 "register_operand" "0, w")
7468 (match_operand:SVE_FULL 2 "register_operand" "w, w")
7469 (match_operand:SI 3 "const_int_operand")]
7472 && IN_RANGE (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode), 0, 255)"
7474 operands[3] = GEN_INT (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode));
7475 return (which_alternative == 0
7476 ? "ext\\t%0.b, %0.b, %2.b, #%3"
7477 : "movprfx\t%0, %1\;ext\\t%0.b, %0.b, %2.b, #%3");
7479 [(set_attr "movprfx" "*,yes")]
7482 ;; -------------------------------------------------------------------------
7483 ;; ---- [PRED] Special-purpose unary permutes
7484 ;; -------------------------------------------------------------------------
7487 ;; -------------------------------------------------------------------------
7489 (define_insn "@aarch64_sve_rev<mode>"
7490 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
7491 (unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa")]
7494 "rev\t%0.<Vetype>, %1.<Vetype>")
7496 ;; -------------------------------------------------------------------------
7497 ;; ---- [PRED] Special-purpose binary permutes
7498 ;; -------------------------------------------------------------------------
7506 ;; -------------------------------------------------------------------------
7508 ;; Permutes that take half the elements from one vector and half the
7509 ;; elements from the other.
7510 (define_insn "@aarch64_sve_<perm_insn><mode>"
7511 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
7512 (unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa")
7513 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
7516 "<perm_insn>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
7519 ;; =========================================================================
7521 ;; =========================================================================
7523 ;; -------------------------------------------------------------------------
7524 ;; ---- [INT<-INT] Packs
7525 ;; -------------------------------------------------------------------------
7528 ;; -------------------------------------------------------------------------
7530 ;; Integer pack. Use UZP1 on the narrower type, which discards
7531 ;; the high part of each wide element.
7532 (define_insn "vec_pack_trunc_<Vwide>"
7533 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand" "=w")
7534 (unspec:SVE_FULL_BHSI
7535 [(match_operand:<VWIDE> 1 "register_operand" "w")
7536 (match_operand:<VWIDE> 2 "register_operand" "w")]
7539 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
7542 ;; -------------------------------------------------------------------------
7543 ;; ---- [INT<-INT] Unpacks
7544 ;; -------------------------------------------------------------------------
7550 ;; -------------------------------------------------------------------------
7552 ;; Unpack the low or high half of a vector, where "high" refers to
7553 ;; the low-numbered lanes for big-endian and the high-numbered lanes
7554 ;; for little-endian.
7555 (define_expand "vec_unpack<su>_<perm_hilo>_<SVE_FULL_BHSI:mode>"
7556 [(match_operand:<VWIDE> 0 "register_operand")
7558 [(match_operand:SVE_FULL_BHSI 1 "register_operand")] UNPACK)]
7561 emit_insn ((<hi_lanes_optab>
7562 ? gen_aarch64_sve_<su>unpkhi_<SVE_FULL_BHSI:mode>
7563 : gen_aarch64_sve_<su>unpklo_<SVE_FULL_BHSI:mode>)
7564 (operands[0], operands[1]));
7569 (define_insn "@aarch64_sve_<su>unpk<perm_hilo>_<SVE_FULL_BHSI:mode>"
7570 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
7572 [(match_operand:SVE_FULL_BHSI 1 "register_operand" "w")]
7575 "<su>unpk<perm_hilo>\t%0.<Vewtype>, %1.<Vetype>"
7578 ;; -------------------------------------------------------------------------
7579 ;; ---- [INT<-FP] Conversions
7580 ;; -------------------------------------------------------------------------
7584 ;; -------------------------------------------------------------------------
7586 ;; Unpredicated conversion of floats to integers of the same size (HF to HI,
7587 ;; SF to SI or DF to DI).
7588 (define_expand "<optab><mode><v_int_equiv>2"
7589 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
7590 (unspec:<V_INT_EQUIV>
7592 (const_int SVE_RELAXED_GP)
7593 (match_operand:SVE_FULL_F 1 "register_operand")]
7597 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
7601 ;; Predicated float-to-integer conversion, either to the same width or wider.
7602 (define_insn "@aarch64_sve_<optab>_nontrunc<SVE_FULL_F:mode><SVE_FULL_HSDI:mode>"
7603 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
7604 (unspec:SVE_FULL_HSDI
7605 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl")
7606 (match_operand:SI 3 "aarch64_sve_gp_strictness")
7607 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
7609 "TARGET_SVE && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>"
7610 "fcvtz<su>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_F:Vetype>"
7613 ;; Predicated narrowing float-to-integer conversion.
7614 (define_insn "@aarch64_sve_<optab>_trunc<VNx2DF_ONLY:mode><VNx4SI_ONLY:mode>"
7615 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=w")
7617 [(match_operand:VNx2BI 1 "register_operand" "Upl")
7618 (match_operand:SI 3 "aarch64_sve_gp_strictness")
7619 (match_operand:VNx2DF_ONLY 2 "register_operand" "w")]
7622 "fcvtz<su>\t%0.<VNx4SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>"
7625 ;; Predicated float-to-integer conversion with merging, either to the same
7627 (define_expand "@cond_<optab>_nontrunc<SVE_FULL_F:mode><SVE_FULL_HSDI:mode>"
7628 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
7629 (unspec:SVE_FULL_HSDI
7630 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand")
7631 (unspec:SVE_FULL_HSDI
7633 (const_int SVE_STRICT_GP)
7634 (match_operand:SVE_FULL_F 2 "register_operand")]
7636 (match_operand:SVE_FULL_HSDI 3 "aarch64_simd_reg_or_zero")]
7638 "TARGET_SVE && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>"
7641 ;; The first alternative doesn't need the earlyclobber, but the only case
7642 ;; it would help is the uninteresting one in which operands 2 and 3 are
7643 ;; the same register (despite having different modes). Making all the
7644 ;; alternatives earlyclobber makes things more consistent for the
7645 ;; register allocator.
7646 (define_insn_and_rewrite "*cond_<optab>_nontrunc<SVE_FULL_F:mode><SVE_FULL_HSDI:mode>"
7647 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=&w, &w, ?&w")
7648 (unspec:SVE_FULL_HSDI
7649 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl, Upl, Upl")
7650 (unspec:SVE_FULL_HSDI
7652 (match_operand:SI 5 "aarch64_sve_gp_strictness")
7653 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")]
7655 (match_operand:SVE_FULL_HSDI 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
7658 && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>
7659 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
7661 fcvtz<su>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_F:Vetype>
7662 movprfx\t%0.<SVE_FULL_HSDI:Vetype>, %1/z, %2.<SVE_FULL_HSDI:Vetype>\;fcvtz<su>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_F:Vetype>
7663 movprfx\t%0, %3\;fcvtz<su>\t%0.<SVE_FULL_HSDI:Vetype>, %1/m, %2.<SVE_FULL_F:Vetype>"
7664 "&& !rtx_equal_p (operands[1], operands[4])"
7666 operands[4] = copy_rtx (operands[1]);
7668 [(set_attr "movprfx" "*,yes,yes")]
7671 ;; Predicated narrowing float-to-integer conversion with merging.
7672 (define_expand "@cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx4SI_ONLY:mode>"
7673 [(set (match_operand:VNx4SI_ONLY 0 "register_operand")
7675 [(match_operand:VNx2BI 1 "register_operand")
7678 (const_int SVE_STRICT_GP)
7679 (match_operand:VNx2DF_ONLY 2 "register_operand")]
7681 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero")]
7686 (define_insn "*cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx4SI_ONLY:mode>"
7687 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=&w, &w, ?&w")
7689 [(match_operand:VNx2BI 1 "register_operand" "Upl, Upl, Upl")
7692 (match_operand:SI 4 "aarch64_sve_gp_strictness")
7693 (match_operand:VNx2DF_ONLY 2 "register_operand" "w, w, w")]
7695 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
7699 fcvtz<su>\t%0.<VNx4SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
7700 movprfx\t%0.<VNx2DF_ONLY:Vetype>, %1/z, %2.<VNx2DF_ONLY:Vetype>\;fcvtz<su>\t%0.<VNx4SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
7701 movprfx\t%0, %3\;fcvtz<su>\t%0.<VNx4SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>"
7702 [(set_attr "movprfx" "*,yes,yes")]
7705 ;; -------------------------------------------------------------------------
7706 ;; ---- [INT<-FP] Packs
7707 ;; -------------------------------------------------------------------------
7708 ;; The patterns in this section are synthetic.
7709 ;; -------------------------------------------------------------------------
7711 ;; Convert two vectors of DF to SI and pack the results into a single vector.
7712 (define_expand "vec_pack_<su>fix_trunc_vnx2df"
7716 (const_int SVE_RELAXED_GP)
7717 (match_operand:VNx2DF 1 "register_operand")]
7722 (const_int SVE_RELAXED_GP)
7723 (match_operand:VNx2DF 2 "register_operand")]
7725 (set (match_operand:VNx4SI 0 "register_operand")
7726 (unspec:VNx4SI [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
7729 operands[3] = aarch64_ptrue_reg (VNx2BImode);
7730 operands[4] = gen_reg_rtx (VNx4SImode);
7731 operands[5] = gen_reg_rtx (VNx4SImode);
7735 ;; -------------------------------------------------------------------------
7736 ;; ---- [INT<-FP] Unpacks
7737 ;; -------------------------------------------------------------------------
7738 ;; No patterns here yet!
7739 ;; -------------------------------------------------------------------------
7741 ;; -------------------------------------------------------------------------
7742 ;; ---- [FP<-INT] Conversions
7743 ;; -------------------------------------------------------------------------
7747 ;; -------------------------------------------------------------------------
7749 ;; Unpredicated conversion of integers to floats of the same size
7750 ;; (HI to HF, SI to SF or DI to DF).
7751 (define_expand "<optab><v_int_equiv><mode>2"
7752 [(set (match_operand:SVE_FULL_F 0 "register_operand")
7755 (const_int SVE_RELAXED_GP)
7756 (match_operand:<V_INT_EQUIV> 1 "register_operand")]
7760 operands[2] = aarch64_ptrue_reg (<VPRED>mode);
7764 ;; Predicated integer-to-float conversion, either to the same width or
7766 (define_insn "@aarch64_sve_<optab>_nonextend<SVE_FULL_HSDI:mode><SVE_FULL_F:mode>"
7767 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
7769 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl")
7770 (match_operand:SI 3 "aarch64_sve_gp_strictness")
7771 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")]
7773 "TARGET_SVE && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>"
7774 "<su>cvtf\t%0.<SVE_FULL_F:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>"
7777 ;; Predicated widening integer-to-float conversion.
7778 (define_insn "@aarch64_sve_<optab>_extend<VNx4SI_ONLY:mode><VNx2DF_ONLY:mode>"
7779 [(set (match_operand:VNx2DF_ONLY 0 "register_operand" "=w")
7781 [(match_operand:VNx2BI 1 "register_operand" "Upl")
7782 (match_operand:SI 3 "aarch64_sve_gp_strictness")
7783 (match_operand:VNx4SI_ONLY 2 "register_operand" "w")]
7786 "<su>cvtf\t%0.<VNx2DF_ONLY:Vetype>, %1/m, %2.<VNx4SI_ONLY:Vetype>"
7789 ;; Predicated integer-to-float conversion with merging, either to the same
7790 ;; width or narrower.
7791 (define_expand "@cond_<optab>_nonextend<SVE_FULL_HSDI:mode><SVE_FULL_F:mode>"
7792 [(set (match_operand:SVE_FULL_F 0 "register_operand")
7794 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand")
7797 (const_int SVE_STRICT_GP)
7798 (match_operand:SVE_FULL_HSDI 2 "register_operand")]
7800 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]
7802 "TARGET_SVE && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>"
7805 ;; The first alternative doesn't need the earlyclobber, but the only case
7806 ;; it would help is the uninteresting one in which operands 2 and 3 are
7807 ;; the same register (despite having different modes). Making all the
7808 ;; alternatives earlyclobber makes things more consistent for the
7809 ;; register allocator.
7810 (define_insn_and_rewrite "*cond_<optab>_nonextend<SVE_FULL_HSDI:mode><SVE_FULL_F:mode>"
7811 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=&w, &w, ?&w")
7813 [(match_operand:<SVE_FULL_HSDI:VPRED> 1 "register_operand" "Upl, Upl, Upl")
7816 (match_operand:SI 5 "aarch64_sve_gp_strictness")
7817 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w, w")]
7819 (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
7822 && <SVE_FULL_HSDI:elem_bits> >= <SVE_FULL_F:elem_bits>
7823 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
7825 <su>cvtf\t%0.<SVE_FULL_F:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>
7826 movprfx\t%0.<SVE_FULL_HSDI:Vetype>, %1/z, %2.<SVE_FULL_HSDI:Vetype>\;<su>cvtf\t%0.<SVE_FULL_F:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>
7827 movprfx\t%0, %3\;<su>cvtf\t%0.<SVE_FULL_F:Vetype>, %1/m, %2.<SVE_FULL_HSDI:Vetype>"
7828 "&& !rtx_equal_p (operands[1], operands[4])"
7830 operands[4] = copy_rtx (operands[1]);
7832 [(set_attr "movprfx" "*,yes,yes")]
7835 ;; Predicated widening integer-to-float conversion with merging.
7836 (define_expand "@cond_<optab>_extend<VNx4SI_ONLY:mode><VNx2DF_ONLY:mode>"
7837 [(set (match_operand:VNx2DF_ONLY 0 "register_operand")
7839 [(match_operand:VNx2BI 1 "register_operand")
7842 (const_int SVE_STRICT_GP)
7843 (match_operand:VNx4SI_ONLY 2 "register_operand")]
7845 (match_operand:VNx2DF_ONLY 3 "aarch64_simd_reg_or_zero")]
7850 (define_insn "*cond_<optab>_extend<VNx4SI_ONLY:mode><VNx2DF_ONLY:mode>"
7851 [(set (match_operand:VNx2DF_ONLY 0 "register_operand" "=w, ?&w, ?&w")
7853 [(match_operand:VNx2BI 1 "register_operand" "Upl, Upl, Upl")
7856 (match_operand:SI 4 "aarch64_sve_gp_strictness")
7857 (match_operand:VNx4SI_ONLY 2 "register_operand" "w, w, w")]
7859 (match_operand:VNx2DF_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
7863 <su>cvtf\t%0.<VNx2DF_ONLY:Vetype>, %1/m, %2.<VNx4SI_ONLY:Vetype>
7864 movprfx\t%0.<VNx2DF_ONLY:Vetype>, %1/z, %2.<VNx2DF_ONLY:Vetype>\;<su>cvtf\t%0.<VNx2DF_ONLY:Vetype>, %1/m, %2.<VNx4SI_ONLY:Vetype>
7865 movprfx\t%0, %3\;<su>cvtf\t%0.<VNx2DF_ONLY:Vetype>, %1/m, %2.<VNx4SI_ONLY:Vetype>"
7866 [(set_attr "movprfx" "*,yes,yes")]
7869 ;; -------------------------------------------------------------------------
7870 ;; ---- [FP<-INT] Packs
7871 ;; -------------------------------------------------------------------------
7872 ;; No patterns here yet!
7873 ;; -------------------------------------------------------------------------
7875 ;; -------------------------------------------------------------------------
7876 ;; ---- [FP<-INT] Unpacks
7877 ;; -------------------------------------------------------------------------
7878 ;; The patterns in this section are synthetic.
7879 ;; -------------------------------------------------------------------------
7881 ;; Unpack one half of a VNx4SI to VNx2DF. First unpack from VNx4SI
7882 ;; to VNx2DI, reinterpret the VNx2DI as a VNx4SI, then convert the
7883 ;; unpacked VNx4SI to VNx2DF.
7884 (define_expand "vec_unpack<su_optab>_float_<perm_hilo>_vnx4si"
7885 [(match_operand:VNx2DF 0 "register_operand")
7887 (unspec:VNx2DI [(match_operand:VNx4SI 1 "register_operand")]
7891 /* Use ZIP to do the unpack, since we don't care about the upper halves
7892 and since it has the nice property of not needing any subregs.
7893 If using UUNPK* turns out to be preferable, we could model it as
7894 a ZIP whose first operand is zero. */
7895 rtx temp = gen_reg_rtx (VNx4SImode);
7896 emit_insn ((<hi_lanes_optab>
7897 ? gen_aarch64_sve_zip2vnx4si
7898 : gen_aarch64_sve_zip1vnx4si)
7899 (temp, operands[1], operands[1]));
7900 rtx ptrue = aarch64_ptrue_reg (VNx2BImode);
7901 rtx strictness = gen_int_mode (SVE_RELAXED_GP, SImode);
7902 emit_insn (gen_aarch64_sve_<FLOATUORS:optab>_extendvnx4sivnx2df
7903 (operands[0], ptrue, temp, strictness));
7908 ;; -------------------------------------------------------------------------
7909 ;; ---- [FP<-FP] Packs
7910 ;; -------------------------------------------------------------------------
7913 ;; -------------------------------------------------------------------------
7915 ;; Convert two vectors of DF to SF, or two vectors of SF to HF, and pack
7916 ;; the results into a single vector.
7917 (define_expand "vec_pack_trunc_<Vwide>"
7919 (unspec:SVE_FULL_HSF
7921 (const_int SVE_RELAXED_GP)
7922 (match_operand:<VWIDE> 1 "register_operand")]
7925 (unspec:SVE_FULL_HSF
7927 (const_int SVE_RELAXED_GP)
7928 (match_operand:<VWIDE> 2 "register_operand")]
7930 (set (match_operand:SVE_FULL_HSF 0 "register_operand")
7931 (unspec:SVE_FULL_HSF [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
7934 operands[3] = aarch64_ptrue_reg (<VWIDE_PRED>mode);
7935 operands[4] = gen_reg_rtx (<MODE>mode);
7936 operands[5] = gen_reg_rtx (<MODE>mode);
7940 ;; Predicated float-to-float truncation.
7941 (define_insn "@aarch64_sve_<optab>_trunc<SVE_FULL_SDF:mode><SVE_FULL_HSF:mode>"
7942 [(set (match_operand:SVE_FULL_HSF 0 "register_operand" "=w")
7943 (unspec:SVE_FULL_HSF
7944 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand" "Upl")
7945 (match_operand:SI 3 "aarch64_sve_gp_strictness")
7946 (match_operand:SVE_FULL_SDF 2 "register_operand" "w")]
7948 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
7949 "fcvt\t%0.<SVE_FULL_HSF:Vetype>, %1/m, %2.<SVE_FULL_SDF:Vetype>"
7952 ;; Predicated float-to-float truncation with merging.
7953 (define_expand "@cond_<optab>_trunc<SVE_FULL_SDF:mode><SVE_FULL_HSF:mode>"
7954 [(set (match_operand:SVE_FULL_HSF 0 "register_operand")
7955 (unspec:SVE_FULL_HSF
7956 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand")
7957 (unspec:SVE_FULL_HSF
7959 (const_int SVE_STRICT_GP)
7960 (match_operand:SVE_FULL_SDF 2 "register_operand")]
7962 (match_operand:SVE_FULL_HSF 3 "aarch64_simd_reg_or_zero")]
7964 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
7967 (define_insn "*cond_<optab>_trunc<SVE_FULL_SDF:mode><SVE_FULL_HSF:mode>"
7968 [(set (match_operand:SVE_FULL_HSF 0 "register_operand" "=w, ?&w, ?&w")
7969 (unspec:SVE_FULL_HSF
7970 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand" "Upl, Upl, Upl")
7971 (unspec:SVE_FULL_HSF
7973 (match_operand:SI 4 "aarch64_sve_gp_strictness")
7974 (match_operand:SVE_FULL_SDF 2 "register_operand" "w, w, w")]
7976 (match_operand:SVE_FULL_HSF 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
7978 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
7980 fcvt\t%0.<SVE_FULL_HSF:Vetype>, %1/m, %2.<SVE_FULL_SDF:Vetype>
7981 movprfx\t%0.<SVE_FULL_SDF:Vetype>, %1/z, %2.<SVE_FULL_SDF:Vetype>\;fcvt\t%0.<SVE_FULL_HSF:Vetype>, %1/m, %2.<SVE_FULL_SDF:Vetype>
7982 movprfx\t%0, %3\;fcvt\t%0.<SVE_FULL_HSF:Vetype>, %1/m, %2.<SVE_FULL_SDF:Vetype>"
7983 [(set_attr "movprfx" "*,yes,yes")]
7986 ;; -------------------------------------------------------------------------
7987 ;; ---- [FP<-FP] Unpacks
7988 ;; -------------------------------------------------------------------------
7991 ;; -------------------------------------------------------------------------
7993 ;; Unpack one half of a VNx4SF to VNx2DF, or one half of a VNx8HF to VNx4SF.
7994 ;; First unpack the source without conversion, then float-convert the
7996 (define_expand "vec_unpacks_<perm_hilo>_<mode>"
7997 [(match_operand:<VWIDE> 0 "register_operand")
7998 (unspec:SVE_FULL_HSF
7999 [(match_operand:SVE_FULL_HSF 1 "register_operand")]
8003 /* Use ZIP to do the unpack, since we don't care about the upper halves
8004 and since it has the nice property of not needing any subregs.
8005 If using UUNPK* turns out to be preferable, we could model it as
8006 a ZIP whose first operand is zero. */
8007 rtx temp = gen_reg_rtx (<MODE>mode);
8008 emit_insn ((<hi_lanes_optab>
8009 ? gen_aarch64_sve_zip2<mode>
8010 : gen_aarch64_sve_zip1<mode>)
8011 (temp, operands[1], operands[1]));
8012 rtx ptrue = aarch64_ptrue_reg (<VWIDE_PRED>mode);
8013 rtx strictness = gen_int_mode (SVE_RELAXED_GP, SImode);
8014 emit_insn (gen_aarch64_sve_fcvt_nontrunc<mode><Vwide>
8015 (operands[0], ptrue, temp, strictness));
8020 ;; Predicated float-to-float extension.
8021 (define_insn "@aarch64_sve_<optab>_nontrunc<SVE_FULL_HSF:mode><SVE_FULL_SDF:mode>"
8022 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w")
8023 (unspec:SVE_FULL_SDF
8024 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand" "Upl")
8025 (match_operand:SI 3 "aarch64_sve_gp_strictness")
8026 (match_operand:SVE_FULL_HSF 2 "register_operand" "w")]
8028 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
8029 "fcvt\t%0.<SVE_FULL_SDF:Vetype>, %1/m, %2.<SVE_FULL_HSF:Vetype>"
8032 ;; Predicated float-to-float extension with merging.
8033 (define_expand "@cond_<optab>_nontrunc<SVE_FULL_HSF:mode><SVE_FULL_SDF:mode>"
8034 [(set (match_operand:SVE_FULL_SDF 0 "register_operand")
8035 (unspec:SVE_FULL_SDF
8036 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand")
8037 (unspec:SVE_FULL_SDF
8039 (const_int SVE_STRICT_GP)
8040 (match_operand:SVE_FULL_HSF 2 "register_operand")]
8042 (match_operand:SVE_FULL_SDF 3 "aarch64_simd_reg_or_zero")]
8044 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
8047 (define_insn "*cond_<optab>_nontrunc<SVE_FULL_HSF:mode><SVE_FULL_SDF:mode>"
8048 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w, ?&w, ?&w")
8049 (unspec:SVE_FULL_SDF
8050 [(match_operand:<SVE_FULL_SDF:VPRED> 1 "register_operand" "Upl, Upl, Upl")
8051 (unspec:SVE_FULL_SDF
8053 (match_operand:SI 4 "aarch64_sve_gp_strictness")
8054 (match_operand:SVE_FULL_HSF 2 "register_operand" "w, w, w")]
8056 (match_operand:SVE_FULL_SDF 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
8058 "TARGET_SVE && <SVE_FULL_SDF:elem_bits> > <SVE_FULL_HSF:elem_bits>"
8060 fcvt\t%0.<SVE_FULL_SDF:Vetype>, %1/m, %2.<SVE_FULL_HSF:Vetype>
8061 movprfx\t%0.<SVE_FULL_SDF:Vetype>, %1/z, %2.<SVE_FULL_SDF:Vetype>\;fcvt\t%0.<SVE_FULL_SDF:Vetype>, %1/m, %2.<SVE_FULL_HSF:Vetype>
8062 movprfx\t%0, %3\;fcvt\t%0.<SVE_FULL_SDF:Vetype>, %1/m, %2.<SVE_FULL_HSF:Vetype>"
8063 [(set_attr "movprfx" "*,yes,yes")]
8066 ;; -------------------------------------------------------------------------
8067 ;; ---- [PRED<-PRED] Packs
8068 ;; -------------------------------------------------------------------------
8071 ;; -------------------------------------------------------------------------
8073 ;; Predicate pack. Use UZP1 on the narrower type, which discards
8074 ;; the high part of each wide element.
8075 (define_insn "vec_pack_trunc_<Vwide>"
8076 [(set (match_operand:PRED_BHS 0 "register_operand" "=Upa")
8078 [(match_operand:<VWIDE> 1 "register_operand" "Upa")
8079 (match_operand:<VWIDE> 2 "register_operand" "Upa")]
8082 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
8085 ;; -------------------------------------------------------------------------
8086 ;; ---- [PRED<-PRED] Unpacks
8087 ;; -------------------------------------------------------------------------
8091 ;; -------------------------------------------------------------------------
8093 ;; Unpack the low or high half of a predicate, where "high" refers to
8094 ;; the low-numbered lanes for big-endian and the high-numbered lanes
8095 ;; for little-endian.
8096 (define_expand "vec_unpack<su>_<perm_hilo>_<mode>"
8097 [(match_operand:<VWIDE> 0 "register_operand")
8098 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand")]
8102 emit_insn ((<hi_lanes_optab>
8103 ? gen_aarch64_sve_punpkhi_<PRED_BHS:mode>
8104 : gen_aarch64_sve_punpklo_<PRED_BHS:mode>)
8105 (operands[0], operands[1]));
8110 (define_insn "@aarch64_sve_punpk<perm_hilo>_<mode>"
8111 [(set (match_operand:<VWIDE> 0 "register_operand" "=Upa")
8112 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand" "Upa")]
8115 "punpk<perm_hilo>\t%0.h, %1.b"
8118 ;; =========================================================================
8119 ;; == Vector partitioning
8120 ;; =========================================================================
8122 ;; -------------------------------------------------------------------------
8123 ;; ---- [PRED] Unary partitioning
8124 ;; -------------------------------------------------------------------------
8130 ;; -------------------------------------------------------------------------
8132 ;; Note that unlike most other instructions that have both merging and
8133 ;; zeroing forms, these instructions don't operate elementwise and so
8134 ;; don't fit the IFN_COND model.
8135 (define_insn "@aarch64_brk<brk_op>"
8136 [(set (match_operand:VNx16BI 0 "register_operand" "=Upa, Upa")
8138 [(match_operand:VNx16BI 1 "register_operand" "Upa, Upa")
8139 (match_operand:VNx16BI 2 "register_operand" "Upa, Upa")
8140 (match_operand:VNx16BI 3 "aarch64_simd_reg_or_zero" "Dz, 0")]
8144 brk<brk_op>\t%0.b, %1/z, %2.b
8145 brk<brk_op>\t%0.b, %1/m, %2.b"
8148 ;; Same, but also producing a flags result.
8149 (define_insn "*aarch64_brk<brk_op>_cc"
8150 [(set (reg:CC_NZC CC_REGNUM)
8152 [(match_operand:VNx16BI 1 "register_operand" "Upa, Upa")
8154 (match_operand:SI 4 "aarch64_sve_ptrue_flag")
8157 (match_operand:VNx16BI 2 "register_operand" "Upa, Upa")
8158 (match_operand:VNx16BI 3 "aarch64_simd_reg_or_zero" "Dz, 0")]
8161 (set (match_operand:VNx16BI 0 "register_operand" "=Upa, Upa")
8169 brk<brk_op>s\t%0.b, %1/z, %2.b
8170 brk<brk_op>s\t%0.b, %1/m, %2.b"
8173 ;; Same, but with only the flags result being interesting.
8174 (define_insn "*aarch64_brk<brk_op>_ptest"
8175 [(set (reg:CC_NZC CC_REGNUM)
8177 [(match_operand:VNx16BI 1 "register_operand" "Upa, Upa")
8179 (match_operand:SI 4 "aarch64_sve_ptrue_flag")
8182 (match_operand:VNx16BI 2 "register_operand" "Upa, Upa")
8183 (match_operand:VNx16BI 3 "aarch64_simd_reg_or_zero" "Dz, 0")]
8186 (clobber (match_scratch:VNx16BI 0 "=Upa, Upa"))]
8189 brk<brk_op>s\t%0.b, %1/z, %2.b
8190 brk<brk_op>s\t%0.b, %1/m, %2.b"
8193 ;; -------------------------------------------------------------------------
8194 ;; ---- [PRED] Binary partitioning
8195 ;; -------------------------------------------------------------------------
8203 ;; -------------------------------------------------------------------------
8205 ;; Binary BRKs (BRKN, BRKPA, BRKPB).
8206 (define_insn "@aarch64_brk<brk_op>"
8207 [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
8209 [(match_operand:VNx16BI 1 "register_operand" "Upa")
8210 (match_operand:VNx16BI 2 "register_operand" "Upa")
8211 (match_operand:VNx16BI 3 "register_operand" "<brk_reg_con>")]
8214 "brk<brk_op>\t%0.b, %1/z, %2.b, %<brk_reg_opno>.b"
8217 ;; Same, but also producing a flags result.
8218 (define_insn "*aarch64_brk<brk_op>_cc"
8219 [(set (reg:CC_NZC CC_REGNUM)
8221 [(match_operand:VNx16BI 1 "register_operand" "Upa")
8223 (match_operand:SI 4 "aarch64_sve_ptrue_flag")
8226 (match_operand:VNx16BI 2 "register_operand" "Upa")
8227 (match_operand:VNx16BI 3 "register_operand" "<brk_reg_con>")]
8230 (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
8237 "brk<brk_op>s\t%0.b, %1/z, %2.b, %<brk_reg_opno>.b"
8240 ;; Same, but with only the flags result being interesting.
8241 (define_insn "*aarch64_brk<brk_op>_ptest"
8242 [(set (reg:CC_NZC CC_REGNUM)
8244 [(match_operand:VNx16BI 1 "register_operand" "Upa")
8246 (match_operand:SI 4 "aarch64_sve_ptrue_flag")
8249 (match_operand:VNx16BI 2 "register_operand" "Upa")
8250 (match_operand:VNx16BI 3 "register_operand" "<brk_reg_con>")]
8253 (clobber (match_scratch:VNx16BI 0 "=Upa"))]
8255 "brk<brk_op>s\t%0.b, %1/z, %2.b, %<brk_reg_opno>.b"
8258 ;; -------------------------------------------------------------------------
8259 ;; ---- [PRED] Scalarization
8260 ;; -------------------------------------------------------------------------
8264 ;; -------------------------------------------------------------------------
8266 (define_insn "@aarch64_sve_<sve_pred_op><mode>"
8267 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
8269 [(match_operand:PRED_ALL 1 "register_operand" "Upa")
8270 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
8271 (match_operand:PRED_ALL 3 "register_operand" "0")]
8273 (clobber (reg:CC_NZC CC_REGNUM))]
8274 "TARGET_SVE && <max_elem_bits> >= <elem_bits>"
8275 "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
8278 ;; Same, but also producing a flags result.
8279 (define_insn_and_rewrite "*aarch64_sve_<sve_pred_op><mode>_cc"
8280 [(set (reg:CC_NZC CC_REGNUM)
8282 [(match_operand:VNx16BI 1 "register_operand" "Upa")
8284 (match_operand:SI 3 "aarch64_sve_ptrue_flag")
8287 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
8288 (match_operand:PRED_ALL 6 "register_operand" "0")]
8291 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
8298 && <max_elem_bits> >= <elem_bits>
8299 && aarch64_sve_same_pred_for_ptest_p (&operands[2], &operands[4])"
8300 "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
8301 "&& !rtx_equal_p (operands[2], operands[4])"
8303 operands[4] = operands[2];
8304 operands[5] = operands[3];
8308 ;; Same, but with only the flags result being interesting.
8309 (define_insn_and_rewrite "*aarch64_sve_<sve_pred_op><mode>_ptest"
8310 [(set (reg:CC_NZC CC_REGNUM)
8312 [(match_operand:VNx16BI 1 "register_operand" "Upa")
8314 (match_operand:SI 3 "aarch64_sve_ptrue_flag")
8317 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
8318 (match_operand:PRED_ALL 6 "register_operand" "0")]
8321 (clobber (match_scratch:PRED_ALL 0 "=Upa"))]
8323 && <max_elem_bits> >= <elem_bits>
8324 && aarch64_sve_same_pred_for_ptest_p (&operands[2], &operands[4])"
8325 "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
8326 "&& !rtx_equal_p (operands[2], operands[4])"
8328 operands[4] = operands[2];
8329 operands[5] = operands[3];
8333 ;; =========================================================================
8334 ;; == Counting elements
8335 ;; =========================================================================
8337 ;; -------------------------------------------------------------------------
8338 ;; ---- [INT] Count elements in a pattern (scalar)
8339 ;; -------------------------------------------------------------------------
8345 ;; -------------------------------------------------------------------------
8347 ;; Count the number of elements in an svpattern. Operand 1 is the pattern,
8348 ;; operand 2 is the number of elements that fit in a 128-bit block, and
8349 ;; operand 3 is a multiplier in the range [1, 16].
8351 ;; Note that this pattern isn't used for SV_ALL (but would work for that too).
8352 (define_insn "aarch64_sve_cnt_pat"
8353 [(set (match_operand:DI 0 "register_operand" "=r")
8355 (unspec:SI [(match_operand:DI 1 "const_int_operand")
8356 (match_operand:DI 2 "const_int_operand")
8357 (match_operand:DI 3 "const_int_operand")]
8358 UNSPEC_SVE_CNT_PAT)))]
8361 return aarch64_output_sve_cnt_pat_immediate ("cnt", "%x0", operands + 1);
8365 ;; -------------------------------------------------------------------------
8366 ;; ---- [INT] Increment by the number of elements in a pattern (scalar)
8367 ;; -------------------------------------------------------------------------
8372 ;; -------------------------------------------------------------------------
8374 ;; Increment a DImode register by the number of elements in an svpattern.
8375 ;; See aarch64_sve_cnt_pat for the counting behavior.
8376 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8377 [(set (match_operand:DI 0 "register_operand" "=r")
8378 (ANY_PLUS:DI (zero_extend:DI
8379 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8380 (match_operand:DI 3 "const_int_operand")
8381 (match_operand:DI 4 "const_int_operand")]
8382 UNSPEC_SVE_CNT_PAT))
8383 (match_operand:DI_ONLY 1 "register_operand" "0")))]
8386 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%x0",
8391 ;; Increment an SImode register by the number of elements in an svpattern
8392 ;; using modular arithmetic. See aarch64_sve_cnt_pat for the counting
8394 (define_insn "*aarch64_sve_incsi_pat"
8395 [(set (match_operand:SI 0 "register_operand" "=r")
8396 (plus:SI (unspec:SI [(match_operand:DI 2 "const_int_operand")
8397 (match_operand:DI 3 "const_int_operand")
8398 (match_operand:DI 4 "const_int_operand")]
8400 (match_operand:SI 1 "register_operand" "0")))]
8403 return aarch64_output_sve_cnt_pat_immediate ("inc", "%x0", operands + 2);
8407 ;; Increment an SImode register by the number of elements in an svpattern
8408 ;; using saturating arithmetic, extending the result to 64 bits.
8410 ;; See aarch64_sve_cnt_pat for the counting behavior.
8411 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8412 [(set (match_operand:DI 0 "register_operand" "=r")
8415 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8416 (match_operand:DI 3 "const_int_operand")
8417 (match_operand:DI 4 "const_int_operand")]
8419 (match_operand:SI_ONLY 1 "register_operand" "0"))))]
8422 const char *registers = (<CODE> == SS_PLUS ? "%x0, %w0" : "%w0");
8423 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", registers,
8428 ;; -------------------------------------------------------------------------
8429 ;; ---- [INT] Increment by the number of elements in a pattern (vector)
8430 ;; -------------------------------------------------------------------------
8435 ;; -------------------------------------------------------------------------
8437 ;; Increment a vector of DIs by the number of elements in an svpattern.
8438 ;; See aarch64_sve_cnt_pat for the counting behavior.
8439 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8440 [(set (match_operand:VNx2DI 0 "register_operand" "=w, ?&w")
8442 (vec_duplicate:VNx2DI
8444 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8445 (match_operand:DI 3 "const_int_operand")
8446 (match_operand:DI 4 "const_int_operand")]
8447 UNSPEC_SVE_CNT_PAT)))
8448 (match_operand:VNx2DI_ONLY 1 "register_operand" "0, w")))]
8451 if (which_alternative == 1)
8452 output_asm_insn ("movprfx\t%0, %1", operands);
8453 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8456 [(set_attr "movprfx" "*,yes")]
8459 ;; Increment a vector of SIs by the number of elements in an svpattern.
8460 ;; See aarch64_sve_cnt_pat for the counting behavior.
8461 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8462 [(set (match_operand:VNx4SI 0 "register_operand" "=w, ?&w")
8464 (vec_duplicate:VNx4SI
8465 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8466 (match_operand:DI 3 "const_int_operand")
8467 (match_operand:DI 4 "const_int_operand")]
8468 UNSPEC_SVE_CNT_PAT))
8469 (match_operand:VNx4SI_ONLY 1 "register_operand" "0, w")))]
8472 if (which_alternative == 1)
8473 output_asm_insn ("movprfx\t%0, %1", operands);
8474 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8477 [(set_attr "movprfx" "*,yes")]
8480 ;; Increment a vector of HIs by the number of elements in an svpattern.
8481 ;; See aarch64_sve_cnt_pat for the counting behavior.
8482 (define_expand "@aarch64_sve_<inc_dec><mode>_pat"
8483 [(set (match_operand:VNx8HI 0 "register_operand")
8485 (vec_duplicate:VNx8HI
8487 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8488 (match_operand:DI 3 "const_int_operand")
8489 (match_operand:DI 4 "const_int_operand")]
8490 UNSPEC_SVE_CNT_PAT)))
8491 (match_operand:VNx8HI_ONLY 1 "register_operand")))]
8495 (define_insn "*aarch64_sve_<inc_dec><mode>_pat"
8496 [(set (match_operand:VNx8HI 0 "register_operand" "=w, ?&w")
8498 (vec_duplicate:VNx8HI
8499 (match_operator:HI 5 "subreg_lowpart_operator"
8500 [(unspec:SI [(match_operand:DI 2 "const_int_operand")
8501 (match_operand:DI 3 "const_int_operand")
8502 (match_operand:DI 4 "const_int_operand")]
8503 UNSPEC_SVE_CNT_PAT)]))
8504 (match_operand:VNx8HI_ONLY 1 "register_operand" "0, w")))]
8507 if (which_alternative == 1)
8508 output_asm_insn ("movprfx\t%0, %1", operands);
8509 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8512 [(set_attr "movprfx" "*,yes")]
8515 ;; -------------------------------------------------------------------------
8516 ;; ---- [INT] Decrement by the number of elements in a pattern (scalar)
8517 ;; -------------------------------------------------------------------------
8522 ;; -------------------------------------------------------------------------
8524 ;; Decrement a DImode register by the number of elements in an svpattern.
8525 ;; See aarch64_sve_cnt_pat for the counting behavior.
8526 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8527 [(set (match_operand:DI 0 "register_operand" "=r")
8528 (ANY_MINUS:DI (match_operand:DI_ONLY 1 "register_operand" "0")
8530 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8531 (match_operand:DI 3 "const_int_operand")
8532 (match_operand:DI 4 "const_int_operand")]
8533 UNSPEC_SVE_CNT_PAT))))]
8536 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%x0",
8541 ;; Decrement an SImode register by the number of elements in an svpattern
8542 ;; using modular arithmetic. See aarch64_sve_cnt_pat for the counting
8544 (define_insn "*aarch64_sve_decsi_pat"
8545 [(set (match_operand:SI 0 "register_operand" "=r")
8546 (minus:SI (match_operand:SI 1 "register_operand" "0")
8547 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8548 (match_operand:DI 3 "const_int_operand")
8549 (match_operand:DI 4 "const_int_operand")]
8550 UNSPEC_SVE_CNT_PAT)))]
8553 return aarch64_output_sve_cnt_pat_immediate ("dec", "%x0", operands + 2);
8557 ;; Decrement an SImode register by the number of elements in an svpattern
8558 ;; using saturating arithmetic, extending the result to 64 bits.
8560 ;; See aarch64_sve_cnt_pat for the counting behavior.
8561 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8562 [(set (match_operand:DI 0 "register_operand" "=r")
8565 (match_operand:SI_ONLY 1 "register_operand" "0")
8566 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8567 (match_operand:DI 3 "const_int_operand")
8568 (match_operand:DI 4 "const_int_operand")]
8569 UNSPEC_SVE_CNT_PAT))))]
8572 const char *registers = (<CODE> == SS_MINUS ? "%x0, %w0" : "%w0");
8573 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", registers,
8578 ;; -------------------------------------------------------------------------
8579 ;; ---- [INT] Decrement by the number of elements in a pattern (vector)
8580 ;; -------------------------------------------------------------------------
8585 ;; -------------------------------------------------------------------------
8587 ;; Decrement a vector of DIs by the number of elements in an svpattern.
8588 ;; See aarch64_sve_cnt_pat for the counting behavior.
8589 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8590 [(set (match_operand:VNx2DI 0 "register_operand" "=w, ?&w")
8592 (match_operand:VNx2DI_ONLY 1 "register_operand" "0, w")
8593 (vec_duplicate:VNx2DI
8595 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8596 (match_operand:DI 3 "const_int_operand")
8597 (match_operand:DI 4 "const_int_operand")]
8598 UNSPEC_SVE_CNT_PAT)))))]
8601 if (which_alternative == 1)
8602 output_asm_insn ("movprfx\t%0, %1", operands);
8603 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8606 [(set_attr "movprfx" "*,yes")]
8609 ;; Decrement a vector of SIs by the number of elements in an svpattern.
8610 ;; See aarch64_sve_cnt_pat for the counting behavior.
8611 (define_insn "@aarch64_sve_<inc_dec><mode>_pat"
8612 [(set (match_operand:VNx4SI 0 "register_operand" "=w, ?&w")
8614 (match_operand:VNx4SI_ONLY 1 "register_operand" "0, w")
8615 (vec_duplicate:VNx4SI
8616 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8617 (match_operand:DI 3 "const_int_operand")
8618 (match_operand:DI 4 "const_int_operand")]
8619 UNSPEC_SVE_CNT_PAT))))]
8622 if (which_alternative == 1)
8623 output_asm_insn ("movprfx\t%0, %1", operands);
8624 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8627 [(set_attr "movprfx" "*,yes")]
8630 ;; Decrement a vector of HIs by the number of elements in an svpattern.
8631 ;; See aarch64_sve_cnt_pat for the counting behavior.
8632 (define_expand "@aarch64_sve_<inc_dec><mode>_pat"
8633 [(set (match_operand:VNx8HI 0 "register_operand")
8635 (match_operand:VNx8HI_ONLY 1 "register_operand")
8636 (vec_duplicate:VNx8HI
8638 (unspec:SI [(match_operand:DI 2 "const_int_operand")
8639 (match_operand:DI 3 "const_int_operand")
8640 (match_operand:DI 4 "const_int_operand")]
8641 UNSPEC_SVE_CNT_PAT)))))]
8645 (define_insn "*aarch64_sve_<inc_dec><mode>_pat"
8646 [(set (match_operand:VNx8HI 0 "register_operand" "=w, ?&w")
8648 (match_operand:VNx8HI_ONLY 1 "register_operand" "0, w")
8649 (vec_duplicate:VNx8HI
8650 (match_operator:HI 5 "subreg_lowpart_operator"
8651 [(unspec:SI [(match_operand:DI 2 "const_int_operand")
8652 (match_operand:DI 3 "const_int_operand")
8653 (match_operand:DI 4 "const_int_operand")]
8654 UNSPEC_SVE_CNT_PAT)]))))]
8657 if (which_alternative == 1)
8658 output_asm_insn ("movprfx\t%0, %1", operands);
8659 return aarch64_output_sve_cnt_pat_immediate ("<inc_dec>", "%0.<Vetype>",
8662 [(set_attr "movprfx" "*,yes")]
8665 ;; -------------------------------------------------------------------------
8666 ;; ---- [INT] Count elements in a predicate (scalar)
8667 ;; -------------------------------------------------------------------------
8670 ;; -------------------------------------------------------------------------
8672 ;; Count the number of set bits in a predicate. Operand 3 is true if
8673 ;; operand 1 is known to be all-true.
8674 (define_insn "@aarch64_pred_cntp<mode>"
8675 [(set (match_operand:DI 0 "register_operand" "=r")
8677 (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upl")
8678 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
8679 (match_operand:PRED_ALL 3 "register_operand" "Upa")]
8682 "cntp\t%x0, %1, %3.<Vetype>")
8684 ;; -------------------------------------------------------------------------
8685 ;; ---- [INT] Increment by the number of elements in a predicate (scalar)
8686 ;; -------------------------------------------------------------------------
8691 ;; -------------------------------------------------------------------------
8693 ;; Increment a DImode register by the number of set bits in a predicate.
8694 ;; See aarch64_sve_cntp for a description of the operands.
8695 (define_expand "@aarch64_sve_<inc_dec><DI_ONLY:mode><PRED_ALL:mode>_cntp"
8696 [(set (match_operand:DI 0 "register_operand")
8699 (unspec:SI [(match_dup 3)
8700 (const_int SVE_KNOWN_PTRUE)
8701 (match_operand:PRED_ALL 2 "register_operand")]
8703 (match_operand:DI_ONLY 1 "register_operand")))]
8706 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8710 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><DI_ONLY:mode><PRED_ALL:mode>_cntp"
8711 [(set (match_operand:DI 0 "register_operand" "=r")
8714 (unspec:SI [(match_operand 3)
8715 (const_int SVE_KNOWN_PTRUE)
8716 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
8718 (match_operand:DI_ONLY 1 "register_operand" "0")))]
8720 "<inc_dec>p\t%x0, %2.<PRED_ALL:Vetype>"
8721 "&& !CONSTANT_P (operands[3])"
8723 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8727 ;; Increment an SImode register by the number of set bits in a predicate
8728 ;; using modular arithmetic. See aarch64_sve_cntp for a description of
8730 (define_insn_and_rewrite "*aarch64_incsi<mode>_cntp"
8731 [(set (match_operand:SI 0 "register_operand" "=r")
8733 (unspec:SI [(match_operand 3)
8734 (const_int SVE_KNOWN_PTRUE)
8735 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
8737 (match_operand:SI 1 "register_operand" "0")))]
8739 "incp\t%x0, %2.<Vetype>"
8740 "&& !CONSTANT_P (operands[3])"
8742 operands[3] = CONSTM1_RTX (<MODE>mode);
8746 ;; Increment an SImode register by the number of set bits in a predicate
8747 ;; using saturating arithmetic, extending the result to 64 bits.
8749 ;; See aarch64_sve_cntp for a description of the operands.
8750 (define_expand "@aarch64_sve_<inc_dec><SI_ONLY:mode><PRED_ALL:mode>_cntp"
8751 [(set (match_operand:DI 0 "register_operand")
8754 (unspec:SI [(match_dup 3)
8755 (const_int SVE_KNOWN_PTRUE)
8756 (match_operand:PRED_ALL 2 "register_operand")]
8758 (match_operand:SI_ONLY 1 "register_operand"))))]
8761 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8765 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><SI_ONLY:mode><PRED_ALL:mode>_cntp"
8766 [(set (match_operand:DI 0 "register_operand" "=r")
8769 (unspec:SI [(match_operand 3)
8770 (const_int SVE_KNOWN_PTRUE)
8771 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
8773 (match_operand:SI_ONLY 1 "register_operand" "0"))))]
8776 if (<CODE> == SS_PLUS)
8777 return "<inc_dec>p\t%x0, %2.<PRED_ALL:Vetype>, %w0";
8779 return "<inc_dec>p\t%w0, %2.<PRED_ALL:Vetype>";
8781 "&& !CONSTANT_P (operands[3])"
8783 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8787 ;; -------------------------------------------------------------------------
8788 ;; ---- [INT] Increment by the number of elements in a predicate (vector)
8789 ;; -------------------------------------------------------------------------
8794 ;; -------------------------------------------------------------------------
8796 ;; Increment a vector of DIs by the number of set bits in a predicate.
8797 ;; See aarch64_sve_cntp for a description of the operands.
8798 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
8799 [(set (match_operand:VNx2DI 0 "register_operand")
8801 (vec_duplicate:VNx2DI
8805 (const_int SVE_KNOWN_PTRUE)
8806 (match_operand:<VPRED> 2 "register_operand")]
8808 (match_operand:VNx2DI_ONLY 1 "register_operand")))]
8811 operands[3] = CONSTM1_RTX (<VPRED>mode);
8815 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
8816 [(set (match_operand:VNx2DI 0 "register_operand" "=w, ?&w")
8818 (vec_duplicate:VNx2DI
8822 (const_int SVE_KNOWN_PTRUE)
8823 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
8825 (match_operand:VNx2DI_ONLY 1 "register_operand" "0, w")))]
8828 <inc_dec>p\t%0.d, %2
8829 movprfx\t%0, %1\;<inc_dec>p\t%0.d, %2"
8830 "&& !CONSTANT_P (operands[3])"
8832 operands[3] = CONSTM1_RTX (<VPRED>mode);
8834 [(set_attr "movprfx" "*,yes")]
8837 ;; Increment a vector of SIs by the number of set bits in a predicate.
8838 ;; See aarch64_sve_cntp for a description of the operands.
8839 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
8840 [(set (match_operand:VNx4SI 0 "register_operand")
8842 (vec_duplicate:VNx4SI
8845 (const_int SVE_KNOWN_PTRUE)
8846 (match_operand:<VPRED> 2 "register_operand")]
8848 (match_operand:VNx4SI_ONLY 1 "register_operand")))]
8851 operands[3] = CONSTM1_RTX (<VPRED>mode);
8855 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
8856 [(set (match_operand:VNx4SI 0 "register_operand" "=w, ?&w")
8858 (vec_duplicate:VNx4SI
8861 (const_int SVE_KNOWN_PTRUE)
8862 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
8864 (match_operand:VNx4SI_ONLY 1 "register_operand" "0, w")))]
8867 <inc_dec>p\t%0.s, %2
8868 movprfx\t%0, %1\;<inc_dec>p\t%0.s, %2"
8869 "&& !CONSTANT_P (operands[3])"
8871 operands[3] = CONSTM1_RTX (<VPRED>mode);
8873 [(set_attr "movprfx" "*,yes")]
8876 ;; Increment a vector of HIs by the number of set bits in a predicate.
8877 ;; See aarch64_sve_cntp for a description of the operands.
8878 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
8879 [(set (match_operand:VNx8HI 0 "register_operand")
8881 (vec_duplicate:VNx8HI
8885 (const_int SVE_KNOWN_PTRUE)
8886 (match_operand:<VPRED> 2 "register_operand")]
8888 (match_operand:VNx8HI_ONLY 1 "register_operand")))]
8891 operands[3] = CONSTM1_RTX (<VPRED>mode);
8895 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
8896 [(set (match_operand:VNx8HI 0 "register_operand" "=w, ?&w")
8898 (vec_duplicate:VNx8HI
8899 (match_operator:HI 3 "subreg_lowpart_operator"
8902 (const_int SVE_KNOWN_PTRUE)
8903 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
8905 (match_operand:VNx8HI_ONLY 1 "register_operand" "0, w")))]
8908 <inc_dec>p\t%0.h, %2
8909 movprfx\t%0, %1\;<inc_dec>p\t%0.h, %2"
8910 "&& !CONSTANT_P (operands[4])"
8912 operands[4] = CONSTM1_RTX (<VPRED>mode);
8914 [(set_attr "movprfx" "*,yes")]
8917 ;; -------------------------------------------------------------------------
8918 ;; ---- [INT] Decrement by the number of elements in a predicate (scalar)
8919 ;; -------------------------------------------------------------------------
8924 ;; -------------------------------------------------------------------------
8926 ;; Decrement a DImode register by the number of set bits in a predicate.
8927 ;; See aarch64_sve_cntp for a description of the operands.
8928 (define_expand "@aarch64_sve_<inc_dec><DI_ONLY:mode><PRED_ALL:mode>_cntp"
8929 [(set (match_operand:DI 0 "register_operand")
8931 (match_operand:DI_ONLY 1 "register_operand")
8933 (unspec:SI [(match_dup 3)
8934 (const_int SVE_KNOWN_PTRUE)
8935 (match_operand:PRED_ALL 2 "register_operand")]
8939 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8943 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><DI_ONLY:mode><PRED_ALL:mode>_cntp"
8944 [(set (match_operand:DI 0 "register_operand" "=r")
8946 (match_operand:DI_ONLY 1 "register_operand" "0")
8948 (unspec:SI [(match_operand 3)
8949 (const_int SVE_KNOWN_PTRUE)
8950 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
8953 "<inc_dec>p\t%x0, %2.<PRED_ALL:Vetype>"
8954 "&& !CONSTANT_P (operands[3])"
8956 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8960 ;; Decrement an SImode register by the number of set bits in a predicate
8961 ;; using modular arithmetic. See aarch64_sve_cntp for a description of the
8963 (define_insn_and_rewrite "*aarch64_decsi<mode>_cntp"
8964 [(set (match_operand:SI 0 "register_operand" "=r")
8966 (match_operand:SI 1 "register_operand" "0")
8967 (unspec:SI [(match_operand 3)
8968 (const_int SVE_KNOWN_PTRUE)
8969 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
8972 "decp\t%x0, %2.<Vetype>"
8973 "&& !CONSTANT_P (operands[3])"
8975 operands[3] = CONSTM1_RTX (<MODE>mode);
8979 ;; Decrement an SImode register by the number of set bits in a predicate
8980 ;; using saturating arithmetic, extending the result to 64 bits.
8982 ;; See aarch64_sve_cntp for a description of the operands.
8983 (define_expand "@aarch64_sve_<inc_dec><SI_ONLY:mode><PRED_ALL:mode>_cntp"
8984 [(set (match_operand:DI 0 "register_operand")
8987 (match_operand:SI_ONLY 1 "register_operand")
8988 (unspec:SI [(match_dup 3)
8989 (const_int SVE_KNOWN_PTRUE)
8990 (match_operand:PRED_ALL 2 "register_operand")]
8994 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
8998 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><SI_ONLY:mode><PRED_ALL:mode>_cntp"
8999 [(set (match_operand:DI 0 "register_operand" "=r")
9002 (match_operand:SI_ONLY 1 "register_operand" "0")
9003 (unspec:SI [(match_operand 3)
9004 (const_int SVE_KNOWN_PTRUE)
9005 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
9009 if (<CODE> == SS_MINUS)
9010 return "<inc_dec>p\t%x0, %2.<PRED_ALL:Vetype>, %w0";
9012 return "<inc_dec>p\t%w0, %2.<PRED_ALL:Vetype>";
9014 "&& !CONSTANT_P (operands[3])"
9016 operands[3] = CONSTM1_RTX (<PRED_ALL:MODE>mode);
9020 ;; -------------------------------------------------------------------------
9021 ;; ---- [INT] Decrement by the number of elements in a predicate (vector)
9022 ;; -------------------------------------------------------------------------
9027 ;; -------------------------------------------------------------------------
9029 ;; Decrement a vector of DIs by the number of set bits in a predicate.
9030 ;; See aarch64_sve_cntp for a description of the operands.
9031 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
9032 [(set (match_operand:VNx2DI 0 "register_operand")
9034 (match_operand:VNx2DI_ONLY 1 "register_operand")
9035 (vec_duplicate:VNx2DI
9039 (const_int SVE_KNOWN_PTRUE)
9040 (match_operand:<VPRED> 2 "register_operand")]
9044 operands[3] = CONSTM1_RTX (<VPRED>mode);
9048 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
9049 [(set (match_operand:VNx2DI 0 "register_operand" "=w, ?&w")
9051 (match_operand:VNx2DI_ONLY 1 "register_operand" "0, w")
9052 (vec_duplicate:VNx2DI
9056 (const_int SVE_KNOWN_PTRUE)
9057 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
9061 <inc_dec>p\t%0.d, %2
9062 movprfx\t%0, %1\;<inc_dec>p\t%0.d, %2"
9063 "&& !CONSTANT_P (operands[3])"
9065 operands[3] = CONSTM1_RTX (<VPRED>mode);
9067 [(set_attr "movprfx" "*,yes")]
9070 ;; Decrement a vector of SIs by the number of set bits in a predicate.
9071 ;; See aarch64_sve_cntp for a description of the operands.
9072 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
9073 [(set (match_operand:VNx4SI 0 "register_operand")
9075 (match_operand:VNx4SI_ONLY 1 "register_operand")
9076 (vec_duplicate:VNx4SI
9079 (const_int SVE_KNOWN_PTRUE)
9080 (match_operand:<VPRED> 2 "register_operand")]
9084 operands[3] = CONSTM1_RTX (<VPRED>mode);
9088 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
9089 [(set (match_operand:VNx4SI 0 "register_operand" "=w, ?&w")
9091 (match_operand:VNx4SI_ONLY 1 "register_operand" "0, w")
9092 (vec_duplicate:VNx4SI
9095 (const_int SVE_KNOWN_PTRUE)
9096 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
9100 <inc_dec>p\t%0.s, %2
9101 movprfx\t%0, %1\;<inc_dec>p\t%0.s, %2"
9102 "&& !CONSTANT_P (operands[3])"
9104 operands[3] = CONSTM1_RTX (<VPRED>mode);
9106 [(set_attr "movprfx" "*,yes")]
9109 ;; Decrement a vector of HIs by the number of set bits in a predicate.
9110 ;; See aarch64_sve_cntp for a description of the operands.
9111 (define_expand "@aarch64_sve_<inc_dec><mode>_cntp"
9112 [(set (match_operand:VNx8HI 0 "register_operand")
9114 (match_operand:VNx8HI_ONLY 1 "register_operand")
9115 (vec_duplicate:VNx8HI
9119 (const_int SVE_KNOWN_PTRUE)
9120 (match_operand:<VPRED> 2 "register_operand")]
9124 operands[3] = CONSTM1_RTX (<VPRED>mode);
9128 (define_insn_and_rewrite "*aarch64_sve_<inc_dec><mode>_cntp"
9129 [(set (match_operand:VNx8HI 0 "register_operand" "=w, ?&w")
9131 (match_operand:VNx8HI_ONLY 1 "register_operand" "0, w")
9132 (vec_duplicate:VNx8HI
9133 (match_operator:HI 3 "subreg_lowpart_operator"
9136 (const_int SVE_KNOWN_PTRUE)
9137 (match_operand:<VPRED> 2 "register_operand" "Upa, Upa")]
9141 <inc_dec>p\t%0.h, %2
9142 movprfx\t%0, %1\;<inc_dec>p\t%0.h, %2"
9143 "&& !CONSTANT_P (operands[4])"
9145 operands[4] = CONSTM1_RTX (<VPRED>mode);
9147 [(set_attr "movprfx" "*,yes")]