1 ;; Machine description for AArch64 SVE2.
2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; The file is organised into the following sections (search for the full
25 ;; ---- Non-temporal gather loads
26 ;; ---- Non-temporal scatter stores
28 ;; == Uniform binary arithmnetic
29 ;; ---- [INT] Multiplication
30 ;; ---- [INT] Scaled high-part multiplication
31 ;; ---- [INT] General binary arithmetic that maps to unspecs
32 ;; ---- [INT] Saturating binary arithmetic
33 ;; ---- [INT] Saturating left shifts
35 ;; == Uniform ternary arithmnetic
36 ;; ---- [INT] General ternary arithmetic that maps to unspecs
37 ;; ---- [INT] Multiply-and-accumulate operations
38 ;; ---- [INT] Binary logic operations with rotation
39 ;; ---- [INT] Ternary logic operations
40 ;; ---- [INT] Shift-and-accumulate operations
41 ;; ---- [INT] Shift-and-insert operations
42 ;; ---- [INT] Sum of absolute differences
44 ;; == Extending arithmetic
45 ;; ---- [INT] Wide binary arithmetic
46 ;; ---- [INT] Long binary arithmetic
47 ;; ---- [INT] Long left shifts
48 ;; ---- [INT] Long binary arithmetic with accumulation
49 ;; ---- [FP] Long multiplication with accumulation
51 ;; == Narrowing arithnetic
52 ;; ---- [INT] Narrowing unary arithmetic
53 ;; ---- [INT] Narrowing binary arithmetic
54 ;; ---- [INT] Narrowing right shifts
56 ;; == Pairwise arithmetic
57 ;; ---- [INT] Pairwise arithmetic
58 ;; ---- [FP] Pairwise arithmetic
59 ;; ---- [INT] Pairwise arithmetic with accumulation
61 ;; == Complex arithmetic
62 ;; ---- [INT] Complex binary operations
63 ;; ---- [INT] Complex ternary operations
64 ;; ---- [INT] Complex dot product
67 ;; ---- [FP<-FP] Widening conversions
68 ;; ---- [FP<-FP] Narrowing conversions
70 ;; == Other arithmetic
71 ;; ---- [INT] Reciprocal approximation
72 ;; ---- [INT<-FP] Base-2 logarithm
73 ;; ---- [INT] Polynomial multiplication
76 ;; ---- [INT,FP] General permutes
77 ;; ---- [INT] Optional bit-permute extensions
80 ;; ---- Check for aliases between pointers
81 ;; ---- Histogram processing
82 ;; ---- String matching
84 ;; == Crypotographic extensions
85 ;; ---- Optional AES extensions
86 ;; ---- Optional SHA-3 extensions
87 ;; ---- Optional SM4 extensions
89 ;; =========================================================================
91 ;; =========================================================================
93 ;; -------------------------------------------------------------------------
94 ;; ---- Non-temporal gather loads
95 ;; -------------------------------------------------------------------------
96 ;; Includes gather forms of:
101 ;; -------------------------------------------------------------------------
103 ;; Non-extending loads.
104 (define_insn "@aarch64_gather_ldnt<mode>"
105 [(set (match_operand:SVE_FULL_SD 0 "register_operand" "=w, w")
107 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
108 (match_operand:DI 2 "aarch64_reg_or_zero" "Z, r")
109 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w, w")
111 UNSPEC_LDNT1_GATHER))]
114 ldnt1<Vesize>\t%0.<Vetype>, %1/z, [%3.<Vetype>]
115 ldnt1<Vesize>\t%0.<Vetype>, %1/z, [%3.<Vetype>, %2]"
119 (define_insn_and_rewrite "@aarch64_gather_ldnt_<ANY_EXTEND:optab><SVE_FULL_SDI:mode><SVE_PARTIAL_I:mode>"
120 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, w")
122 [(match_operand:<SVE_FULL_SDI:VPRED> 4 "general_operand" "UplDnm, UplDnm")
123 (ANY_EXTEND:SVE_FULL_SDI
124 (unspec:SVE_PARTIAL_I
125 [(match_operand:<SVE_FULL_SDI:VPRED> 1 "register_operand" "Upl, Upl")
126 (match_operand:DI 2 "aarch64_reg_or_zero" "Z, r")
127 (match_operand:<SVE_FULL_SDI:V_INT_EQUIV> 3 "register_operand" "w, w")
129 UNSPEC_LDNT1_GATHER))]
132 && (~<SVE_FULL_SDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
134 ldnt1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_SDI:Vetype>, %1/z, [%3.<SVE_FULL_SDI:Vetype>]
135 ldnt1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_SDI:Vetype>, %1/z, [%3.<SVE_FULL_SDI:Vetype>, %2]"
136 "&& !CONSTANT_P (operands[4])"
138 operands[4] = CONSTM1_RTX (<SVE_FULL_SDI:VPRED>mode);
142 ;; -------------------------------------------------------------------------
143 ;; ---- Non-temporal scatter stores
144 ;; -------------------------------------------------------------------------
145 ;; Includes scatter forms of:
150 ;; -------------------------------------------------------------------------
152 ;; Non-truncating stores.
153 (define_insn "@aarch64_scatter_stnt<mode>"
154 [(set (mem:BLK (scratch))
156 [(match_operand:<VPRED> 0 "register_operand" "Upl, Upl")
157 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, r")
158 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w")
159 (match_operand:SVE_FULL_SD 3 "register_operand" "w, w")]
161 UNSPEC_STNT1_SCATTER))]
164 stnt1<Vesize>\t%3.<Vetype>, %0, [%2.<Vetype>]
165 stnt1<Vesize>\t%3.<Vetype>, %0, [%2.<Vetype>, %1]"
168 ;; Truncating stores.
169 (define_insn "@aarch64_scatter_stnt_<SVE_FULL_SDI:mode><SVE_PARTIAL_I:mode>"
170 [(set (mem:BLK (scratch))
172 [(match_operand:<SVE_FULL_SDI:VPRED> 0 "register_operand" "Upl, Upl")
173 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, r")
174 (match_operand:<SVE_FULL_SDI:V_INT_EQUIV> 2 "register_operand" "w, w")
175 (truncate:SVE_PARTIAL_I
176 (match_operand:SVE_FULL_SDI 3 "register_operand" "w, w"))]
177 UNSPEC_STNT1_SCATTER))]
179 && (~<SVE_FULL_SDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0"
181 stnt1<SVE_PARTIAL_I:Vesize>\t%3.<SVE_FULL_SDI:Vetype>, %0, [%2.<SVE_FULL_SDI:Vetype>]
182 stnt1<SVE_PARTIAL_I:Vesize>\t%3.<SVE_FULL_SDI:Vetype>, %0, [%2.<SVE_FULL_SDI:Vetype>, %1]"
185 ;; =========================================================================
186 ;; == Uniform binary arithmnetic
187 ;; =========================================================================
189 ;; -------------------------------------------------------------------------
190 ;; ---- [INT] Multiplication
191 ;; -------------------------------------------------------------------------
192 ;; Includes the lane forms of:
194 ;; -------------------------------------------------------------------------
196 (define_insn "@aarch64_mul_lane_<mode>"
197 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
199 (unspec:SVE_FULL_HSDI
200 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "<sve_lane_con>")
201 (match_operand:SI 3 "const_int_operand")]
202 UNSPEC_SVE_LANE_SELECT)
203 (match_operand:SVE_FULL_HSDI 1 "register_operand" "w")))]
205 "mul\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]"
208 ;; -------------------------------------------------------------------------
209 ;; ---- [INT] Scaled high-part multiplication
210 ;; -------------------------------------------------------------------------
211 ;; The patterns in this section are synthetic.
212 ;; -------------------------------------------------------------------------
214 ;; Unpredicated integer multiply-high-with-(round-and-)scale.
215 (define_expand "<su>mulh<r>s<mode>3"
216 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand")
217 (unspec:SVE_FULL_BHSI
219 (unspec:SVE_FULL_BHSI
220 [(match_operand:SVE_FULL_BHSI 1 "register_operand")
221 (match_operand:SVE_FULL_BHSI 2 "register_operand")]
226 operands[3] = aarch64_ptrue_reg (<VPRED>mode);
228 rtx prod_b = gen_reg_rtx (<VWIDE>mode);
229 rtx prod_t = gen_reg_rtx (<VWIDE>mode);
230 emit_insn (gen_aarch64_sve_<su>mullb<Vwide> (prod_b, operands[1],
232 emit_insn (gen_aarch64_sve_<su>mullt<Vwide> (prod_t, operands[1],
235 rtx shift = GEN_INT (GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1);
236 emit_insn (gen_aarch64_sve_<r>shrnb<Vwide> (operands[0], prod_b, shift));
237 emit_insn (gen_aarch64_sve_<r>shrnt<Vwide> (operands[0], operands[0],
244 ;; -------------------------------------------------------------------------
245 ;; ---- [INT] General binary arithmetic that maps to unspecs
246 ;; -------------------------------------------------------------------------
266 ;; -------------------------------------------------------------------------
268 ;; Integer average (floor).
269 (define_expand "<u>avg<mode>3_floor"
270 [(set (match_operand:SVE_FULL_I 0 "register_operand")
274 [(match_operand:SVE_FULL_I 1 "register_operand")
275 (match_operand:SVE_FULL_I 2 "register_operand")]
280 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
284 ;; Integer average (rounding).
285 (define_expand "<u>avg<mode>3_ceil"
286 [(set (match_operand:SVE_FULL_I 0 "register_operand")
290 [(match_operand:SVE_FULL_I 1 "register_operand")
291 (match_operand:SVE_FULL_I 2 "register_operand")]
296 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
300 ;; The immediate form of SQADD acts as an immediate form of SUQADD
301 ;; over its full range. In contrast to the ss_plus pattern, we do
302 ;; not need to treat byte immediates specially. E.g.:
304 ;; SQADD Z0.B, Z0.B, #128
309 ;; SUQADD Z0.B, P0/M, Z0.B, Z1.B
311 ;; even though it's not equivalent to:
314 ;; SQADD Z0.B, P0/M, Z0.B, Z1.B // Saturating subtraction of 128
315 (define_insn "@aarch64_sve_suqadd<mode>_const"
316 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
318 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w")
319 (match_operand:SVE_FULL_I 2 "aarch64_sve_arith_immediate")]
323 sqadd\t%0.<Vetype>, %0.<Vetype>, #%D2
324 movprfx\t%0, %1\;sqadd\t%0.<Vetype>, %0.<Vetype>, #%D2"
325 [(set_attr "movprfx" "*,yes")]
328 ;; General predicated binary arithmetic. All operations handled here
329 ;; are commutative or have a reversed form.
330 (define_insn "@aarch64_pred_<sve_int_op><mode>"
331 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w")
333 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
335 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w, w")
336 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w")]
337 SVE2_COND_INT_BINARY_REV)]
341 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
342 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
343 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
344 [(set_attr "movprfx" "*,*,yes")]
347 ;; Predicated binary arithmetic with merging.
348 (define_expand "@cond_<sve_int_op><mode>"
349 [(set (match_operand:SVE_FULL_I 0 "register_operand")
351 [(match_operand:<VPRED> 1 "register_operand")
355 [(match_operand:SVE_FULL_I 2 "register_operand")
356 (match_operand:SVE_FULL_I 3 "register_operand")]
357 SVE2_COND_INT_BINARY)]
359 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
363 operands[5] = CONSTM1_RTX (<MODE>mode);
367 ;; Predicated binary arithmetic, merging with the first input.
368 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_2"
369 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
371 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
375 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
376 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
377 SVE2_COND_INT_BINARY)]
383 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
384 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
385 "&& !CONSTANT_P (operands[4])"
387 operands[4] = CONSTM1_RTX (<VPRED>mode);
389 [(set_attr "movprfx" "*,yes")]
392 ;; Predicated binary arithmetic, merging with the second input.
393 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_3"
394 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
396 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
400 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w")
401 (match_operand:SVE_FULL_I 3 "register_operand" "0, w")]
402 SVE2_COND_INT_BINARY_REV)]
408 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
409 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
410 "&& !CONSTANT_P (operands[4])"
412 operands[4] = CONSTM1_RTX (<VPRED>mode);
414 [(set_attr "movprfx" "*,yes")]
417 ;; Predicated binary operations, merging with an independent value.
418 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_any"
419 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, ?&w")
421 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
425 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w, w, w, w")
426 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w, w, w")]
427 SVE2_COND_INT_BINARY_REV)]
429 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")]
432 && !rtx_equal_p (operands[2], operands[4])
433 && !rtx_equal_p (operands[3], operands[4])"
435 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
436 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
437 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
438 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
443 && register_operand (operands[4], <MODE>mode)
444 && !rtx_equal_p (operands[0], operands[4]))
446 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
447 operands[4], operands[1]));
448 operands[4] = operands[2] = operands[0];
450 else if (!CONSTANT_P (operands[5]))
451 operands[5] = CONSTM1_RTX (<VPRED>mode);
455 [(set_attr "movprfx" "yes")]
458 ;; Predicated binary operations with no reverse form, merging with zero.
459 ;; At present we don't generate these patterns via a cond_* optab,
460 ;; so there's no correctness requirement to handle merging with an
461 ;; independent value.
462 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_z"
463 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w")
465 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
469 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
470 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
471 SVE2_COND_INT_BINARY_NOREV)]
473 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_zero")]
477 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
478 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
479 "&& !CONSTANT_P (operands[5])"
481 operands[5] = CONSTM1_RTX (<VPRED>mode);
483 [(set_attr "movprfx" "yes")]
486 ;; -------------------------------------------------------------------------
487 ;; ---- [INT] Saturating binary arithmetic
488 ;; -------------------------------------------------------------------------
492 ;; -------------------------------------------------------------------------
494 (define_insn "@aarch64_sve_<sve_int_op><mode>"
495 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
497 [(match_operand:SVE_FULL_I 1 "register_operand" "w")
498 (match_operand:SVE_FULL_I 2 "register_operand" "w")]
501 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
504 (define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>"
505 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
506 (unspec:SVE_FULL_HSDI
507 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")
508 (unspec:SVE_FULL_HSDI
509 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "<sve_lane_con>")
510 (match_operand:SI 3 "const_int_operand")]
511 UNSPEC_SVE_LANE_SELECT)]
512 SVE2_INT_BINARY_LANE))]
514 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]"
517 ;; -------------------------------------------------------------------------
518 ;; ---- [INT] Saturating left shifts
519 ;; -------------------------------------------------------------------------
525 ;; -------------------------------------------------------------------------
527 ;; Predicated left shifts.
528 (define_insn "@aarch64_pred_<sve_int_op><mode>"
529 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, w, ?&w, ?&w")
531 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl")
533 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w, w")
534 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, 0, D<lr>, w")]
535 SVE2_COND_INT_SHIFT)]
539 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
540 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
541 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
542 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
543 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
544 [(set_attr "movprfx" "*,*,*,yes,yes")]
547 ;; Predicated left shifts with merging.
548 (define_expand "@cond_<sve_int_op><mode>"
549 [(set (match_operand:SVE_FULL_I 0 "register_operand")
551 [(match_operand:<VPRED> 1 "register_operand")
555 [(match_operand:SVE_FULL_I 2 "register_operand")
556 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand")]
557 SVE2_COND_INT_SHIFT)]
559 (match_operand:SVE_FULL_I 4 "register_operand")]
563 operands[5] = CONSTM1_RTX (<VPRED>mode);
567 ;; Predicated left shifts, merging with the first input.
568 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_2"
569 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w, ?&w")
571 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
575 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w")
576 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, D<lr>, w")]
577 SVE2_COND_INT_SHIFT)]
583 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
584 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
585 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
586 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
587 "&& !CONSTANT_P (operands[4])"
589 operands[4] = CONSTM1_RTX (<VPRED>mode);
591 [(set_attr "movprfx" "*,*,yes,yes")]
594 ;; Predicated left shifts, merging with the second input.
595 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_3"
596 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
598 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
602 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w")
603 (match_operand:SVE_FULL_I 3 "register_operand" "0, w")]
604 SVE2_COND_INT_SHIFT)]
610 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
611 movprfx\t%0, %3\;<sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
612 "&& !CONSTANT_P (operands[4])"
614 operands[4] = CONSTM1_RTX (<VPRED>mode);
616 [(set_attr "movprfx" "*,yes")]
619 ;; Predicated left shifts, merging with an independent value.
620 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_any"
621 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, &w, &w, &w, ?&w, ?&w")
623 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl, Upl, Upl, Upl")
627 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w, w, w, w, w, w")
628 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, 0, D<lr>, w, D<lr>, w, D<lr>, w")]
629 SVE2_COND_INT_SHIFT)]
631 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, Dz, 0, 0, w, w")]
634 && !rtx_equal_p (operands[2], operands[4])
635 && (CONSTANT_P (operands[4]) || !rtx_equal_p (operands[3], operands[4]))"
637 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
638 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
639 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
640 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
641 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
642 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
643 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
649 && register_operand (operands[4], <MODE>mode)
650 && !rtx_equal_p (operands[0], operands[4]))
652 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
653 operands[4], operands[1]));
654 operands[4] = operands[2] = operands[0];
656 else if (!CONSTANT_P (operands[5]))
657 operands[5] = CONSTM1_RTX (<VPRED>mode);
661 [(set_attr "movprfx" "yes")]
664 ;; =========================================================================
665 ;; == Uniform ternary arithmnetic
666 ;; =========================================================================
668 ;; -------------------------------------------------------------------------
669 ;; ---- [INT] General ternary arithmetic that maps to unspecs
670 ;; -------------------------------------------------------------------------
680 ;; -------------------------------------------------------------------------
682 (define_insn "@aarch64_sve_<sve_int_op><mode>"
683 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
685 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w")
686 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")
687 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")]
691 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>
692 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
695 (define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>"
696 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
697 (unspec:SVE_FULL_HSDI
698 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w")
699 (unspec:SVE_FULL_HSDI
700 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
701 (match_operand:SI 4 "const_int_operand")]
702 UNSPEC_SVE_LANE_SELECT)
703 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")]
704 SVE2_INT_TERNARY_LANE))]
707 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]
708 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]"
711 ;; -------------------------------------------------------------------------
712 ;; ---- [INT] Multiply-and-accumulate operations
713 ;; -------------------------------------------------------------------------
714 ;; Includes the lane forms of:
717 ;; -------------------------------------------------------------------------
719 (define_insn "@aarch64_sve_add_mul_lane_<mode>"
720 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
723 (unspec:SVE_FULL_HSDI
724 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
725 (match_operand:SI 4 "const_int_operand")]
726 UNSPEC_SVE_LANE_SELECT)
727 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w"))
728 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))]
731 mla\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]
732 movprfx\t%0, %1\;mla\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]"
733 [(set_attr "movprfx" "*,yes")]
736 (define_insn "@aarch64_sve_sub_mul_lane_<mode>"
737 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
739 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")
741 (unspec:SVE_FULL_HSDI
742 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
743 (match_operand:SI 4 "const_int_operand")]
744 UNSPEC_SVE_LANE_SELECT)
745 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w"))))]
748 mls\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]
749 movprfx\t%0, %1\;mls\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]"
750 [(set_attr "movprfx" "*,yes")]
753 ;; -------------------------------------------------------------------------
754 ;; ---- [INT] Binary logic operations with rotation
755 ;; -------------------------------------------------------------------------
758 ;; -------------------------------------------------------------------------
760 (define_insn "@aarch64_sve2_xar<mode>"
761 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
764 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w")
765 (match_operand:SVE_FULL_I 2 "register_operand" "w, w"))
766 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")))]
769 xar\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3
770 movprfx\t%0, %1\;xar\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3"
771 [(set_attr "movprfx" "*,yes")]
774 ;; -------------------------------------------------------------------------
775 ;; ---- [INT] Ternary logic operations
776 ;; -------------------------------------------------------------------------
784 ;; -------------------------------------------------------------------------
786 ;; Unpredicated exclusive OR of AND.
787 (define_insn "@aarch64_sve2_bcax<mode>"
788 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
791 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
792 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
793 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))]
796 bcax\t%0.d, %0.d, %2.d, %3.d
797 movprfx\t%0, %1\;bcax\t%0.d, %0.d, %2.d, %3.d"
798 [(set_attr "movprfx" "*,yes")]
801 ;; Unpredicated 3-way exclusive OR.
802 (define_insn "@aarch64_sve2_eor3<mode>"
803 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, w, ?&w")
806 (match_operand:SVE_FULL_I 1 "register_operand" "0, w, w, w")
807 (match_operand:SVE_FULL_I 2 "register_operand" "w, 0, w, w"))
808 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, 0, w")))]
811 eor3\t%0.d, %0.d, %2.d, %3.d
812 eor3\t%0.d, %0.d, %1.d, %3.d
813 eor3\t%0.d, %0.d, %1.d, %2.d
814 movprfx\t%0, %1\;eor3\t%0.d, %0.d, %2.d, %3.d"
815 [(set_attr "movprfx" "*,*,*,yes")]
818 ;; Use NBSL for vector NOR.
819 (define_insn_and_rewrite "*aarch64_sve2_nor<mode>"
820 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
825 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w"))
827 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")))]
831 nbsl\t%0.d, %0.d, %2.d, %0.d
832 movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %0.d"
833 "&& !CONSTANT_P (operands[3])"
835 operands[3] = CONSTM1_RTX (<VPRED>mode);
837 [(set_attr "movprfx" "*,yes")]
840 ;; Use NBSL for vector NAND.
841 (define_insn_and_rewrite "*aarch64_sve2_nand<mode>"
842 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
847 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w"))
849 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")))]
853 nbsl\t%0.d, %0.d, %2.d, %2.d
854 movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %2.d"
855 "&& !CONSTANT_P (operands[3])"
857 operands[3] = CONSTM1_RTX (<VPRED>mode);
859 [(set_attr "movprfx" "*,yes")]
862 ;; Unpredicated bitwise select.
863 ;; (op3 ? bsl_mov : bsl_dup) == (((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup)
864 (define_expand "@aarch64_sve2_bsl<mode>"
865 [(set (match_operand:SVE_FULL_I 0 "register_operand")
869 (match_operand:SVE_FULL_I 1 "register_operand")
870 (match_operand:SVE_FULL_I 2 "register_operand"))
871 (match_operand:SVE_FULL_I 3 "register_operand"))
876 (define_insn "*aarch64_sve2_bsl<mode>"
877 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
881 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w")
882 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w"))
883 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
884 (match_dup BSL_DUP)))]
887 bsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d
888 movprfx\t%0, %<bsl_mov>\;bsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d"
889 [(set_attr "movprfx" "*,yes")]
892 ;; Unpredicated bitwise inverted select.
893 ;; (~(op3 ? bsl_mov : bsl_dup)) == (~(((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup))
894 (define_expand "@aarch64_sve2_nbsl<mode>"
895 [(set (match_operand:SVE_FULL_I 0 "register_operand")
902 (match_operand:SVE_FULL_I 1 "register_operand")
903 (match_operand:SVE_FULL_I 2 "register_operand"))
904 (match_operand:SVE_FULL_I 3 "register_operand"))
909 operands[4] = CONSTM1_RTX (<VPRED>mode);
913 (define_insn_and_rewrite "*aarch64_sve2_nbsl<mode>"
914 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
921 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w")
922 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w"))
923 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
924 (match_dup BSL_DUP)))]
928 nbsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d
929 movprfx\t%0, %<bsl_mov>\;nbsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d"
930 "&& !CONSTANT_P (operands[4])"
932 operands[4] = CONSTM1_RTX (<VPRED>mode);
934 [(set_attr "movprfx" "*,yes")]
937 ;; Unpredicated bitwise select with inverted first operand.
938 ;; (op3 ? ~bsl_mov : bsl_dup) == ((~(bsl_mov ^ bsl_dup) & op3) ^ bsl_dup)
939 (define_expand "@aarch64_sve2_bsl1n<mode>"
940 [(set (match_operand:SVE_FULL_I 0 "register_operand")
947 (match_operand:SVE_FULL_I 1 "register_operand")
948 (match_operand:SVE_FULL_I 2 "register_operand")))]
950 (match_operand:SVE_FULL_I 3 "register_operand"))
954 operands[4] = CONSTM1_RTX (<VPRED>mode);
958 (define_insn_and_rewrite "*aarch64_sve2_bsl1n<mode>"
959 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
966 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w")
967 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")))]
969 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
970 (match_dup BSL_DUP)))]
973 bsl1n\t%0.d, %0.d, %<bsl_dup>.d, %3.d
974 movprfx\t%0, %<bsl_mov>\;bsl1n\t%0.d, %0.d, %<bsl_dup>.d, %3.d"
975 "&& !CONSTANT_P (operands[4])"
977 operands[4] = CONSTM1_RTX (<VPRED>mode);
979 [(set_attr "movprfx" "*,yes")]
982 ;; Unpredicated bitwise select with inverted second operand.
983 ;; (bsl_dup ? bsl_mov : ~op3) == ((bsl_dup & bsl_mov) | (~op3 & ~bsl_dup))
984 (define_expand "@aarch64_sve2_bsl2n<mode>"
985 [(set (match_operand:SVE_FULL_I 0 "register_operand")
988 (match_operand:SVE_FULL_I 1 "register_operand")
989 (match_operand:SVE_FULL_I 3 "register_operand"))
994 (match_operand:SVE_FULL_I 2 "register_operand"))
1000 operands[4] = CONSTM1_RTX (<VPRED>mode);
1004 (define_insn_and_rewrite "*aarch64_sve2_bsl2n<mode>"
1005 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1008 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w")
1009 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w"))
1014 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
1016 (match_dup BSL_DUP)))]
1020 bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d
1021 movprfx\t%0, %<bsl_mov>\;bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d"
1022 "&& !CONSTANT_P (operands[4])"
1024 operands[4] = CONSTM1_RTX (<VPRED>mode);
1026 [(set_attr "movprfx" "*,yes")]
1029 ;; Unpredicated bitwise select with inverted second operand, alternative form.
1030 ;; (bsl_dup ? bsl_mov : ~op3) == ((bsl_dup & bsl_mov) | (~bsl_dup & ~op3))
1031 (define_insn_and_rewrite "*aarch64_sve2_bsl2n<mode>"
1032 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1035 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w")
1036 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w"))
1041 (match_dup BSL_DUP))
1043 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")))]
1047 bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d
1048 movprfx\t%0, %<bsl_mov>\;bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d"
1049 "&& !CONSTANT_P (operands[4])"
1051 operands[4] = CONSTM1_RTX (<VPRED>mode);
1053 [(set_attr "movprfx" "*,yes")]
1056 ;; -------------------------------------------------------------------------
1057 ;; ---- [INT] Shift-and-accumulate operations
1058 ;; -------------------------------------------------------------------------
1064 ;; -------------------------------------------------------------------------
1066 ;; Provide the natural unpredicated interface for SSRA and USRA.
1067 (define_expand "@aarch64_sve_add_<sve_int_op><mode>"
1068 [(set (match_operand:SVE_FULL_I 0 "register_operand")
1073 (match_operand:SVE_FULL_I 2 "register_operand")
1074 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm"))]
1076 (match_operand:SVE_FULL_I 1 "register_operand")))]
1079 operands[4] = CONSTM1_RTX (<VPRED>mode);
1083 ;; Pattern-match SSRA and USRA as a predicated operation whose predicate
1085 (define_insn_and_rewrite "*aarch64_sve2_sra<mode>"
1086 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1091 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
1092 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm"))]
1094 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))]
1097 <sra_op>sra\t%0.<Vetype>, %2.<Vetype>, #%3
1098 movprfx\t%0, %1\;<sra_op>sra\t%0.<Vetype>, %2.<Vetype>, #%3"
1099 "&& !CONSTANT_P (operands[4])"
1101 operands[4] = CONSTM1_RTX (<VPRED>mode);
1103 [(set_attr "movprfx" "*,yes")]
1107 (define_insn "@aarch64_sve_add_<sve_int_op><mode>"
1108 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1111 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w")
1112 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")]
1114 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))]
1117 <sur>sra\t%0.<Vetype>, %2.<Vetype>, #%3
1118 movprfx\t%0, %1\;<sur>sra\t%0.<Vetype>, %2.<Vetype>, #%3"
1119 [(set_attr "movprfx" "*,yes")]
1122 ;; -------------------------------------------------------------------------
1123 ;; ---- [INT] Shift-and-insert operations
1124 ;; -------------------------------------------------------------------------
1128 ;; -------------------------------------------------------------------------
1130 ;; These instructions do not take MOVPRFX.
1131 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1132 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
1134 [(match_operand:SVE_FULL_I 1 "register_operand" "0")
1135 (match_operand:SVE_FULL_I 2 "register_operand" "w")
1136 (match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm")]
1137 SVE2_INT_SHIFT_INSERT))]
1139 "<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, #%3"
1142 ;; -------------------------------------------------------------------------
1143 ;; ---- [INT] Sum of absolute differences
1144 ;; -------------------------------------------------------------------------
1148 ;; -------------------------------------------------------------------------
1150 ;; Provide the natural unpredicated interface for SABA and UABA.
1151 (define_expand "@aarch64_sve2_<su>aba<mode>"
1152 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1158 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
1159 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))]
1163 (<max_opp>:SVE_FULL_I
1167 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))]
1170 operands[4] = CONSTM1_RTX (<VPRED>mode);
1174 ;; Pattern-match SABA and UABA as an absolute-difference-and-accumulate
1175 ;; operation whose predicates aren't needed.
1176 (define_insn "*aarch64_sve2_<su>aba<mode>"
1177 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1183 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
1184 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))]
1188 (<max_opp>:SVE_FULL_I
1192 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))]
1195 <su>aba\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>
1196 movprfx\t%0, %1\;<su>aba\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
1197 [(set_attr "movprfx" "*,yes")]
1200 ;; =========================================================================
1201 ;; == Extending arithmetic
1202 ;; =========================================================================
1204 ;; -------------------------------------------------------------------------
1205 ;; ---- [INT] Wide binary arithmetic
1206 ;; -------------------------------------------------------------------------
1216 ;; -------------------------------------------------------------------------
1218 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1219 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
1220 (unspec:SVE_FULL_HSDI
1221 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")
1222 (match_operand:<VNARROW> 2 "register_operand" "w")]
1223 SVE2_INT_BINARY_WIDE))]
1225 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Ventype>"
1228 ;; -------------------------------------------------------------------------
1229 ;; ---- [INT] Long binary arithmetic
1230 ;; -------------------------------------------------------------------------
1253 ;; -------------------------------------------------------------------------
1255 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1256 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
1257 (unspec:SVE_FULL_HSDI
1258 [(match_operand:<VNARROW> 1 "register_operand" "w")
1259 (match_operand:<VNARROW> 2 "register_operand" "w")]
1260 SVE2_INT_BINARY_LONG))]
1262 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>"
1265 (define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>"
1266 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w")
1267 (unspec:SVE_FULL_SDI
1268 [(match_operand:<VNARROW> 1 "register_operand" "w")
1270 [(match_operand:<VNARROW> 2 "register_operand" "<sve_lane_con>")
1271 (match_operand:SI 3 "const_int_operand")]
1272 UNSPEC_SVE_LANE_SELECT)]
1273 SVE2_INT_BINARY_LONG_LANE))]
1275 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3]"
1278 ;; -------------------------------------------------------------------------
1279 ;; ---- [INT] Long left shifts
1280 ;; -------------------------------------------------------------------------
1286 ;; -------------------------------------------------------------------------
1288 ;; The immediate range is enforced before generating the instruction.
1289 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1290 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
1291 (unspec:SVE_FULL_HSDI
1292 [(match_operand:<VNARROW> 1 "register_operand" "w")
1293 (match_operand:DI 2 "const_int_operand")]
1294 SVE2_INT_SHIFT_IMM_LONG))]
1296 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, #%2"
1299 ;; -------------------------------------------------------------------------
1300 ;; ---- [INT] Long binary arithmetic with accumulation
1301 ;; -------------------------------------------------------------------------
1321 ;; -------------------------------------------------------------------------
1323 ;; Non-saturating MLA operations.
1324 (define_insn "@aarch64_sve_add_<sve_int_op><mode>"
1325 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
1327 (unspec:SVE_FULL_HSDI
1328 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1329 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1330 SVE2_INT_ADD_BINARY_LONG)
1331 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))]
1334 <sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>
1335 movprfx\t%0, %1\;<sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>"
1336 [(set_attr "movprfx" "*,yes")]
1339 ;; Non-saturating MLA operations with lane select.
1340 (define_insn "@aarch64_sve_add_<sve_int_op>_lane_<mode>"
1341 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1343 (unspec:SVE_FULL_SDI
1344 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1346 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1347 (match_operand:SI 4 "const_int_operand")]
1348 UNSPEC_SVE_LANE_SELECT)]
1349 SVE2_INT_ADD_BINARY_LONG_LANE)
1350 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")))]
1353 <sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]
1354 movprfx\t%0, %1\;<sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]"
1355 [(set_attr "movprfx" "*,yes")]
1358 ;; Saturating MLA operations.
1359 (define_insn "@aarch64_sve_qadd_<sve_int_op><mode>"
1360 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
1361 (ss_plus:SVE_FULL_HSDI
1362 (unspec:SVE_FULL_HSDI
1363 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1364 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1365 SVE2_INT_QADD_BINARY_LONG)
1366 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))]
1369 <sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>
1370 movprfx\t%0, %1\;<sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>"
1371 [(set_attr "movprfx" "*,yes")]
1374 ;; Saturating MLA operations with lane select.
1375 (define_insn "@aarch64_sve_qadd_<sve_int_op>_lane_<mode>"
1376 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1377 (ss_plus:SVE_FULL_SDI
1378 (unspec:SVE_FULL_SDI
1379 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1381 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1382 (match_operand:SI 4 "const_int_operand")]
1383 UNSPEC_SVE_LANE_SELECT)]
1384 SVE2_INT_QADD_BINARY_LONG_LANE)
1385 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")))]
1388 <sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]
1389 movprfx\t%0, %1\;<sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]"
1390 [(set_attr "movprfx" "*,yes")]
1393 ;; Non-saturating MLS operations.
1394 (define_insn "@aarch64_sve_sub_<sve_int_op><mode>"
1395 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
1396 (minus:SVE_FULL_HSDI
1397 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")
1398 (unspec:SVE_FULL_HSDI
1399 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1400 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1401 SVE2_INT_SUB_BINARY_LONG)))]
1404 <sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>
1405 movprfx\t%0, %1\;<sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>"
1406 [(set_attr "movprfx" "*,yes")]
1409 ;; Non-saturating MLS operations with lane select.
1410 (define_insn "@aarch64_sve_sub_<sve_int_op>_lane_<mode>"
1411 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1413 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")
1414 (unspec:SVE_FULL_SDI
1415 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1417 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1418 (match_operand:SI 4 "const_int_operand")]
1419 UNSPEC_SVE_LANE_SELECT)]
1420 SVE2_INT_SUB_BINARY_LONG_LANE)))]
1423 <sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]
1424 movprfx\t%0, %1\;<sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]"
1425 [(set_attr "movprfx" "*,yes")]
1428 ;; Saturating MLS operations.
1429 (define_insn "@aarch64_sve_qsub_<sve_int_op><mode>"
1430 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
1431 (ss_minus:SVE_FULL_HSDI
1432 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")
1433 (unspec:SVE_FULL_HSDI
1434 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1435 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1436 SVE2_INT_QSUB_BINARY_LONG)))]
1439 <sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>
1440 movprfx\t%0, %1\;<sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>"
1441 [(set_attr "movprfx" "*,yes")]
1444 ;; Saturating MLS operations with lane select.
1445 (define_insn "@aarch64_sve_qsub_<sve_int_op>_lane_<mode>"
1446 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1447 (ss_minus:SVE_FULL_SDI
1448 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")
1449 (unspec:SVE_FULL_SDI
1450 [(match_operand:<VNARROW> 2 "register_operand" "w, w")
1452 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1453 (match_operand:SI 4 "const_int_operand")]
1454 UNSPEC_SVE_LANE_SELECT)]
1455 SVE2_INT_QSUB_BINARY_LONG_LANE)))]
1458 <sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]
1459 movprfx\t%0, %1\;<sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]"
1460 [(set_attr "movprfx" "*,yes")]
1462 ;; -------------------------------------------------------------------------
1463 ;; ---- [FP] Long multiplication with accumulation
1464 ;; -------------------------------------------------------------------------
1470 ;; -------------------------------------------------------------------------
1472 (define_insn "@aarch64_sve_<sve_fp_op><mode>"
1473 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w, ?&w")
1475 [(match_operand:<VNARROW> 1 "register_operand" "w, w")
1476 (match_operand:<VNARROW> 2 "register_operand" "w, w")
1477 (match_operand:VNx4SF_ONLY 3 "register_operand" "0, w")]
1478 SVE2_FP_TERNARY_LONG))]
1481 <sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>
1482 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>"
1483 [(set_attr "movprfx" "*,yes")]
1486 (define_insn "@aarch64_<sve_fp_op>_lane_<mode>"
1487 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w, ?&w")
1489 [(match_operand:<VNARROW> 1 "register_operand" "w, w")
1491 [(match_operand:<VNARROW> 2 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1492 (match_operand:SI 3 "const_int_operand")]
1493 UNSPEC_SVE_LANE_SELECT)
1494 (match_operand:VNx4SF_ONLY 4 "register_operand" "0, w")]
1495 SVE2_FP_TERNARY_LONG_LANE))]
1498 <sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3]
1499 movprfx\t%0, %4\;<sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3]"
1500 [(set_attr "movprfx" "*,yes")]
1503 ;; =========================================================================
1504 ;; == Narrowing arithnetic
1505 ;; =========================================================================
1507 ;; -------------------------------------------------------------------------
1508 ;; ---- [INT] Narrowing unary arithmetic
1509 ;; -------------------------------------------------------------------------
1517 ;; -------------------------------------------------------------------------
1519 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1520 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1522 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")]
1523 SVE2_INT_UNARY_NARROWB))]
1525 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>"
1528 ;; These instructions do not take MOVPRFX.
1529 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1530 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1532 [(match_operand:<VNARROW> 1 "register_operand" "0")
1533 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")]
1534 SVE2_INT_UNARY_NARROWT))]
1536 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>"
1539 ;; -------------------------------------------------------------------------
1540 ;; ---- [INT] Narrowing binary arithmetic
1541 ;; -------------------------------------------------------------------------
1551 ;; -------------------------------------------------------------------------
1553 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1554 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1556 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")
1557 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")]
1558 SVE2_INT_BINARY_NARROWB))]
1560 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>, %2.<Vetype>"
1563 ;; These instructions do not take MOVPRFX.
1564 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1565 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1567 [(match_operand:<VNARROW> 1 "register_operand" "0")
1568 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")
1569 (match_operand:SVE_FULL_HSDI 3 "register_operand" "w")]
1570 SVE2_INT_BINARY_NARROWT))]
1572 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>, %3.<Vetype>"
1575 ;; -------------------------------------------------------------------------
1576 ;; ---- [INT] Narrowing right shifts
1577 ;; -------------------------------------------------------------------------
1595 ;; -------------------------------------------------------------------------
1597 ;; The immediate range is enforced before generating the instruction.
1598 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1599 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1601 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")
1602 (match_operand:DI 2 "const_int_operand")]
1603 SVE2_INT_SHIFT_IMM_NARROWB))]
1605 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>, #%2"
1608 ;; The immediate range is enforced before generating the instruction.
1609 ;; These instructions do not take MOVPRFX.
1610 (define_insn "@aarch64_sve_<sve_int_op><mode>"
1611 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1613 [(match_operand:<VNARROW> 1 "register_operand" "0")
1614 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")
1615 (match_operand:DI 3 "const_int_operand")]
1616 SVE2_INT_SHIFT_IMM_NARROWT))]
1618 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>, #%3"
1621 ;; =========================================================================
1622 ;; == Pairwise arithmetic
1623 ;; =========================================================================
1625 ;; -------------------------------------------------------------------------
1626 ;; ---- [INT] Pairwise arithmetic
1627 ;; -------------------------------------------------------------------------
1634 ;; -------------------------------------------------------------------------
1636 (define_insn "@aarch64_pred_<sve_int_op><mode>"
1637 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1639 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1640 (match_operand:SVE_FULL_I 2 "register_operand" "0, w")
1641 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
1642 SVE2_INT_BINARY_PAIR))]
1645 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1646 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1647 [(set_attr "movprfx" "*,yes")]
1650 ;; -------------------------------------------------------------------------
1651 ;; ---- [FP] Pairwise arithmetic
1652 ;; -------------------------------------------------------------------------
1659 ;; -------------------------------------------------------------------------
1661 (define_insn "@aarch64_pred_<sve_fp_op><mode>"
1662 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w")
1664 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1665 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")
1666 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")]
1667 SVE2_FP_BINARY_PAIR))]
1670 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1671 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1672 [(set_attr "movprfx" "*,yes")]
1675 ;; -------------------------------------------------------------------------
1676 ;; ---- [INT] Pairwise arithmetic with accumulation
1677 ;; -------------------------------------------------------------------------
1681 ;; -------------------------------------------------------------------------
1683 ;; Predicated pairwise absolute difference and accumulate with merging.
1684 (define_expand "@cond_<sve_int_op><mode>"
1685 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
1686 (unspec:SVE_FULL_HSDI
1687 [(match_operand:<VPRED> 1 "register_operand")
1688 (unspec:SVE_FULL_HSDI
1690 (match_operand:SVE_FULL_HSDI 2 "register_operand")
1691 (match_operand:<VNARROW> 3 "register_operand")]
1692 SVE2_INT_BINARY_PAIR_LONG)
1693 (match_operand:SVE_FULL_HSDI 4 "aarch64_simd_reg_or_zero")]
1697 /* Only target code is aware of these operations, so we don't need
1698 to handle the fully-general case. */
1699 gcc_assert (rtx_equal_p (operands[2], operands[4])
1700 || CONSTANT_P (operands[4]));
1703 ;; Predicated pairwise absolute difference and accumulate, merging with
1705 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_2"
1706 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w")
1707 (unspec:SVE_FULL_HSDI
1708 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1709 (unspec:SVE_FULL_HSDI
1711 (match_operand:SVE_FULL_HSDI 2 "register_operand" "0, w")
1712 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1713 SVE2_INT_BINARY_PAIR_LONG)
1718 <sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>
1719 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>"
1720 "&& !CONSTANT_P (operands[4])"
1722 operands[4] = CONSTM1_RTX (<VPRED>mode);
1724 [(set_attr "movprfx" "*,yes")]
1727 ;; Predicated pairwise absolute difference and accumulate, merging with zero.
1728 (define_insn_and_rewrite "*cond_<sve_int_op><mode>_z"
1729 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=&w, &w")
1730 (unspec:SVE_FULL_HSDI
1731 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1732 (unspec:SVE_FULL_HSDI
1734 (match_operand:SVE_FULL_HSDI 2 "register_operand" "0, w")
1735 (match_operand:<VNARROW> 3 "register_operand" "w, w")]
1736 SVE2_INT_BINARY_PAIR_LONG)
1737 (match_operand:SVE_FULL_HSDI 4 "aarch64_simd_imm_zero")]
1741 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>
1742 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>"
1743 "&& !CONSTANT_P (operands[5])"
1745 operands[5] = CONSTM1_RTX (<VPRED>mode);
1747 [(set_attr "movprfx" "yes")]
1750 ;; =========================================================================
1751 ;; == Complex arithmetic
1752 ;; =========================================================================
1754 ;; -------------------------------------------------------------------------
1755 ;; ---- [INT] Complex binary operations
1756 ;; -------------------------------------------------------------------------
1760 ;; -------------------------------------------------------------------------
1762 (define_insn "@aarch64_sve_<optab><mode>"
1763 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1765 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w")
1766 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")]
1770 <sve_int_op>\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #<rot>
1771 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #<rot>"
1772 [(set_attr "movprfx" "*,yes")]
1775 ;; -------------------------------------------------------------------------
1776 ;; ---- [INT] Complex ternary operations
1777 ;; -------------------------------------------------------------------------
1781 ;; -------------------------------------------------------------------------
1783 (define_insn "@aarch64_sve_<optab><mode>"
1784 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
1786 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w")
1787 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")
1788 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")]
1792 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>, #<rot>
1793 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>, #<rot>"
1794 [(set_attr "movprfx" "*,yes")]
1797 (define_insn "@aarch64_<optab>_lane_<mode>"
1798 [(set (match_operand:SVE_FULL_HSI 0 "register_operand" "=w, ?&w")
1799 (unspec:SVE_FULL_HSI
1800 [(match_operand:SVE_FULL_HSI 1 "register_operand" "0, w")
1801 (match_operand:SVE_FULL_HSI 2 "register_operand" "w, w")
1802 (unspec:SVE_FULL_HSI
1803 [(match_operand:SVE_FULL_HSI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1804 (match_operand:SI 4 "const_int_operand")]
1805 UNSPEC_SVE_LANE_SELECT)]
1809 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4], #<rot>
1810 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4], #<rot>"
1811 [(set_attr "movprfx" "*,yes")]
1814 ;; -------------------------------------------------------------------------
1815 ;; ---- [INT] Complex dot product
1816 ;; -------------------------------------------------------------------------
1819 ;; -------------------------------------------------------------------------
1821 (define_insn "@aarch64_sve_<optab><mode>"
1822 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1823 (unspec:SVE_FULL_SDI
1824 [(match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")
1825 (match_operand:<VSI2QI> 2 "register_operand" "w, w")
1826 (match_operand:<VSI2QI> 3 "register_operand" "w, w")]
1830 <sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>, #<rot>
1831 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>, #<rot>"
1832 [(set_attr "movprfx" "*,yes")]
1835 (define_insn "@aarch64_<optab>_lane_<mode>"
1836 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w")
1837 (unspec:SVE_FULL_SDI
1838 [(match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")
1839 (match_operand:<VSI2QI> 2 "register_operand" "w, w")
1841 [(match_operand:<VSI2QI> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>")
1842 (match_operand:SI 4 "const_int_operand")]
1843 UNSPEC_SVE_LANE_SELECT)]
1847 <sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>[%4], #<rot>
1848 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>[%4], #<rot>"
1849 [(set_attr "movprfx" "*,yes")]
1852 ;; =========================================================================
1854 ;; =========================================================================
1856 ;; -------------------------------------------------------------------------
1857 ;; ---- [FP<-FP] Widening conversions
1858 ;; -------------------------------------------------------------------------
1861 ;; -------------------------------------------------------------------------
1863 ;; Predicated convert long top.
1864 (define_insn "@aarch64_pred_<sve_fp_op><mode>"
1865 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w")
1866 (unspec:SVE_FULL_SDF
1867 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1868 (match_operand:SI 3 "aarch64_sve_gp_strictness")
1869 (match_operand:<VNARROW> 2 "register_operand" "w")]
1870 SVE2_COND_FP_UNARY_LONG))]
1872 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Ventype>"
1875 ;; Predicated convert long top with merging.
1876 (define_expand "@cond_<sve_fp_op><mode>"
1877 [(set (match_operand:SVE_FULL_SDF 0 "register_operand")
1878 (unspec:SVE_FULL_SDF
1879 [(match_operand:<VPRED> 1 "register_operand")
1880 (unspec:SVE_FULL_SDF
1882 (const_int SVE_STRICT_GP)
1883 (match_operand:<VNARROW> 2 "register_operand")]
1884 SVE2_COND_FP_UNARY_LONG)
1885 (match_operand:SVE_FULL_SDF 3 "register_operand")]
1890 ;; These instructions do not take MOVPRFX.
1891 (define_insn_and_rewrite "*cond_<sve_fp_op><mode>"
1892 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w")
1893 (unspec:SVE_FULL_SDF
1894 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1895 (unspec:SVE_FULL_SDF
1897 (match_operand:SI 5 "aarch64_sve_gp_strictness")
1898 (match_operand:<VNARROW> 2 "register_operand" "w")]
1899 SVE2_COND_FP_UNARY_LONG)
1900 (match_operand:SVE_FULL_SDF 3 "register_operand" "0")]
1902 "TARGET_SVE2 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
1903 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Ventype>"
1904 "&& !rtx_equal_p (operands[1], operands[4])"
1906 operands[4] = copy_rtx (operands[1]);
1910 ;; -------------------------------------------------------------------------
1911 ;; ---- [FP<-FP] Narrowing conversions
1912 ;; -------------------------------------------------------------------------
1917 ;; -------------------------------------------------------------------------
1919 ;; Predicated FCVTNT. This doesn't give a natural aarch64_pred_*/cond_*
1920 ;; pair because the even elements always have to be supplied for active
1921 ;; elements, even if the inactive elements don't matter.
1923 ;; These instructions do not take MOVPRFX.
1924 (define_insn "@aarch64_sve2_cvtnt<mode>"
1925 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1927 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1928 (const_int SVE_STRICT_GP)
1929 (match_operand:<VNARROW> 1 "register_operand" "0")
1930 (match_operand:SVE_FULL_SDF 3 "register_operand" "w")]
1931 UNSPEC_COND_FCVTNT))]
1933 "fcvtnt\t%0.<Ventype>, %2/m, %3.<Vetype>"
1936 ;; Predicated FCVTX (equivalent to what would be FCVTXNB, except that
1937 ;; it supports MOVPRFX).
1938 (define_insn "@aarch64_pred_<sve_fp_op><mode>"
1939 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w")
1941 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl")
1942 (match_operand:SI 3 "aarch64_sve_gp_strictness")
1943 (match_operand:<VWIDE> 2 "register_operand" "w")]
1944 SVE2_COND_FP_UNARY_NARROWB))]
1946 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>"
1949 ;; Predicated FCVTX with merging.
1950 (define_expand "@cond_<sve_fp_op><mode>"
1951 [(set (match_operand:VNx4SF_ONLY 0 "register_operand")
1953 [(match_operand:<VWIDE_PRED> 1 "register_operand")
1956 (const_int SVE_STRICT_GP)
1957 (match_operand:<VWIDE> 2 "register_operand")]
1958 SVE2_COND_FP_UNARY_NARROWB)
1959 (match_operand:VNx4SF_ONLY 3 "aarch64_simd_reg_or_zero")]
1964 (define_insn_and_rewrite "*cond_<sve_fp_op><mode>_any"
1965 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=&w, &w, &w")
1967 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl, Upl, Upl")
1970 (match_operand:SI 5 "aarch64_sve_gp_strictness")
1971 (match_operand:<VWIDE> 2 "register_operand" "w, w, w")]
1972 SVE2_COND_FP_UNARY_NARROWB)
1973 (match_operand:VNx4SF_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
1976 && !rtx_equal_p (operands[2], operands[3])
1977 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
1979 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>
1980 movprfx\t%0.<Vewtype>, %1/z, %2.<Vewtype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>
1981 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>"
1982 "&& !rtx_equal_p (operands[1], operands[4])"
1984 operands[4] = copy_rtx (operands[1]);
1986 [(set_attr "movprfx" "*,yes,yes")]
1989 ;; Predicated FCVTXNT. This doesn't give a natural aarch64_pred_*/cond_*
1990 ;; pair because the even elements always have to be supplied for active
1991 ;; elements, even if the inactive elements don't matter.
1993 ;; These instructions do not take MOVPRFX.
1994 (define_insn "@aarch64_sve2_cvtxnt<mode>"
1995 [(set (match_operand:<VNARROW> 0 "register_operand" "=w")
1997 [(match_operand:<VPRED> 2 "register_operand" "Upl")
1998 (const_int SVE_STRICT_GP)
1999 (match_operand:<VNARROW> 1 "register_operand" "0")
2000 (match_operand:VNx2DF_ONLY 3 "register_operand" "w")]
2001 UNSPEC_COND_FCVTXNT))]
2003 "fcvtxnt\t%0.<Ventype>, %2/m, %3.<Vetype>"
2006 ;; =========================================================================
2007 ;; == Other arithmetic
2008 ;; =========================================================================
2010 ;; -------------------------------------------------------------------------
2011 ;; ---- [INT] Reciprocal approximation
2012 ;; -------------------------------------------------------------------------
2016 ;; -------------------------------------------------------------------------
2018 ;; Predicated integer unary operations.
2019 (define_insn "@aarch64_pred_<sve_int_op><mode>"
2020 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=w")
2022 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2024 [(match_operand:VNx4SI_ONLY 2 "register_operand" "w")]
2028 "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2031 ;; Predicated integer unary operations with merging.
2032 (define_expand "@cond_<sve_int_op><mode>"
2033 [(set (match_operand:VNx4SI_ONLY 0 "register_operand")
2035 [(match_operand:<VPRED> 1 "register_operand")
2039 [(match_operand:VNx4SI_ONLY 2 "register_operand")]
2042 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero")]
2046 operands[4] = CONSTM1_RTX (<MODE>mode);
2050 (define_insn_and_rewrite "*cond_<sve_int_op><mode>"
2051 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=w, ?&w, ?&w")
2053 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2057 [(match_operand:VNx4SI_ONLY 2 "register_operand" "w, w, w")]
2060 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
2064 <sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2065 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2066 movprfx\t%0, %3\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2067 "&& !CONSTANT_P (operands[4])"
2069 operands[4] = CONSTM1_RTX (<VPRED>mode);
2071 [(set_attr "movprfx" "*,yes,yes")]
2074 ;; -------------------------------------------------------------------------
2075 ;; ---- [INT<-FP] Base-2 logarithm
2076 ;; -------------------------------------------------------------------------
2079 ;; -------------------------------------------------------------------------
2081 ;; Predicated FLOGB.
2082 (define_insn "@aarch64_pred_<sve_fp_op><mode>"
2083 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
2084 (unspec:<V_INT_EQUIV>
2085 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2086 (match_operand:SI 3 "aarch64_sve_gp_strictness")
2087 (match_operand:SVE_FULL_F 2 "register_operand" "w")]
2088 SVE2_COND_INT_UNARY_FP))]
2090 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2093 ;; Predicated FLOGB with merging.
2094 (define_expand "@cond_<sve_fp_op><mode>"
2095 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
2096 (unspec:<V_INT_EQUIV>
2097 [(match_operand:<VPRED> 1 "register_operand")
2098 (unspec:<V_INT_EQUIV>
2100 (const_int SVE_STRICT_GP)
2101 (match_operand:SVE_FULL_F 2 "register_operand")]
2102 SVE2_COND_INT_UNARY_FP)
2103 (match_operand:<V_INT_EQUIV> 3 "aarch64_simd_reg_or_zero")]
2108 (define_insn_and_rewrite "*cond_<sve_fp_op><mode>"
2109 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=&w, ?&w, ?&w")
2110 (unspec:<V_INT_EQUIV>
2111 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2112 (unspec:<V_INT_EQUIV>
2114 (match_operand:SI 5 "aarch64_sve_gp_strictness")
2115 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")]
2116 SVE2_COND_INT_UNARY_FP)
2117 (match_operand:<V_INT_EQUIV> 3 "aarch64_simd_reg_or_zero" "0, Dz, w")]
2120 && !rtx_equal_p (operands[2], operands[3])
2121 && aarch64_sve_pred_dominates_p (&operands[4], operands[1])"
2123 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2124 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>
2125 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2126 "&& !rtx_equal_p (operands[1], operands[4])"
2128 operands[4] = copy_rtx (operands[1]);
2130 [(set_attr "movprfx" "*,yes,yes")]
2133 ;; -------------------------------------------------------------------------
2134 ;; ---- [INT] Polynomial multiplication
2135 ;; -------------------------------------------------------------------------
2140 ;; -------------------------------------------------------------------------
2143 (define_insn "@aarch64_sve2_pmul<mode>"
2144 [(set (match_operand:VNx16QI_ONLY 0 "register_operand" "=w")
2145 (unspec:VNx16QI_ONLY
2146 [(match_operand:VNx16QI_ONLY 1 "register_operand" "w")
2147 (match_operand:VNx16QI_ONLY 2 "register_operand" "w")]
2150 "pmul\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
2153 ;; Extending PMUL, with the results modeled as wider vectors.
2154 ;; This representation is only possible for .H and .D, not .Q.
2155 (define_insn "@aarch64_sve_<optab><mode>"
2156 [(set (match_operand:SVE_FULL_HDI 0 "register_operand" "=w")
2157 (unspec:SVE_FULL_HDI
2158 [(match_operand:<VNARROW> 1 "register_operand" "w")
2159 (match_operand:<VNARROW> 2 "register_operand" "w")]
2162 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>"
2165 ;; Extending PMUL, with the results modeled as pairs of values.
2166 ;; This representation works for .H, .D and .Q, with .Q requiring
2167 ;; the AES extension. (This is enforced by the mode iterator.)
2168 (define_insn "@aarch64_sve_<optab><mode>"
2169 [(set (match_operand:SVE2_PMULL_PAIR_I 0 "register_operand" "=w")
2170 (unspec:SVE2_PMULL_PAIR_I
2171 [(match_operand:SVE2_PMULL_PAIR_I 1 "register_operand" "w")
2172 (match_operand:SVE2_PMULL_PAIR_I 2 "register_operand" "w")]
2175 "<sve_int_op>\t%0.<Vewtype>, %1.<Vetype>, %2.<Vetype>"
2178 ;; =========================================================================
2180 ;; =========================================================================
2182 ;; -------------------------------------------------------------------------
2183 ;; ---- [INT,FP] General permutes
2184 ;; -------------------------------------------------------------------------
2186 ;; - TBL (vector pair form)
2188 ;; -------------------------------------------------------------------------
2190 ;; TBL on a pair of data vectors.
2191 (define_insn "@aarch64_sve2_tbl2<mode>"
2192 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
2194 [(match_operand:<VDOUBLE> 1 "register_operand" "w")
2195 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")]
2198 "tbl\t%0.<Vetype>, %1, %2.<Vetype>"
2201 ;; TBX. These instructions do not take MOVPRFX.
2202 (define_insn "@aarch64_sve2_tbx<mode>"
2203 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
2205 [(match_operand:SVE_FULL 1 "register_operand" "0")
2206 (match_operand:SVE_FULL 2 "register_operand" "w")
2207 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w")]
2210 "tbx\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
2213 ;; -------------------------------------------------------------------------
2214 ;; ---- [INT] Optional bit-permute extensions
2215 ;; -------------------------------------------------------------------------
2220 ;; -------------------------------------------------------------------------
2222 (define_insn "@aarch64_sve_<sve_int_op><mode>"
2223 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
2225 [(match_operand:SVE_FULL_I 1 "register_operand" "w")
2226 (match_operand:SVE_FULL_I 2 "register_operand" "w")]
2228 "TARGET_SVE2_BITPERM"
2229 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
2232 ;; =========================================================================
2234 ;; =========================================================================
2236 ;; -------------------------------------------------------------------------
2237 ;; ---- Check for aliases between pointers
2238 ;; -------------------------------------------------------------------------
2239 ;; The patterns in this section are synthetic: WHILERW and WHILEWR are
2240 ;; defined in aarch64-sve.md instead.
2241 ;; -------------------------------------------------------------------------
2243 ;; Use WHILERW and WHILEWR to accelerate alias checks. This is only
2244 ;; possible if the accesses we're checking are exactly the same size
2245 ;; as an SVE vector.
2246 (define_expand "check_<raw_war>_ptrs<mode>"
2247 [(match_operand:GPI 0 "register_operand")
2249 [(match_operand:GPI 1 "register_operand")
2250 (match_operand:GPI 2 "register_operand")
2251 (match_operand:GPI 3 "aarch64_bytes_per_sve_vector_operand")
2252 (match_operand:GPI 4 "const_int_operand")]
2256 /* Use the widest predicate mode we can. */
2257 unsigned int align = INTVAL (operands[4]);
2260 machine_mode pred_mode = aarch64_sve_pred_mode (align).require ();
2262 /* Emit a WHILERW or WHILEWR, setting the condition codes based on
2264 emit_insn (gen_while_ptest
2265 (<SVE2_WHILE_PTR:unspec>, <MODE>mode, pred_mode,
2266 gen_rtx_SCRATCH (pred_mode), operands[1], operands[2],
2267 CONSTM1_RTX (VNx16BImode), CONSTM1_RTX (pred_mode)));
2269 /* Set operand 0 to true if the last bit of the predicate result is set,
2270 i.e. if all elements are free of dependencies. */
2271 rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
2272 rtx cmp = gen_rtx_LTU (<MODE>mode, cc_reg, const0_rtx);
2273 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp, cc_reg));
2277 ;; -------------------------------------------------------------------------
2278 ;; ---- Histogram processing
2279 ;; -------------------------------------------------------------------------
2283 ;; -------------------------------------------------------------------------
2285 (define_insn "@aarch64_sve2_histcnt<mode>"
2286 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w")
2287 (unspec:SVE_FULL_SDI
2288 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2289 (match_operand:SVE_FULL_SDI 2 "register_operand" "w")
2290 (match_operand:SVE_FULL_SDI 3 "register_operand" "w")]
2293 "histcnt\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
2296 (define_insn "@aarch64_sve2_histseg<mode>"
2297 [(set (match_operand:VNx16QI_ONLY 0 "register_operand" "=w")
2298 (unspec:VNx16QI_ONLY
2299 [(match_operand:VNx16QI_ONLY 1 "register_operand" "w")
2300 (match_operand:VNx16QI_ONLY 2 "register_operand" "w")]
2303 "histseg\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
2306 ;; -------------------------------------------------------------------------
2307 ;; ---- String matching
2308 ;; -------------------------------------------------------------------------
2312 ;; -------------------------------------------------------------------------
2314 ;; Predicated string matching.
2315 (define_insn "@aarch64_pred_<sve_int_op><mode>"
2316 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
2318 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2319 (match_operand:SI 2 "aarch64_sve_ptrue_flag")
2321 [(match_operand:SVE_FULL_BHI 3 "register_operand" "w")
2322 (match_operand:SVE_FULL_BHI 4 "register_operand" "w")]
2325 (clobber (reg:CC_NZC CC_REGNUM))]
2327 "<sve_int_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
2330 ;; Predicated string matching in which both the flag and predicate results
2332 (define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_cc"
2333 [(set (reg:CC_NZC CC_REGNUM)
2335 [(match_operand:VNx16BI 1 "register_operand" "Upl")
2337 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
2340 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
2342 [(match_operand:SVE_FULL_BHI 2 "register_operand" "w")
2343 (match_operand:SVE_FULL_BHI 3 "register_operand" "w")]
2347 (set (match_operand:<VPRED> 0 "register_operand" "=Upa")
2357 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
2358 "<sve_int_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
2359 "&& !rtx_equal_p (operands[4], operands[6])"
2361 operands[6] = copy_rtx (operands[4]);
2362 operands[7] = operands[5];
2366 ;; Predicated string matching in which only the flags result is interesting.
2367 (define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_ptest"
2368 [(set (reg:CC_NZC CC_REGNUM)
2370 [(match_operand:VNx16BI 1 "register_operand" "Upl")
2372 (match_operand:SI 5 "aarch64_sve_ptrue_flag")
2375 (match_operand:SI 7 "aarch64_sve_ptrue_flag")
2377 [(match_operand:SVE_FULL_BHI 2 "register_operand" "w")
2378 (match_operand:SVE_FULL_BHI 3 "register_operand" "w")]
2382 (clobber (match_scratch:<VPRED> 0 "=Upa"))]
2384 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
2385 "<sve_int_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
2386 "&& !rtx_equal_p (operands[4], operands[6])"
2388 operands[6] = copy_rtx (operands[4]);
2389 operands[7] = operands[5];
2393 ;; =========================================================================
2394 ;; == Crypotographic extensions
2395 ;; =========================================================================
2397 ;; -------------------------------------------------------------------------
2398 ;; ---- Optional AES extensions
2399 ;; -------------------------------------------------------------------------
2405 ;; -------------------------------------------------------------------------
2408 (define_insn "aarch64_sve2_aes<aes_op>"
2409 [(set (match_operand:VNx16QI 0 "register_operand" "=w")
2412 (match_operand:VNx16QI 1 "register_operand" "%0")
2413 (match_operand:VNx16QI 2 "register_operand" "w"))]
2416 "aes<aes_op>\t%0.b, %0.b, %2.b"
2417 [(set_attr "type" "crypto_aese")]
2420 ;; AESMC and AESIMC. These instructions do not take MOVPRFX.
2421 (define_insn "aarch64_sve2_aes<aesmc_op>"
2422 [(set (match_operand:VNx16QI 0 "register_operand" "=w")
2424 [(match_operand:VNx16QI 1 "register_operand" "0")]
2427 "aes<aesmc_op>\t%0.b, %0.b"
2428 [(set_attr "type" "crypto_aesmc")]
2431 ;; When AESE/AESMC and AESD/AESIMC fusion is enabled, we really want
2432 ;; to keep the two together and enforce the register dependency without
2433 ;; scheduling or register allocation messing up the order or introducing
2434 ;; moves inbetween. Mash the two together during combine.
2436 (define_insn "*aarch64_sve2_aese_fused"
2437 [(set (match_operand:VNx16QI 0 "register_operand" "=w")
2441 (match_operand:VNx16QI 1 "register_operand" "%0")
2442 (match_operand:VNx16QI 2 "register_operand" "w"))]
2445 "TARGET_SVE2_AES && aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)"
2446 "aese\t%0.b, %0.b, %2.b\;aesmc\t%0.b, %0.b"
2447 [(set_attr "type" "crypto_aese")
2448 (set_attr "length" "8")]
2451 (define_insn "*aarch64_sve2_aesd_fused"
2452 [(set (match_operand:VNx16QI 0 "register_operand" "=w")
2456 (match_operand:VNx16QI 1 "register_operand" "%0")
2457 (match_operand:VNx16QI 2 "register_operand" "w"))]
2460 "TARGET_SVE2_AES && aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)"
2461 "aesd\t%0.b, %0.b, %2.b\;aesimc\t%0.b, %0.b"
2462 [(set_attr "type" "crypto_aese")
2463 (set_attr "length" "8")]
2466 ;; -------------------------------------------------------------------------
2467 ;; ---- Optional SHA-3 extensions
2468 ;; -------------------------------------------------------------------------
2471 ;; -------------------------------------------------------------------------
2473 (define_insn "aarch64_sve2_rax1"
2474 [(set (match_operand:VNx2DI 0 "register_operand" "=w")
2477 (match_operand:VNx2DI 2 "register_operand" "w")
2479 (match_operand:VNx2DI 1 "register_operand" "w")))]
2481 "rax1\t%0.d, %1.d, %2.d"
2482 [(set_attr "type" "crypto_sha3")]
2485 ;; -------------------------------------------------------------------------
2486 ;; ---- Optional SM4 extensions
2487 ;; -------------------------------------------------------------------------
2491 ;; -------------------------------------------------------------------------
2493 ;; These instructions do not take MOVPRFX.
2494 (define_insn "aarch64_sve2_sm4e"
2495 [(set (match_operand:VNx4SI 0 "register_operand" "=w")
2497 [(match_operand:VNx4SI 1 "register_operand" "0")
2498 (match_operand:VNx4SI 2 "register_operand" "w")]
2501 "sm4e\t%0.s, %0.s, %2.s"
2502 [(set_attr "type" "crypto_sm4")]
2505 (define_insn "aarch64_sve2_sm4ekey"
2506 [(set (match_operand:VNx4SI 0 "register_operand" "=w")
2508 [(match_operand:VNx4SI 1 "register_operand" "w")
2509 (match_operand:VNx4SI 2 "register_operand" "w")]
2512 "sm4ekey\t%0.s, %1.s, %2.s"
2513 [(set_attr "type" "crypto_sm4")]