1 ;; Machine description for AArch64 SVE.
2 ;; Copyright (C) 2009-2016 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; Note on the handling of big-endian SVE
22 ;; --------------------------------------
24 ;; On big-endian systems, Advanced SIMD mov<mode> patterns act in the
25 ;; same way as movdi or movti would: the first byte of memory goes
26 ;; into the most significant byte of the register and the last byte
27 ;; of memory goes into the least significant byte of the register.
28 ;; This is the most natural ordering for Advanced SIMD and matches
29 ;; the ABI layout for 64-bit and 128-bit vector types.
31 ;; As a result, the order of bytes within the register is what GCC
32 ;; expects for a big-endian target, and subreg offsets therefore work
33 ;; as expected, with the first element in memory having subreg offset 0
34 ;; and the last element in memory having the subreg offset associated
35 ;; with a big-endian lowpart. However, this ordering also means that
36 ;; GCC's lane numbering does not match the architecture's numbering:
37 ;; GCC always treats the element at the lowest address in memory
38 ;; (subreg offset 0) as element 0, while the architecture treats
39 ;; the least significant end of the register as element 0.
41 ;; The situation for SVE is different. We want the layout of the
42 ;; SVE register to be same for mov<mode> as it is for maskload<mode>:
43 ;; logically, a mov<mode> load must be indistinguishable from a
44 ;; maskload<mode> whose mask is all true. We therefore need the
45 ;; register layout to match LD1 rather than LDR. The ABI layout of
46 ;; SVE types also matches LD1 byte ordering rather than LDR byte ordering.
48 ;; As a result, the architecture lane numbering matches GCC's lane
49 ;; numbering, with element 0 always being the first in memory.
52 ;; - Applying a subreg offset to a register does not give the element
53 ;; that GCC expects: the first element in memory has the subreg offset
54 ;; associated with a big-endian lowpart while the last element in memory
55 ;; has subreg offset 0. We handle this via TARGET_CAN_CHANGE_MODE_CLASS.
57 ;; - We cannot use LDR and STR for spill slots that might be accessed
58 ;; via subregs, since although the elements have the order GCC expects,
59 ;; the order of the bytes within the elements is different. We instead
60 ;; access spill slots via LD1 and ST1, using secondary reloads to
61 ;; reserve a predicate register.
65 (define_expand "mov<mode>"
66 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
67 (match_operand:SVE_ALL 1 "general_operand"))]
70 /* Use the predicated load and store patterns where possible.
71 This is required for big-endian targets (see the comment at the
72 head of the file) and increases the addressing choices for
74 if ((MEM_P (operands[0]) || MEM_P (operands[1]))
75 && can_create_pseudo_p ())
77 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
81 if (CONSTANT_P (operands[1]))
83 aarch64_expand_mov_immediate (operands[0], operands[1],
84 gen_vec_duplicate<mode>);
88 /* Optimize subregs on big-endian targets: we can use REV[BHW]
89 instead of going through memory. */
91 && aarch64_maybe_expand_sve_subreg_move (operands[0], operands[1]))
96 ;; A pattern for optimizing SUBREGs that have a reinterpreting effect
97 ;; on big-endian targets; see aarch64_maybe_expand_sve_subreg_move
98 ;; for details. We use a special predicate for operand 2 to reduce
99 ;; the number of patterns.
100 (define_insn_and_split "*aarch64_sve_mov<mode>_subreg_be"
101 [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w")
103 [(match_operand:VNx16BI 1 "register_operand" "Upl")
104 (match_operand 2 "aarch64_any_register_operand" "w")]
106 "TARGET_SVE && BYTES_BIG_ENDIAN"
108 "&& reload_completed"
111 aarch64_split_sve_subreg_move (operands[0], operands[1], operands[2]);
116 ;; Unpredicated moves (little-endian). Only allow memory operations
117 ;; during and after RA; before RA we want the predicated load and
118 ;; store patterns to be used instead.
119 (define_insn "*aarch64_sve_mov<mode>_le"
120 [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
121 (match_operand:SVE_ALL 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
124 && ((lra_in_progress || reload_completed)
125 || (register_operand (operands[0], <MODE>mode)
126 && nonmemory_operand (operands[1], <MODE>mode)))"
131 * return aarch64_output_sve_mov_immediate (operands[1]);"
134 ;; Unpredicated moves (big-endian). Memory accesses require secondary
136 (define_insn "*aarch64_sve_mov<mode>_be"
137 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w")
138 (match_operand:SVE_ALL 1 "aarch64_nonmemory_operand" "w, Dn"))]
139 "TARGET_SVE && BYTES_BIG_ENDIAN"
142 * return aarch64_output_sve_mov_immediate (operands[1]);"
145 ;; Handle big-endian memory reloads. We use byte PTRUE for all modes
146 ;; to try to encourage reuse.
147 (define_expand "aarch64_sve_reload_be"
149 [(set (match_operand 0)
151 (clobber (match_operand:VNx16BI 2 "register_operand" "=Upl"))])]
152 "TARGET_SVE && BYTES_BIG_ENDIAN"
154 /* Create a PTRUE. */
155 emit_move_insn (operands[2], CONSTM1_RTX (VNx16BImode));
157 /* Refer to the PTRUE in the appropriate mode for this move. */
158 machine_mode mode = GET_MODE (operands[0]);
159 machine_mode pred_mode
160 = aarch64_sve_pred_mode (GET_MODE_UNIT_SIZE (mode)).require ();
161 rtx pred = gen_lowpart (pred_mode, operands[2]);
163 /* Emit a predicated load or store. */
164 aarch64_emit_sve_pred_move (operands[0], pred, operands[1]);
169 ;; A predicated load or store for which the predicate is known to be
170 ;; all-true. Note that this pattern is generated directly by
171 ;; aarch64_emit_sve_pred_move, so changes to this pattern will
172 ;; need changes there as well.
173 (define_insn "*pred_mov<mode>"
174 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand" "=w, m")
176 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
177 (match_operand:SVE_ALL 2 "nonimmediate_operand" "m, w")]
178 UNSPEC_MERGE_PTRUE))]
180 && (register_operand (operands[0], <MODE>mode)
181 || register_operand (operands[2], <MODE>mode))"
183 ld1<Vesize>\t%0.<Vetype>, %1/z, %2
184 st1<Vesize>\t%2.<Vetype>, %1, %0"
187 (define_expand "movmisalign<mode>"
188 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
189 (match_operand:SVE_ALL 1 "general_operand"))]
192 /* Equivalent to a normal move for our purpooses. */
193 emit_move_insn (operands[0], operands[1]);
198 (define_insn "maskload<mode><vpred>"
199 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
201 [(match_operand:<VPRED> 2 "register_operand" "Upl")
202 (match_operand:SVE_ALL 1 "memory_operand" "m")]
205 "ld1<Vesize>\t%0.<Vetype>, %2/z, %1"
208 (define_insn "maskstore<mode><vpred>"
209 [(set (match_operand:SVE_ALL 0 "memory_operand" "+m")
210 (unspec:SVE_ALL [(match_operand:<VPRED> 2 "register_operand" "Upl")
211 (match_operand:SVE_ALL 1 "register_operand" "w")
215 "st1<Vesize>\t%1.<Vetype>, %2, %0"
218 ;; Unpredicated gather loads.
219 (define_expand "gather_load<mode>"
220 [(set (match_operand:SVE_SD 0 "register_operand")
223 (match_operand:DI 1 "aarch64_reg_or_zero")
224 (match_operand:<V_INT_EQUIV> 2 "register_operand")
225 (match_operand:DI 3 "const_int_operand")
226 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>")
231 operands[5] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
235 ;; Predicated gather loads for 32-bit elements. Operand 3 is true for
236 ;; unsigned extension and false for signed extension.
237 (define_insn "mask_gather_load<mode>"
238 [(set (match_operand:SVE_S 0 "register_operand" "=w, w, w, w, w")
240 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl, Upl, Upl")
241 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, rk, rk, rk, rk")
242 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w, w, w, w")
243 (match_operand:DI 3 "const_int_operand" "i, Z, Ui1, Z, Ui1")
244 (match_operand:DI 4 "aarch64_gather_scale_operand_w" "Ui1, Ui1, Ui1, i, i")
249 ld1w\t%0.s, %5/z, [%2.s]
250 ld1w\t%0.s, %5/z, [%1, %2.s, sxtw]
251 ld1w\t%0.s, %5/z, [%1, %2.s, uxtw]
252 ld1w\t%0.s, %5/z, [%1, %2.s, sxtw %p4]
253 ld1w\t%0.s, %5/z, [%1, %2.s, uxtw %p4]"
256 ;; Predicated gather loads for 64-bit elements. The value of operand 3
257 ;; doesn't matter in this case.
258 (define_insn "mask_gather_load<mode>"
259 [(set (match_operand:SVE_D 0 "register_operand" "=w, w, w")
261 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl")
262 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, rk, rk")
263 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w, w")
264 (match_operand:DI 3 "const_int_operand")
265 (match_operand:DI 4 "aarch64_gather_scale_operand_d" "Ui1, Ui1, i")
270 ld1d\t%0.d, %5/z, [%2.d]
271 ld1d\t%0.d, %5/z, [%1, %2.d]
272 ld1d\t%0.d, %5/z, [%1, %2.d, lsl %p4]"
275 ;; Unpredicated scatter store.
276 (define_expand "scatter_store<mode>"
277 [(set (mem:BLK (scratch))
280 (match_operand:DI 0 "aarch64_reg_or_zero")
281 (match_operand:<V_INT_EQUIV> 1 "register_operand")
282 (match_operand:DI 2 "const_int_operand")
283 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>")
284 (match_operand:SVE_SD 4 "register_operand")]
285 UNSPEC_ST1_SCATTER))]
288 operands[5] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
292 ;; Predicated scatter stores for 32-bit elements. Operand 2 is true for
293 ;; unsigned extension and false for signed extension.
294 (define_insn "mask_scatter_store<mode>"
295 [(set (mem:BLK (scratch))
297 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl, Upl, Upl")
298 (match_operand:DI 0 "aarch64_reg_or_zero" "Z, rk, rk, rk, rk")
299 (match_operand:<V_INT_EQUIV> 1 "register_operand" "w, w, w, w, w")
300 (match_operand:DI 2 "const_int_operand" "i, Z, Ui1, Z, Ui1")
301 (match_operand:DI 3 "aarch64_gather_scale_operand_w" "Ui1, Ui1, Ui1, i, i")
302 (match_operand:SVE_S 4 "register_operand" "w, w, w, w, w")]
303 UNSPEC_ST1_SCATTER))]
306 st1w\t%4.s, %5, [%1.s]
307 st1w\t%4.s, %5, [%0, %1.s, sxtw]
308 st1w\t%4.s, %5, [%0, %1.s, uxtw]
309 st1w\t%4.s, %5, [%0, %1.s, sxtw %p3]
310 st1w\t%4.s, %5, [%0, %1.s, uxtw %p3]"
313 ;; Predicated scatter stores for 64-bit elements. The value of operand 2
314 ;; doesn't matter in this case.
315 (define_insn "mask_scatter_store<mode>"
316 [(set (mem:BLK (scratch))
318 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl")
319 (match_operand:DI 0 "aarch64_reg_or_zero" "Z, rk, rk")
320 (match_operand:<V_INT_EQUIV> 1 "register_operand" "w, w, w")
321 (match_operand:DI 2 "const_int_operand")
322 (match_operand:DI 3 "aarch64_gather_scale_operand_d" "Ui1, Ui1, i")
323 (match_operand:SVE_D 4 "register_operand" "w, w, w")]
324 UNSPEC_ST1_SCATTER))]
327 st1d\t%4.d, %5, [%1.d]
328 st1d\t%4.d, %5, [%0, %1.d]
329 st1d\t%4.d, %5, [%0, %1.d, lsl %p3]"
332 ;; SVE structure moves.
333 (define_expand "mov<mode>"
334 [(set (match_operand:SVE_STRUCT 0 "nonimmediate_operand")
335 (match_operand:SVE_STRUCT 1 "general_operand"))]
338 /* Big-endian loads and stores need to be done via LD1 and ST1;
339 see the comment at the head of the file for details. */
340 if ((MEM_P (operands[0]) || MEM_P (operands[1]))
343 gcc_assert (can_create_pseudo_p ());
344 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
348 if (CONSTANT_P (operands[1]))
350 aarch64_expand_mov_immediate (operands[0], operands[1]);
356 ;; Unpredicated structure moves (little-endian).
357 (define_insn "*aarch64_sve_mov<mode>_le"
358 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
359 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
360 "TARGET_SVE && !BYTES_BIG_ENDIAN"
362 [(set_attr "length" "<insn_length>")]
365 ;; Unpredicated structure moves (big-endian). Memory accesses require
366 ;; secondary reloads.
367 (define_insn "*aarch64_sve_mov<mode>_le"
368 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w, w")
369 (match_operand:SVE_STRUCT 1 "aarch64_nonmemory_operand" "w, Dn"))]
370 "TARGET_SVE && BYTES_BIG_ENDIAN"
372 [(set_attr "length" "<insn_length>")]
375 ;; Split unpredicated structure moves into pieces. This is the same
376 ;; for both big-endian and little-endian code, although it only needs
377 ;; to handle memory operands for little-endian code.
379 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand")
380 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand"))]
381 "TARGET_SVE && reload_completed"
384 rtx dest = operands[0];
385 rtx src = operands[1];
386 if (REG_P (dest) && REG_P (src))
387 aarch64_simd_emit_reg_reg_move (operands, <VSINGLE>mode, <vector_count>);
389 for (unsigned int i = 0; i < <vector_count>; ++i)
391 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, dest, <MODE>mode,
392 i * BYTES_PER_SVE_VECTOR);
393 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, src, <MODE>mode,
394 i * BYTES_PER_SVE_VECTOR);
395 emit_insn (gen_rtx_SET (subdest, subsrc));
401 ;; Predicated structure moves. This works for both endiannesses but in
402 ;; practice is only useful for big-endian.
403 (define_insn_and_split "pred_mov<mode>"
404 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_struct_nonimmediate_operand" "=w, Utx")
406 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
407 (match_operand:SVE_STRUCT 2 "aarch64_sve_struct_nonimmediate_operand" "Utx, w")]
408 UNSPEC_MERGE_PTRUE))]
410 && (register_operand (operands[0], <MODE>mode)
411 || register_operand (operands[2], <MODE>mode))"
413 "&& reload_completed"
416 for (unsigned int i = 0; i < <vector_count>; ++i)
418 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, operands[0],
420 i * BYTES_PER_SVE_VECTOR);
421 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, operands[2],
423 i * BYTES_PER_SVE_VECTOR);
424 aarch64_emit_sve_pred_move (subdest, operands[1], subsrc);
428 [(set_attr "length" "<insn_length>")]
431 (define_expand "mov<mode>"
432 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand")
433 (match_operand:PRED_ALL 1 "general_operand"))]
436 if (GET_CODE (operands[0]) == MEM)
437 operands[1] = force_reg (<MODE>mode, operands[1]);
441 (define_insn "*aarch64_sve_mov<mode>"
442 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand" "=Upa, m, Upa, Upa, Upa")
443 (match_operand:PRED_ALL 1 "general_operand" "Upa, Upa, m, Dz, Dm"))]
445 && (register_operand (operands[0], <MODE>mode)
446 || register_operand (operands[1], <MODE>mode))"
452 * return aarch64_output_ptrue (<MODE>mode, '<Vetype>');"
455 ;; Handle extractions from a predicate by converting to an integer vector
456 ;; and extracting from there.
457 (define_expand "vec_extract<vpred><Vel>"
458 [(match_operand:<VEL> 0 "register_operand")
459 (match_operand:<VPRED> 1 "register_operand")
460 (match_operand:SI 2 "nonmemory_operand")
461 ;; Dummy operand to which we can attach the iterator.
462 (reg:SVE_I V0_REGNUM)]
465 rtx tmp = gen_reg_rtx (<MODE>mode);
466 emit_insn (gen_aarch64_sve_dup<mode>_const (tmp, operands[1],
467 CONST1_RTX (<MODE>mode),
468 CONST0_RTX (<MODE>mode)));
469 emit_insn (gen_vec_extract<mode><Vel> (operands[0], tmp, operands[2]));
474 (define_expand "vec_extract<mode><Vel>"
475 [(set (match_operand:<VEL> 0 "register_operand")
477 (match_operand:SVE_ALL 1 "register_operand")
478 (parallel [(match_operand:SI 2 "nonmemory_operand")])))]
482 if (poly_int_rtx_p (operands[2], &val)
483 && known_eq (val, GET_MODE_NUNITS (<MODE>mode) - 1))
485 /* The last element can be extracted with a LASTB and a false
487 rtx sel = force_reg (<VPRED>mode, CONST0_RTX (<VPRED>mode));
488 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1]));
491 if (!CONST_INT_P (operands[2]))
493 /* Create an index with operand[2] as the base and -1 as the step.
494 It will then be zero for the element we care about. */
495 rtx index = gen_lowpart (<VEL_INT>mode, operands[2]);
496 index = force_reg (<VEL_INT>mode, index);
497 rtx series = gen_reg_rtx (<V_INT_EQUIV>mode);
498 emit_insn (gen_vec_series<v_int_equiv> (series, index, constm1_rtx));
500 /* Get a predicate that is true for only that element. */
501 rtx zero = CONST0_RTX (<V_INT_EQUIV>mode);
502 rtx cmp = gen_rtx_EQ (<V_INT_EQUIV>mode, series, zero);
503 rtx sel = gen_reg_rtx (<VPRED>mode);
504 emit_insn (gen_vec_cmp<v_int_equiv><vpred> (sel, cmp, series, zero));
506 /* Select the element using LASTB. */
507 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1]));
513 ;; Extract element zero. This is a special case because we want to force
514 ;; the registers to be the same for the second alternative, and then
515 ;; split the instruction into nothing after RA.
516 (define_insn_and_split "*vec_extract<mode><Vel>_0"
517 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
519 (match_operand:SVE_ALL 1 "register_operand" "w, 0, w")
520 (parallel [(const_int 0)])))]
523 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
524 switch (which_alternative)
527 return "umov\\t%<vwcore>0, %1.<Vetype>[0]";
531 return "st1\\t{%1.<Vetype>}[0], %0";
537 && REG_P (operands[0])
538 && REGNO (operands[0]) == REGNO (operands[1])"
541 emit_note (NOTE_INSN_DELETED);
544 [(set_attr "type" "neon_to_gp_q, untyped, neon_store1_one_lane_q")]
547 ;; Extract an element from the Advanced SIMD portion of the register.
548 ;; We don't just reuse the aarch64-simd.md pattern because we don't
549 ;; want any change in lane number on big-endian targets.
550 (define_insn "*vec_extract<mode><Vel>_v128"
551 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
553 (match_operand:SVE_ALL 1 "register_operand" "w, w, w")
554 (parallel [(match_operand:SI 2 "const_int_operand")])))]
556 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 1, 15)"
558 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
559 switch (which_alternative)
562 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
564 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
566 return "st1\\t{%1.<Vetype>}[%2], %0";
571 [(set_attr "type" "neon_to_gp_q, neon_dup_q, neon_store1_one_lane_q")]
574 ;; Extract an element in the range of DUP. This pattern allows the
575 ;; source and destination to be different.
576 (define_insn "*vec_extract<mode><Vel>_dup"
577 [(set (match_operand:<VEL> 0 "register_operand" "=w")
579 (match_operand:SVE_ALL 1 "register_operand" "w")
580 (parallel [(match_operand:SI 2 "const_int_operand")])))]
582 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 16, 63)"
584 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
585 return "dup\t%0.<Vetype>, %1.<Vetype>[%2]";
589 ;; Extract an element outside the range of DUP. This pattern requires the
590 ;; source and destination to be the same.
591 (define_insn "*vec_extract<mode><Vel>_ext"
592 [(set (match_operand:<VEL> 0 "register_operand" "=w")
594 (match_operand:SVE_ALL 1 "register_operand" "0")
595 (parallel [(match_operand:SI 2 "const_int_operand")])))]
596 "TARGET_SVE && INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode) >= 64"
598 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
599 operands[2] = GEN_INT (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode));
600 return "ext\t%0.b, %0.b, %0.b, #%2";
604 ;; Extract the last active element of operand 1 into operand 0.
605 ;; If no elements are active, extract the last inactive element instead.
606 (define_insn "extract_last_<mode>"
607 [(set (match_operand:<VEL> 0 "register_operand" "=r, w")
609 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
610 (match_operand:SVE_ALL 2 "register_operand" "w, w")]
614 lastb\t%<vwcore>0, %1, %2.<Vetype>
615 lastb\t%<Vetype>0, %1, %2.<Vetype>"
618 (define_expand "vec_duplicate<mode>"
620 [(set (match_operand:SVE_ALL 0 "register_operand")
621 (vec_duplicate:SVE_ALL
622 (match_operand:<VEL> 1 "aarch64_sve_dup_operand")))
623 (clobber (scratch:<VPRED>))])]
626 if (MEM_P (operands[1]))
628 rtx ptrue = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
629 emit_insn (gen_sve_ld1r<mode> (operands[0], ptrue, operands[1],
630 CONST0_RTX (<MODE>mode)));
636 ;; Accept memory operands for the benefit of combine, and also in case
637 ;; the scalar input gets spilled to memory during RA. We want to split
638 ;; the load at the first opportunity in order to allow the PTRUE to be
639 ;; optimized with surrounding code.
640 (define_insn_and_split "*vec_duplicate<mode>_reg"
641 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w, w")
642 (vec_duplicate:SVE_ALL
643 (match_operand:<VEL> 1 "aarch64_sve_dup_operand" "r, w, Uty")))
644 (clobber (match_scratch:<VPRED> 2 "=X, X, Upl"))]
647 mov\t%0.<Vetype>, %<vwcore>1
648 mov\t%0.<Vetype>, %<Vetype>1
650 "&& MEM_P (operands[1])"
653 if (GET_CODE (operands[2]) == SCRATCH)
654 operands[2] = gen_reg_rtx (<VPRED>mode);
655 emit_move_insn (operands[2], CONSTM1_RTX (<VPRED>mode));
656 emit_insn (gen_sve_ld1r<mode> (operands[0], operands[2], operands[1],
657 CONST0_RTX (<MODE>mode)));
660 [(set_attr "length" "4,4,8")]
663 ;; This is used for vec_duplicate<mode>s from memory, but can also
664 ;; be used by combine to optimize selects of a a vec_duplicate<mode>
666 (define_insn "sve_ld1r<mode>"
667 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
669 [(match_operand:<VPRED> 1 "register_operand" "Upl")
670 (vec_duplicate:SVE_ALL
671 (match_operand:<VEL> 2 "aarch64_sve_ld1r_operand" "Uty"))
672 (match_operand:SVE_ALL 3 "aarch64_simd_imm_zero")]
675 "ld1r<Vesize>\t%0.<Vetype>, %1/z, %2"
678 ;; Load 128 bits from memory and duplicate to fill a vector. Since there
679 ;; are so few operations on 128-bit "elements", we don't define a VNx1TI
680 ;; and simply use vectors of bytes instead.
681 (define_insn "*sve_ld1rq<Vesize>"
682 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
684 [(match_operand:<VPRED> 1 "register_operand" "Upl")
685 (match_operand:TI 2 "aarch64_sve_ld1r_operand" "Uty")]
688 "ld1rq<Vesize>\t%0.<Vetype>, %1/z, %2"
691 ;; Implement a predicate broadcast by shifting the low bit of the scalar
692 ;; input into the top bit and using a WHILELO. An alternative would be to
693 ;; duplicate the input and do a compare with zero.
694 (define_expand "vec_duplicate<mode>"
695 [(set (match_operand:PRED_ALL 0 "register_operand")
696 (vec_duplicate:PRED_ALL (match_operand 1 "register_operand")))]
699 rtx tmp = gen_reg_rtx (DImode);
700 rtx op1 = gen_lowpart (DImode, operands[1]);
701 emit_insn (gen_ashldi3 (tmp, op1, gen_int_mode (63, DImode)));
702 emit_insn (gen_while_ultdi<mode> (operands[0], const0_rtx, tmp));
707 (define_insn "vec_series<mode>"
708 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w")
710 (match_operand:<VEL> 1 "aarch64_sve_index_operand" "Usi, r, r")
711 (match_operand:<VEL> 2 "aarch64_sve_index_operand" "r, Usi, r")))]
714 index\t%0.<Vetype>, #%1, %<vw>2
715 index\t%0.<Vetype>, %<vw>1, #%2
716 index\t%0.<Vetype>, %<vw>1, %<vw>2"
719 ;; Optimize {x, x, x, x, ...} + {0, n, 2*n, 3*n, ...} if n is in range
720 ;; of an INDEX instruction.
721 (define_insn "*vec_series<mode>_plus"
722 [(set (match_operand:SVE_I 0 "register_operand" "=w")
725 (match_operand:<VEL> 1 "register_operand" "r"))
726 (match_operand:SVE_I 2 "immediate_operand")))]
727 "TARGET_SVE && aarch64_check_zero_based_sve_index_immediate (operands[2])"
729 operands[2] = aarch64_check_zero_based_sve_index_immediate (operands[2]);
730 return "index\t%0.<Vetype>, %<vw>1, #%2";
734 ;; Unpredicated LD[234].
735 (define_expand "vec_load_lanes<mode><vsingle>"
736 [(set (match_operand:SVE_STRUCT 0 "register_operand")
739 (match_operand:SVE_STRUCT 1 "memory_operand")]
743 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
747 ;; Predicated LD[234].
748 (define_insn "vec_mask_load_lanes<mode><vsingle>"
749 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w")
751 [(match_operand:<VPRED> 2 "register_operand" "Upl")
752 (match_operand:SVE_STRUCT 1 "memory_operand" "m")]
755 "ld<vector_count><Vesize>\t%0, %2/z, %1"
758 ;; Unpredicated ST[234]. This is always a full update, so the dependence
759 ;; on the old value of the memory location (via (match_dup 0)) is redundant.
760 ;; There doesn't seem to be any obvious benefit to treating the all-true
761 ;; case differently though. In particular, it's very unlikely that we'll
762 ;; only find out during RTL that a store_lanes is dead.
763 (define_expand "vec_store_lanes<mode><vsingle>"
764 [(set (match_operand:SVE_STRUCT 0 "memory_operand")
767 (match_operand:SVE_STRUCT 1 "register_operand")
772 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
776 ;; Predicated ST[234].
777 (define_insn "vec_mask_store_lanes<mode><vsingle>"
778 [(set (match_operand:SVE_STRUCT 0 "memory_operand" "+m")
780 [(match_operand:<VPRED> 2 "register_operand" "Upl")
781 (match_operand:SVE_STRUCT 1 "register_operand" "w")
785 "st<vector_count><Vesize>\t%1, %2, %0"
788 (define_expand "vec_perm<mode>"
789 [(match_operand:SVE_ALL 0 "register_operand")
790 (match_operand:SVE_ALL 1 "register_operand")
791 (match_operand:SVE_ALL 2 "register_operand")
792 (match_operand:<V_INT_EQUIV> 3 "aarch64_sve_vec_perm_operand")]
793 "TARGET_SVE && GET_MODE_NUNITS (<MODE>mode).is_constant ()"
795 aarch64_expand_sve_vec_perm (operands[0], operands[1],
796 operands[2], operands[3]);
801 (define_insn "*aarch64_sve_tbl<mode>"
802 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
804 [(match_operand:SVE_ALL 1 "register_operand" "w")
805 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")]
808 "tbl\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
811 (define_insn "*aarch64_sve_<perm_insn><perm_hilo><mode>"
812 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
813 (unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa")
814 (match_operand:PRED_ALL 2 "register_operand" "Upa")]
817 "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
820 (define_insn "aarch64_sve_<perm_insn><perm_hilo><mode>"
821 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
822 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")
823 (match_operand:SVE_ALL 2 "register_operand" "w")]
826 "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
829 (define_insn "*aarch64_sve_rev64<mode>"
830 [(set (match_operand:SVE_BHS 0 "register_operand" "=w")
832 [(match_operand:VNx2BI 1 "register_operand" "Upl")
833 (unspec:SVE_BHS [(match_operand:SVE_BHS 2 "register_operand" "w")]
835 UNSPEC_MERGE_PTRUE))]
837 "rev<Vesize>\t%0.d, %1/m, %2.d"
840 (define_insn "*aarch64_sve_rev32<mode>"
841 [(set (match_operand:SVE_BH 0 "register_operand" "=w")
843 [(match_operand:VNx4BI 1 "register_operand" "Upl")
844 (unspec:SVE_BH [(match_operand:SVE_BH 2 "register_operand" "w")]
846 UNSPEC_MERGE_PTRUE))]
848 "rev<Vesize>\t%0.s, %1/m, %2.s"
851 (define_insn "*aarch64_sve_rev16vnx16qi"
852 [(set (match_operand:VNx16QI 0 "register_operand" "=w")
854 [(match_operand:VNx8BI 1 "register_operand" "Upl")
855 (unspec:VNx16QI [(match_operand:VNx16QI 2 "register_operand" "w")]
857 UNSPEC_MERGE_PTRUE))]
859 "revb\t%0.h, %1/m, %2.h"
862 (define_insn "*aarch64_sve_rev<mode>"
863 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
864 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")]
867 "rev\t%0.<Vetype>, %1.<Vetype>")
869 (define_insn "*aarch64_sve_dup_lane<mode>"
870 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
871 (vec_duplicate:SVE_ALL
873 (match_operand:SVE_ALL 1 "register_operand" "w")
874 (parallel [(match_operand:SI 2 "const_int_operand")]))))]
876 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 0, 63)"
877 "dup\t%0.<Vetype>, %1.<Vetype>[%2]"
880 ;; Note that the immediate (third) operand is the lane index not
882 (define_insn "*aarch64_sve_ext<mode>"
883 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
884 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "0")
885 (match_operand:SVE_ALL 2 "register_operand" "w")
886 (match_operand:SI 3 "const_int_operand")]
889 && IN_RANGE (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode), 0, 255)"
891 operands[3] = GEN_INT (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode));
892 return "ext\\t%0.b, %0.b, %2.b, #%3";
896 (define_insn "add<mode>3"
897 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w, w")
899 (match_operand:SVE_I 1 "register_operand" "%0, 0, 0, w")
900 (match_operand:SVE_I 2 "aarch64_sve_add_operand" "vsa, vsn, vsi, w")))]
903 add\t%0.<Vetype>, %0.<Vetype>, #%D2
904 sub\t%0.<Vetype>, %0.<Vetype>, #%N2
905 * return aarch64_output_sve_inc_dec_immediate (\"%0.<Vetype>\", operands[2]);
906 add\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
909 (define_insn "sub<mode>3"
910 [(set (match_operand:SVE_I 0 "register_operand" "=w, w")
912 (match_operand:SVE_I 1 "aarch64_sve_arith_operand" "w, vsa")
913 (match_operand:SVE_I 2 "register_operand" "w, 0")))]
916 sub\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>
917 subr\t%0.<Vetype>, %0.<Vetype>, #%D1"
920 ;; Unpredicated multiplication.
921 (define_expand "mul<mode>3"
922 [(set (match_operand:SVE_I 0 "register_operand")
926 (match_operand:SVE_I 1 "register_operand")
927 (match_operand:SVE_I 2 "aarch64_sve_mul_operand"))]
928 UNSPEC_MERGE_PTRUE))]
931 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
935 ;; Multiplication predicated with a PTRUE. We don't actually need the
936 ;; predicate for the first alternative, but using Upa or X isn't likely
937 ;; to gain much and would make the instruction seem less uniform to the
938 ;; register allocator.
939 (define_insn_and_split "*mul<mode>3"
940 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w")
942 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
944 (match_operand:SVE_I 2 "register_operand" "%0, 0, w")
945 (match_operand:SVE_I 3 "aarch64_sve_mul_operand" "vsm, w, w"))]
946 UNSPEC_MERGE_PTRUE))]
950 mul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
951 movprfx\t%0, %2\;mul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
952 ; Split the unpredicated form after reload, so that we don't have
953 ; the unnecessary PTRUE.
955 && !register_operand (operands[3], <MODE>mode)"
956 [(set (match_dup 0) (mult:SVE_I (match_dup 2) (match_dup 3)))]
958 [(set_attr "movprfx" "*,*,yes")]
961 ;; Unpredicated multiplications by a constant (post-RA only).
962 ;; These are generated by splitting a predicated instruction whose
963 ;; predicate is unused.
964 (define_insn "*post_ra_mul<mode>3"
965 [(set (match_operand:SVE_I 0 "register_operand" "=w")
967 (match_operand:SVE_I 1 "register_operand" "0")
968 (match_operand:SVE_I 2 "aarch64_sve_mul_immediate")))]
969 "TARGET_SVE && reload_completed"
970 "mul\t%0.<Vetype>, %0.<Vetype>, #%2"
973 (define_insn "*madd<mode>"
974 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w")
977 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
978 (mult:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w, w")
979 (match_operand:SVE_I 3 "register_operand" "w, w, w"))]
981 (match_operand:SVE_I 4 "register_operand" "w, 0, w")))]
984 mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
985 mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
986 movprfx\t%0, %4\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
987 [(set_attr "movprfx" "*,*,yes")]
990 (define_insn "*msub<mode>3"
991 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w")
993 (match_operand:SVE_I 4 "register_operand" "w, 0, w")
995 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
996 (mult:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w, w")
997 (match_operand:SVE_I 3 "register_operand" "w, w, w"))]
998 UNSPEC_MERGE_PTRUE)))]
1001 msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
1002 mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
1003 movprfx\t%0, %4\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
1004 [(set_attr "movprfx" "*,*,yes")]
1007 ;; Unpredicated highpart multiplication.
1008 (define_expand "<su>mul<mode>3_highpart"
1009 [(set (match_operand:SVE_I 0 "register_operand")
1012 (unspec:SVE_I [(match_operand:SVE_I 1 "register_operand")
1013 (match_operand:SVE_I 2 "register_operand")]
1015 UNSPEC_MERGE_PTRUE))]
1018 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1022 ;; Predicated highpart multiplication.
1023 (define_insn "*<su>mul<mode>3_highpart"
1024 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
1026 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1027 (unspec:SVE_I [(match_operand:SVE_I 2 "register_operand" "%0, w")
1028 (match_operand:SVE_I 3 "register_operand" "w, w")]
1030 UNSPEC_MERGE_PTRUE))]
1033 <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1034 movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1035 [(set_attr "movprfx" "*,yes")]
1038 ;; Unpredicated division.
1039 (define_expand "<optab><mode>3"
1040 [(set (match_operand:SVE_SDI 0 "register_operand")
1043 (SVE_INT_BINARY_SD:SVE_SDI
1044 (match_operand:SVE_SDI 1 "register_operand")
1045 (match_operand:SVE_SDI 2 "register_operand"))]
1046 UNSPEC_MERGE_PTRUE))]
1049 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1053 ;; Division predicated with a PTRUE.
1054 (define_insn "*<optab><mode>3"
1055 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, w, ?&w")
1057 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
1058 (SVE_INT_BINARY_SD:SVE_SDI
1059 (match_operand:SVE_SDI 2 "register_operand" "0, w, w")
1060 (match_operand:SVE_SDI 3 "aarch64_sve_mul_operand" "w, 0, w"))]
1061 UNSPEC_MERGE_PTRUE))]
1064 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1065 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
1066 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1067 [(set_attr "movprfx" "*,*,yes")]
1070 ;; Unpredicated NEG, NOT and POPCOUNT.
1071 (define_expand "<optab><mode>2"
1072 [(set (match_operand:SVE_I 0 "register_operand")
1075 (SVE_INT_UNARY:SVE_I (match_operand:SVE_I 1 "register_operand"))]
1076 UNSPEC_MERGE_PTRUE))]
1079 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1083 ;; NEG, NOT and POPCOUNT predicated with a PTRUE.
1084 (define_insn "*<optab><mode>2"
1085 [(set (match_operand:SVE_I 0 "register_operand" "=w")
1087 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1088 (SVE_INT_UNARY:SVE_I
1089 (match_operand:SVE_I 2 "register_operand" "w"))]
1090 UNSPEC_MERGE_PTRUE))]
1092 "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
1095 ;; Vector AND, ORR and XOR.
1096 (define_insn "<optab><mode>3"
1097 [(set (match_operand:SVE_I 0 "register_operand" "=w, w")
1099 (match_operand:SVE_I 1 "register_operand" "%0, w")
1100 (match_operand:SVE_I 2 "aarch64_sve_logical_operand" "vsl, w")))]
1103 <logical>\t%0.<Vetype>, %0.<Vetype>, #%C2
1104 <logical>\t%0.d, %1.d, %2.d"
1107 ;; Vector AND, ORR and XOR on floating-point modes. We avoid subregs
1108 ;; by providing this, but we need to use UNSPECs since rtx logical ops
1109 ;; aren't defined for floating-point modes.
1110 (define_insn "*<optab><mode>3"
1111 [(set (match_operand:SVE_F 0 "register_operand" "=w")
1112 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand" "w")
1113 (match_operand:SVE_F 2 "register_operand" "w")]
1116 "<logicalf_op>\t%0.d, %1.d, %2.d"
1119 ;; REG_EQUAL notes on "not<mode>3" should ensure that we can generate
1120 ;; this pattern even though the NOT instruction itself is predicated.
1121 (define_insn "bic<mode>3"
1122 [(set (match_operand:SVE_I 0 "register_operand" "=w")
1124 (not:SVE_I (match_operand:SVE_I 1 "register_operand" "w"))
1125 (match_operand:SVE_I 2 "register_operand" "w")))]
1127 "bic\t%0.d, %2.d, %1.d"
1130 ;; Predicate AND. We can reuse one of the inputs as the GP.
1131 (define_insn "and<mode>3"
1132 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1133 (and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand" "Upa")
1134 (match_operand:PRED_ALL 2 "register_operand" "Upa")))]
1136 "and\t%0.b, %1/z, %1.b, %2.b"
1139 ;; Unpredicated predicate ORR and XOR.
1140 (define_expand "<optab><mode>3"
1141 [(set (match_operand:PRED_ALL 0 "register_operand")
1143 (LOGICAL_OR:PRED_ALL
1144 (match_operand:PRED_ALL 1 "register_operand")
1145 (match_operand:PRED_ALL 2 "register_operand"))
1149 operands[3] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));
1153 ;; Predicated predicate ORR and XOR.
1154 (define_insn "pred_<optab><mode>3"
1155 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1158 (match_operand:PRED_ALL 2 "register_operand" "Upa")
1159 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
1160 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
1162 "<logical>\t%0.b, %1/z, %2.b, %3.b"
1165 ;; Perform a logical operation on operands 2 and 3, using operand 1 as
1166 ;; the GP (which is known to be a PTRUE). Store the result in operand 0
1167 ;; and set the flags in the same way as for PTEST. The (and ...) in the
1168 ;; UNSPEC_PTEST_PTRUE is logically redundant, but means that the tested
1169 ;; value is structurally equivalent to rhs of the second set.
1170 (define_insn "*<optab><mode>3_cc"
1171 [(set (reg:CC CC_REGNUM)
1173 (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upa")
1176 (match_operand:PRED_ALL 2 "register_operand" "Upa")
1177 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
1181 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1182 (and:PRED_ALL (LOGICAL:PRED_ALL (match_dup 2) (match_dup 3))
1185 "<logical>s\t%0.b, %1/z, %2.b, %3.b"
1188 ;; Unpredicated predicate inverse.
1189 (define_expand "one_cmpl<mode>2"
1190 [(set (match_operand:PRED_ALL 0 "register_operand")
1192 (not:PRED_ALL (match_operand:PRED_ALL 1 "register_operand"))
1196 operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));
1200 ;; Predicated predicate inverse.
1201 (define_insn "*one_cmpl<mode>3"
1202 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1204 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
1205 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
1207 "not\t%0.b, %1/z, %2.b"
1210 ;; Predicated predicate BIC and ORN.
1211 (define_insn "*<nlogical><mode>3"
1212 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1215 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
1216 (match_operand:PRED_ALL 3 "register_operand" "Upa"))
1217 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
1219 "<nlogical>\t%0.b, %1/z, %3.b, %2.b"
1222 ;; Predicated predicate NAND and NOR.
1223 (define_insn "*<logical_nn><mode>3"
1224 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1227 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
1228 (not:PRED_ALL (match_operand:PRED_ALL 3 "register_operand" "Upa")))
1229 (match_operand:PRED_ALL 1 "register_operand" "Upa")))]
1231 "<logical_nn>\t%0.b, %1/z, %2.b, %3.b"
1234 ;; Unpredicated LSL, LSR and ASR by a vector.
1235 (define_expand "v<optab><mode>3"
1236 [(set (match_operand:SVE_I 0 "register_operand")
1240 (match_operand:SVE_I 1 "register_operand")
1241 (match_operand:SVE_I 2 "aarch64_sve_<lr>shift_operand"))]
1242 UNSPEC_MERGE_PTRUE))]
1245 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1249 ;; LSL, LSR and ASR by a vector, predicated with a PTRUE. We don't
1250 ;; actually need the predicate for the first alternative, but using Upa
1251 ;; or X isn't likely to gain much and would make the instruction seem
1252 ;; less uniform to the register allocator.
1253 (define_insn_and_split "*v<optab><mode>3"
1254 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w")
1256 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
1258 (match_operand:SVE_I 2 "register_operand" "w, 0, w")
1259 (match_operand:SVE_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, w"))]
1260 UNSPEC_MERGE_PTRUE))]
1264 <shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1265 movprfx\t%0, %2\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1266 "&& reload_completed
1267 && !register_operand (operands[3], <MODE>mode)"
1268 [(set (match_dup 0) (ASHIFT:SVE_I (match_dup 2) (match_dup 3)))]
1270 [(set_attr "movprfx" "*,*,yes")]
1273 ;; Unpredicated shift operations by a constant (post-RA only).
1274 ;; These are generated by splitting a predicated instruction whose
1275 ;; predicate is unused.
1276 (define_insn "*post_ra_v<optab><mode>3"
1277 [(set (match_operand:SVE_I 0 "register_operand" "=w")
1279 (match_operand:SVE_I 1 "register_operand" "w")
1280 (match_operand:SVE_I 2 "aarch64_simd_<lr>shift_imm")))]
1281 "TARGET_SVE && reload_completed"
1282 "<shift>\t%0.<Vetype>, %1.<Vetype>, #%2"
1285 ;; LSL, LSR and ASR by a scalar, which expands into one of the vector
1287 (define_expand "<ASHIFT:optab><mode>3"
1288 [(set (match_operand:SVE_I 0 "register_operand")
1289 (ASHIFT:SVE_I (match_operand:SVE_I 1 "register_operand")
1290 (match_operand:<VEL> 2 "general_operand")))]
1294 if (CONST_INT_P (operands[2]))
1296 amount = gen_const_vec_duplicate (<MODE>mode, operands[2]);
1297 if (!aarch64_sve_<lr>shift_operand (operands[2], <MODE>mode))
1298 amount = force_reg (<MODE>mode, amount);
1302 amount = gen_reg_rtx (<MODE>mode);
1303 emit_insn (gen_vec_duplicate<mode> (amount,
1304 convert_to_mode (<VEL>mode,
1307 emit_insn (gen_v<optab><mode>3 (operands[0], operands[1], amount));
1312 ;; Test all bits of operand 1. Operand 0 is a GP that is known to hold PTRUE.
1314 ;; Using UNSPEC_PTEST_PTRUE allows combine patterns to assume that the GP
1315 ;; is a PTRUE even if the optimizers haven't yet been able to propagate
1316 ;; the constant. We would use a separate unspec code for PTESTs involving
1317 ;; GPs that might not be PTRUEs.
1318 (define_insn "ptest_ptrue<mode>"
1319 [(set (reg:CC CC_REGNUM)
1321 (unspec:SI [(match_operand:PRED_ALL 0 "register_operand" "Upa")
1322 (match_operand:PRED_ALL 1 "register_operand" "Upa")]
1329 ;; Set element I of the result if operand1 + J < operand2 for all J in [0, I].
1330 ;; with the comparison being unsigned.
1331 (define_insn "while_ult<GPI:mode><PRED_ALL:mode>"
1332 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1333 (unspec:PRED_ALL [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
1334 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
1336 (clobber (reg:CC CC_REGNUM))]
1338 "whilelo\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
1341 ;; WHILELO sets the flags in the same way as a PTEST with a PTRUE GP.
1342 ;; Handle the case in which both results are useful. The GP operand
1343 ;; to the PTEST isn't needed, so we allow it to be anything.
1344 (define_insn_and_split "while_ult<GPI:mode><PRED_ALL:mode>_cc"
1345 [(set (reg:CC CC_REGNUM)
1347 (unspec:SI [(match_operand:PRED_ALL 1)
1349 [(match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")
1350 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")]
1354 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
1355 (unspec:PRED_ALL [(match_dup 2)
1359 "whilelo\t%0.<PRED_ALL:Vetype>, %<w>2, %<w>3"
1360 ;; Force the compiler to drop the unused predicate operand, so that we
1361 ;; don't have an unnecessary PTRUE.
1362 "&& !CONSTANT_P (operands[1])"
1365 emit_insn (gen_while_ult<GPI:mode><PRED_ALL:mode>_cc
1366 (operands[0], CONSTM1_RTX (<MODE>mode),
1367 operands[2], operands[3]));
1372 ;; Integer comparisons predicated with a PTRUE.
1373 (define_insn "*cmp<cmp_op><mode>"
1374 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1376 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1377 (SVE_INT_CMP:<VPRED>
1378 (match_operand:SVE_I 2 "register_operand" "w, w")
1379 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
1380 UNSPEC_MERGE_PTRUE))
1381 (clobber (reg:CC CC_REGNUM))]
1384 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
1385 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1388 ;; Integer comparisons predicated with a PTRUE in which only the flags result
1390 (define_insn "*cmp<cmp_op><mode>_ptest"
1391 [(set (reg:CC CC_REGNUM)
1394 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1397 (SVE_INT_CMP:<VPRED>
1398 (match_operand:SVE_I 2 "register_operand" "w, w")
1399 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
1400 UNSPEC_MERGE_PTRUE)]
1403 (clobber (match_scratch:<VPRED> 0 "=Upa, Upa"))]
1406 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
1407 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1410 ;; Integer comparisons predicated with a PTRUE in which both the flag and
1411 ;; predicate results are interesting.
1412 (define_insn "*cmp<cmp_op><mode>_cc"
1413 [(set (reg:CC CC_REGNUM)
1416 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1419 (SVE_INT_CMP:<VPRED>
1420 (match_operand:SVE_I 2 "register_operand" "w, w")
1421 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
1422 UNSPEC_MERGE_PTRUE)]
1425 (set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1428 (SVE_INT_CMP:<VPRED>
1431 UNSPEC_MERGE_PTRUE))]
1434 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
1435 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1438 ;; Predicated integer comparisons, formed by combining a PTRUE-predicated
1439 ;; comparison with an AND. Split the instruction into its preferred form
1440 ;; (below) at the earliest opportunity, in order to get rid of the
1441 ;; redundant operand 1.
1442 (define_insn_and_split "*pred_cmp<cmp_op><mode>_combine"
1443 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1446 [(match_operand:<VPRED> 1)
1447 (SVE_INT_CMP:<VPRED>
1448 (match_operand:SVE_I 2 "register_operand" "w, w")
1449 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
1451 (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))
1452 (clobber (reg:CC CC_REGNUM))]
1459 (SVE_INT_CMP:<VPRED>
1463 (clobber (reg:CC CC_REGNUM))])]
1466 ;; Predicated integer comparisons.
1467 (define_insn "*pred_cmp<cmp_op><mode>"
1468 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1470 (SVE_INT_CMP:<VPRED>
1471 (match_operand:SVE_I 2 "register_operand" "w, w")
1472 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))
1473 (match_operand:<VPRED> 1 "register_operand" "Upl, Upl")))
1474 (clobber (reg:CC CC_REGNUM))]
1477 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
1478 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1481 ;; Floating-point comparisons predicated with a PTRUE.
1482 (define_insn "*fcm<cmp_op><mode>"
1483 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1485 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1487 (match_operand:SVE_F 2 "register_operand" "w, w")
1488 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))]
1489 UNSPEC_MERGE_PTRUE))]
1492 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
1493 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1496 (define_insn "*fcmuo<mode>"
1497 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
1499 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1501 (match_operand:SVE_F 2 "register_operand" "w")
1502 (match_operand:SVE_F 3 "register_operand" "w"))]
1503 UNSPEC_MERGE_PTRUE))]
1505 "fcmuo\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1508 ;; Floating-point comparisons predicated on a PTRUE, with the results ANDed
1509 ;; with another predicate P. This does not have the same trapping behavior
1510 ;; as predicating the comparison itself on P, but it's a legitimate fold,
1511 ;; since we can drop any potentially-trapping operations whose results
1514 ;; Split the instruction into its preferred form (below) at the earliest
1515 ;; opportunity, in order to get rid of the redundant operand 1.
1516 (define_insn_and_split "*fcm<cmp_op><mode>_and_combine"
1517 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1520 [(match_operand:<VPRED> 1)
1522 (match_operand:SVE_F 2 "register_operand" "w, w")
1523 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))]
1525 (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))]
1537 (define_insn_and_split "*fcmuo<mode>_and_combine"
1538 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
1541 [(match_operand:<VPRED> 1)
1543 (match_operand:SVE_F 2 "register_operand" "w")
1544 (match_operand:SVE_F 3 "register_operand" "w"))]
1546 (match_operand:<VPRED> 4 "register_operand" "Upl")))]
1558 ;; Unpredicated floating-point comparisons, with the results ANDed
1559 ;; with another predicate. This is a valid fold for the same reasons
1561 (define_insn "*fcm<cmp_op><mode>_and"
1562 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1565 (match_operand:SVE_F 2 "register_operand" "w, w")
1566 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))
1567 (match_operand:<VPRED> 1 "register_operand" "Upl, Upl")))]
1570 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
1571 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1574 (define_insn "*fcmuo<mode>_and"
1575 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
1578 (match_operand:SVE_F 2 "register_operand" "w")
1579 (match_operand:SVE_F 3 "register_operand" "w"))
1580 (match_operand:<VPRED> 1 "register_operand" "Upl")))]
1582 "fcmuo\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1585 ;; Predicated floating-point comparisons. We don't need a version
1586 ;; of this for unordered comparisons.
1587 (define_insn "*pred_fcm<cmp_op><mode>"
1588 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
1590 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1591 (match_operand:SVE_F 2 "register_operand" "w, w")
1592 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
1596 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
1597 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
1600 ;; vcond_mask operand order: true, false, mask
1601 ;; UNSPEC_SEL operand order: mask, true, false (as for VEC_COND_EXPR)
1602 ;; SEL operand order: mask, true, false
1603 (define_insn "vcond_mask_<mode><vpred>"
1604 [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
1606 [(match_operand:<VPRED> 3 "register_operand" "Upa")
1607 (match_operand:SVE_ALL 1 "register_operand" "w")
1608 (match_operand:SVE_ALL 2 "register_operand" "w")]
1611 "sel\t%0.<Vetype>, %3, %1.<Vetype>, %2.<Vetype>"
1614 ;; Selects between a duplicated immediate and zero.
1615 (define_insn "aarch64_sve_dup<mode>_const"
1616 [(set (match_operand:SVE_I 0 "register_operand" "=w")
1618 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1619 (match_operand:SVE_I 2 "aarch64_sve_dup_immediate")
1620 (match_operand:SVE_I 3 "aarch64_simd_imm_zero")]
1623 "mov\t%0.<Vetype>, %1/z, #%2"
1626 ;; Integer (signed) vcond. Don't enforce an immediate range here, since it
1627 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
1628 (define_expand "vcond<mode><v_int_equiv>"
1629 [(set (match_operand:SVE_ALL 0 "register_operand")
1630 (if_then_else:SVE_ALL
1631 (match_operator 3 "comparison_operator"
1632 [(match_operand:<V_INT_EQUIV> 4 "register_operand")
1633 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")])
1634 (match_operand:SVE_ALL 1 "register_operand")
1635 (match_operand:SVE_ALL 2 "register_operand")))]
1638 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands);
1643 ;; Integer vcondu. Don't enforce an immediate range here, since it
1644 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
1645 (define_expand "vcondu<mode><v_int_equiv>"
1646 [(set (match_operand:SVE_ALL 0 "register_operand")
1647 (if_then_else:SVE_ALL
1648 (match_operator 3 "comparison_operator"
1649 [(match_operand:<V_INT_EQUIV> 4 "register_operand")
1650 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")])
1651 (match_operand:SVE_ALL 1 "register_operand")
1652 (match_operand:SVE_ALL 2 "register_operand")))]
1655 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands);
1660 ;; Floating-point vcond. All comparisons except FCMUO allow a zero
1661 ;; operand; aarch64_expand_sve_vcond handles the case of an FCMUO
1663 (define_expand "vcond<mode><v_fp_equiv>"
1664 [(set (match_operand:SVE_SD 0 "register_operand")
1665 (if_then_else:SVE_SD
1666 (match_operator 3 "comparison_operator"
1667 [(match_operand:<V_FP_EQUIV> 4 "register_operand")
1668 (match_operand:<V_FP_EQUIV> 5 "aarch64_simd_reg_or_zero")])
1669 (match_operand:SVE_SD 1 "register_operand")
1670 (match_operand:SVE_SD 2 "register_operand")))]
1673 aarch64_expand_sve_vcond (<MODE>mode, <V_FP_EQUIV>mode, operands);
1678 ;; Signed integer comparisons. Don't enforce an immediate range here, since
1679 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
1681 (define_expand "vec_cmp<mode><vpred>"
1683 [(set (match_operand:<VPRED> 0 "register_operand")
1684 (match_operator:<VPRED> 1 "comparison_operator"
1685 [(match_operand:SVE_I 2 "register_operand")
1686 (match_operand:SVE_I 3 "nonmemory_operand")]))
1687 (clobber (reg:CC CC_REGNUM))])]
1690 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
1691 operands[2], operands[3]);
1696 ;; Unsigned integer comparisons. Don't enforce an immediate range here, since
1697 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
1699 (define_expand "vec_cmpu<mode><vpred>"
1701 [(set (match_operand:<VPRED> 0 "register_operand")
1702 (match_operator:<VPRED> 1 "comparison_operator"
1703 [(match_operand:SVE_I 2 "register_operand")
1704 (match_operand:SVE_I 3 "nonmemory_operand")]))
1705 (clobber (reg:CC CC_REGNUM))])]
1708 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
1709 operands[2], operands[3]);
1714 ;; Floating-point comparisons. All comparisons except FCMUO allow a zero
1715 ;; operand; aarch64_expand_sve_vec_cmp_float handles the case of an FCMUO
1717 (define_expand "vec_cmp<mode><vpred>"
1718 [(set (match_operand:<VPRED> 0 "register_operand")
1719 (match_operator:<VPRED> 1 "comparison_operator"
1720 [(match_operand:SVE_F 2 "register_operand")
1721 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]))]
1724 aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
1725 operands[2], operands[3], false);
1730 ;; Branch based on predicate equality or inequality.
1731 (define_expand "cbranch<mode>4"
1734 (match_operator 0 "aarch64_equality_operator"
1735 [(match_operand:PRED_ALL 1 "register_operand")
1736 (match_operand:PRED_ALL 2 "aarch64_simd_reg_or_zero")])
1737 (label_ref (match_operand 3 ""))
1741 rtx ptrue = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));
1743 if (operands[2] == CONST0_RTX (<MODE>mode))
1747 pred = gen_reg_rtx (<MODE>mode);
1748 emit_insn (gen_pred_xor<mode>3 (pred, ptrue, operands[1],
1751 emit_insn (gen_ptest_ptrue<mode> (ptrue, pred));
1752 operands[1] = gen_rtx_REG (CCmode, CC_REGNUM);
1753 operands[2] = const0_rtx;
1757 ;; Unpredicated integer MIN/MAX.
1758 (define_expand "<su><maxmin><mode>3"
1759 [(set (match_operand:SVE_I 0 "register_operand")
1762 (MAXMIN:SVE_I (match_operand:SVE_I 1 "register_operand")
1763 (match_operand:SVE_I 2 "register_operand"))]
1764 UNSPEC_MERGE_PTRUE))]
1767 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1771 ;; Integer MIN/MAX predicated with a PTRUE.
1772 (define_insn "*<su><maxmin><mode>3"
1773 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
1775 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1776 (MAXMIN:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w")
1777 (match_operand:SVE_I 3 "register_operand" "w, w"))]
1778 UNSPEC_MERGE_PTRUE))]
1781 <su><maxmin>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1782 movprfx\t%0, %2\;<su><maxmin>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1783 [(set_attr "movprfx" "*,yes")]
1786 ;; Unpredicated floating-point MIN/MAX.
1787 (define_expand "<su><maxmin><mode>3"
1788 [(set (match_operand:SVE_F 0 "register_operand")
1791 (FMAXMIN:SVE_F (match_operand:SVE_F 1 "register_operand")
1792 (match_operand:SVE_F 2 "register_operand"))]
1793 UNSPEC_MERGE_PTRUE))]
1796 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1800 ;; Floating-point MIN/MAX predicated with a PTRUE.
1801 (define_insn "*<su><maxmin><mode>3"
1802 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
1804 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1805 (FMAXMIN:SVE_F (match_operand:SVE_F 2 "register_operand" "%0, w")
1806 (match_operand:SVE_F 3 "register_operand" "w, w"))]
1807 UNSPEC_MERGE_PTRUE))]
1810 f<maxmin>nm\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1811 movprfx\t%0, %2\;f<maxmin>nm\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1812 [(set_attr "movprfx" "*,yes")]
1815 ;; Unpredicated fmin/fmax.
1816 (define_expand "<maxmin_uns><mode>3"
1817 [(set (match_operand:SVE_F 0 "register_operand")
1820 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand")
1821 (match_operand:SVE_F 2 "register_operand")]
1823 UNSPEC_MERGE_PTRUE))]
1826 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
1830 ;; fmin/fmax predicated with a PTRUE.
1831 (define_insn "*<maxmin_uns><mode>3"
1832 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
1834 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1835 (unspec:SVE_F [(match_operand:SVE_F 2 "register_operand" "%0, w")
1836 (match_operand:SVE_F 3 "register_operand" "w, w")]
1838 UNSPEC_MERGE_PTRUE))]
1841 <maxmin_uns_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1842 movprfx\t%0, %2\;<maxmin_uns_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1843 [(set_attr "movprfx" "*,yes")]
1846 ;; Predicated integer operations with select.
1847 (define_expand "cond_<optab><mode>"
1848 [(set (match_operand:SVE_I 0 "register_operand")
1850 [(match_operand:<VPRED> 1 "register_operand")
1851 (SVE_INT_BINARY:SVE_I
1852 (match_operand:SVE_I 2 "register_operand")
1853 (match_operand:SVE_I 3 "register_operand"))
1854 (match_operand:SVE_I 4 "aarch64_simd_reg_or_zero")]
1859 (define_expand "cond_<optab><mode>"
1860 [(set (match_operand:SVE_SDI 0 "register_operand")
1862 [(match_operand:<VPRED> 1 "register_operand")
1863 (SVE_INT_BINARY_SD:SVE_SDI
1864 (match_operand:SVE_SDI 2 "register_operand")
1865 (match_operand:SVE_SDI 3 "register_operand"))
1866 (match_operand:SVE_SDI 4 "aarch64_simd_reg_or_zero")]
1871 ;; Predicated integer operations with select matching the output operand.
1872 (define_insn "*cond_<optab><mode>_0"
1873 [(set (match_operand:SVE_I 0 "register_operand" "+w, w, ?&w")
1875 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
1876 (SVE_INT_BINARY:SVE_I
1877 (match_operand:SVE_I 2 "register_operand" "0, w, w")
1878 (match_operand:SVE_I 3 "register_operand" "w, 0, w"))
1883 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1884 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
1885 movprfx\t%0, %1/m, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1886 [(set_attr "movprfx" "*,*,yes")]
1889 (define_insn "*cond_<optab><mode>_0"
1890 [(set (match_operand:SVE_SDI 0 "register_operand" "+w, w, ?&w")
1892 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
1893 (SVE_INT_BINARY_SD:SVE_SDI
1894 (match_operand:SVE_SDI 2 "register_operand" "0, w, w")
1895 (match_operand:SVE_SDI 3 "register_operand" "w, 0, w"))
1900 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1901 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
1902 movprfx\t%0, %1/m, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1903 [(set_attr "movprfx" "*,*,yes")]
1906 ;; Predicated integer operations with select matching the first operand.
1907 (define_insn "*cond_<optab><mode>_2"
1908 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
1910 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1911 (SVE_INT_BINARY:SVE_I
1912 (match_operand:SVE_I 2 "register_operand" "0, w")
1913 (match_operand:SVE_I 3 "register_operand" "w, w"))
1918 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1919 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1920 [(set_attr "movprfx" "*,yes")]
1923 (define_insn "*cond_<optab><mode>_2"
1924 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, ?&w")
1926 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1927 (SVE_INT_BINARY_SD:SVE_SDI
1928 (match_operand:SVE_SDI 2 "register_operand" "0, w")
1929 (match_operand:SVE_SDI 3 "register_operand" "w, w"))
1934 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
1935 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1936 [(set_attr "movprfx" "*,yes")]
1939 ;; Predicated integer operations with select matching the second operand.
1940 (define_insn "*cond_<optab><mode>_3"
1941 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
1943 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1944 (SVE_INT_BINARY:SVE_I
1945 (match_operand:SVE_I 2 "register_operand" "w, w")
1946 (match_operand:SVE_I 3 "register_operand" "0, w"))
1951 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
1952 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
1953 [(set_attr "movprfx" "*,yes")]
1956 (define_insn "*cond_<optab><mode>_3"
1957 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, ?&w")
1959 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
1960 (SVE_INT_BINARY_SD:SVE_SDI
1961 (match_operand:SVE_SDI 2 "register_operand" "w, w")
1962 (match_operand:SVE_SDI 3 "register_operand" "0, w"))
1967 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
1968 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
1969 [(set_attr "movprfx" "*,yes")]
1972 ;; Predicated integer operations with select matching zero.
1973 (define_insn "*cond_<optab><mode>_z"
1974 [(set (match_operand:SVE_I 0 "register_operand" "=&w")
1976 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1977 (SVE_INT_BINARY:SVE_I
1978 (match_operand:SVE_I 2 "register_operand" "w")
1979 (match_operand:SVE_I 3 "register_operand" "w"))
1980 (match_operand:SVE_I 4 "aarch64_simd_imm_zero")]
1983 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1984 [(set_attr "movprfx" "yes")]
1987 (define_insn "*cond_<optab><mode>_z"
1988 [(set (match_operand:SVE_SDI 0 "register_operand" "=&w")
1990 [(match_operand:<VPRED> 1 "register_operand" "Upl")
1991 (SVE_INT_BINARY_SD:SVE_SDI
1992 (match_operand:SVE_SDI 2 "register_operand" "w")
1993 (match_operand:SVE_SDI 3 "register_operand" "w"))
1994 (match_operand:SVE_SDI 4 "aarch64_simd_imm_zero")]
1997 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
1998 [(set_attr "movprfx" "yes")]
2001 ;; Synthetic predications with select unmatched.
2002 (define_insn "*cond_<optab><mode>_any"
2003 [(set (match_operand:SVE_I 0 "register_operand" "=&w")
2005 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2006 (SVE_INT_BINARY:SVE_I
2007 (match_operand:SVE_I 2 "register_operand" "w")
2008 (match_operand:SVE_I 3 "register_operand" "w"))
2009 (match_operand:SVE_I 4 "register_operand" "w")]
2015 (define_insn "*cond_<optab><mode>_any"
2016 [(set (match_operand:SVE_SDI 0 "register_operand" "=&w")
2018 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2019 (SVE_INT_BINARY_SD:SVE_I
2020 (match_operand:SVE_SDI 2 "register_operand" "w")
2021 (match_operand:SVE_SDI 3 "register_operand" "w"))
2022 (match_operand:SVE_SDI 4 "register_operand" "w")]
2029 [(set (match_operand:SVE_I 0 "register_operand")
2031 [(match_operand:<VPRED> 1 "register_operand")
2032 (match_operator:SVE_I 5 "aarch64_sve_any_binary_operator"
2033 [(match_operand:SVE_I 2 "register_operand")
2034 (match_operand:SVE_I 3 "register_operand")])
2035 (match_operand:SVE_I 4 "register_operand")]
2037 "TARGET_SVE && reload_completed
2038 && !(rtx_equal_p (operands[0], operands[4])
2039 || rtx_equal_p (operands[2], operands[4])
2040 || rtx_equal_p (operands[3], operands[4]))"
2041 ; Not matchable by any one insn or movprfx insn. We need a separate select.
2043 (unspec:SVE_I [(match_dup 1) (match_dup 2) (match_dup 4)]
2048 (match_op_dup 5 [(match_dup 0) (match_dup 3)])
2053 ;; Set operand 0 to the last active element in operand 3, or to tied
2054 ;; operand 1 if no elements are active.
2055 (define_insn "fold_extract_last_<mode>"
2056 [(set (match_operand:<VEL> 0 "register_operand" "=r, w")
2058 [(match_operand:<VEL> 1 "register_operand" "0, 0")
2059 (match_operand:<VPRED> 2 "register_operand" "Upl, Upl")
2060 (match_operand:SVE_ALL 3 "register_operand" "w, w")]
2064 clastb\t%<vwcore>0, %2, %<vwcore>0, %3.<Vetype>
2065 clastb\t%<vw>0, %2, %<vw>0, %3.<Vetype>"
2068 ;; Unpredicated integer add reduction.
2069 (define_expand "reduc_plus_scal_<mode>"
2070 [(set (match_operand:<VEL> 0 "register_operand")
2071 (unspec:<VEL> [(match_dup 2)
2072 (match_operand:SVE_I 1 "register_operand")]
2076 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2080 ;; Predicated integer add reduction. The result is always 64-bits.
2081 (define_insn "*reduc_plus_scal_<mode>"
2082 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2083 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2084 (match_operand:SVE_I 2 "register_operand" "w")]
2087 "uaddv\t%d0, %1, %2.<Vetype>"
2090 ;; Unpredicated floating-point add reduction.
2091 (define_expand "reduc_plus_scal_<mode>"
2092 [(set (match_operand:<VEL> 0 "register_operand")
2093 (unspec:<VEL> [(match_dup 2)
2094 (match_operand:SVE_F 1 "register_operand")]
2098 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2102 ;; Predicated floating-point add reduction.
2103 (define_insn "*reduc_plus_scal_<mode>"
2104 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2105 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2106 (match_operand:SVE_F 2 "register_operand" "w")]
2109 "faddv\t%<Vetype>0, %1, %2.<Vetype>"
2112 ;; Unpredicated integer MIN/MAX reduction.
2113 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2114 [(set (match_operand:<VEL> 0 "register_operand")
2115 (unspec:<VEL> [(match_dup 2)
2116 (match_operand:SVE_I 1 "register_operand")]
2120 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2124 ;; Predicated integer MIN/MAX reduction.
2125 (define_insn "*reduc_<maxmin_uns>_scal_<mode>"
2126 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2127 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2128 (match_operand:SVE_I 2 "register_operand" "w")]
2131 "<maxmin_uns_op>v\t%<Vetype>0, %1, %2.<Vetype>"
2134 ;; Unpredicated floating-point MIN/MAX reduction.
2135 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2136 [(set (match_operand:<VEL> 0 "register_operand")
2137 (unspec:<VEL> [(match_dup 2)
2138 (match_operand:SVE_F 1 "register_operand")]
2142 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2146 ;; Predicated floating-point MIN/MAX reduction.
2147 (define_insn "*reduc_<maxmin_uns>_scal_<mode>"
2148 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2149 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2150 (match_operand:SVE_F 2 "register_operand" "w")]
2153 "<maxmin_uns_op>v\t%<Vetype>0, %1, %2.<Vetype>"
2156 (define_expand "reduc_<optab>_scal_<mode>"
2157 [(set (match_operand:<VEL> 0 "register_operand")
2158 (unspec:<VEL> [(match_dup 2)
2159 (match_operand:SVE_I 1 "register_operand")]
2163 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2167 (define_insn "*reduc_<optab>_scal_<mode>"
2168 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2169 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2170 (match_operand:SVE_I 2 "register_operand" "w")]
2173 "<bit_reduc_op>\t%<Vetype>0, %1, %2.<Vetype>"
2176 ;; Unpredicated in-order FP reductions.
2177 (define_expand "fold_left_plus_<mode>"
2178 [(set (match_operand:<VEL> 0 "register_operand")
2179 (unspec:<VEL> [(match_dup 3)
2180 (match_operand:<VEL> 1 "register_operand")
2181 (match_operand:SVE_F 2 "register_operand")]
2185 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2189 ;; In-order FP reductions predicated with PTRUE.
2190 (define_insn "*fold_left_plus_<mode>"
2191 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2192 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
2193 (match_operand:<VEL> 2 "register_operand" "0")
2194 (match_operand:SVE_F 3 "register_operand" "w")]
2197 "fadda\t%<Vetype>0, %1, %<Vetype>0, %3.<Vetype>"
2200 ;; Predicated form of the above in-order reduction.
2201 (define_insn "*pred_fold_left_plus_<mode>"
2202 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2204 [(match_operand:<VEL> 1 "register_operand" "0")
2206 [(match_operand:<VPRED> 2 "register_operand" "Upl")
2207 (match_operand:SVE_F 3 "register_operand" "w")
2208 (match_operand:SVE_F 4 "aarch64_simd_imm_zero")]
2212 "fadda\t%<Vetype>0, %2, %<Vetype>0, %3.<Vetype>"
2215 ;; Unpredicated floating-point addition.
2216 (define_expand "add<mode>3"
2217 [(set (match_operand:SVE_F 0 "register_operand")
2221 (match_operand:SVE_F 1 "register_operand")
2222 (match_operand:SVE_F 2 "aarch64_sve_float_arith_with_sub_operand"))]
2223 UNSPEC_MERGE_PTRUE))]
2226 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2230 ;; Floating-point addition predicated with a PTRUE.
2231 (define_insn_and_split "*add<mode>3"
2232 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w")
2234 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2236 (match_operand:SVE_F 2 "register_operand" "%0, 0, w")
2237 (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "vsA, vsN, w"))]
2238 UNSPEC_MERGE_PTRUE))]
2241 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
2242 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
2244 ; Split the unpredicated form after reload, so that we don't have
2245 ; the unnecessary PTRUE.
2246 "&& reload_completed
2247 && register_operand (operands[3], <MODE>mode)"
2248 [(set (match_dup 0) (plus:SVE_F (match_dup 2) (match_dup 3)))]
2251 ;; Unpredicated floating-point subtraction.
2252 (define_expand "sub<mode>3"
2253 [(set (match_operand:SVE_F 0 "register_operand")
2257 (match_operand:SVE_F 1 "aarch64_sve_float_arith_operand")
2258 (match_operand:SVE_F 2 "register_operand"))]
2259 UNSPEC_MERGE_PTRUE))]
2262 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2266 ;; Floating-point subtraction predicated with a PTRUE.
2267 (define_insn_and_split "*sub<mode>3"
2268 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w, w")
2270 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
2272 (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand" "0, 0, vsA, w")
2273 (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "vsA, vsN, 0, w"))]
2274 UNSPEC_MERGE_PTRUE))]
2276 && (register_operand (operands[2], <MODE>mode)
2277 || register_operand (operands[3], <MODE>mode))"
2279 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
2280 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
2281 fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
2283 ; Split the unpredicated form after reload, so that we don't have
2284 ; the unnecessary PTRUE.
2285 "&& reload_completed
2286 && register_operand (operands[2], <MODE>mode)
2287 && register_operand (operands[3], <MODE>mode)"
2288 [(set (match_dup 0) (minus:SVE_F (match_dup 2) (match_dup 3)))]
2291 ;; Unpredicated floating-point multiplication.
2292 (define_expand "mul<mode>3"
2293 [(set (match_operand:SVE_F 0 "register_operand")
2297 (match_operand:SVE_F 1 "register_operand")
2298 (match_operand:SVE_F 2 "aarch64_sve_float_mul_operand"))]
2299 UNSPEC_MERGE_PTRUE))]
2302 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2306 ;; Floating-point multiplication predicated with a PTRUE.
2307 (define_insn_and_split "*mul<mode>3"
2308 [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
2310 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2312 (match_operand:SVE_F 2 "register_operand" "%0, w")
2313 (match_operand:SVE_F 3 "aarch64_sve_float_mul_operand" "vsM, w"))]
2314 UNSPEC_MERGE_PTRUE))]
2317 fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
2319 ; Split the unpredicated form after reload, so that we don't have
2320 ; the unnecessary PTRUE.
2321 "&& reload_completed
2322 && register_operand (operands[3], <MODE>mode)"
2323 [(set (match_dup 0) (mult:SVE_F (match_dup 2) (match_dup 3)))]
2326 ;; Unpredicated floating-point binary operations (post-RA only).
2327 ;; These are generated by splitting a predicated instruction whose
2328 ;; predicate is unused.
2329 (define_insn "*post_ra_<sve_fp_op><mode>3"
2330 [(set (match_operand:SVE_F 0 "register_operand" "=w")
2331 (SVE_UNPRED_FP_BINARY:SVE_F
2332 (match_operand:SVE_F 1 "register_operand" "w")
2333 (match_operand:SVE_F 2 "register_operand" "w")))]
2334 "TARGET_SVE && reload_completed"
2335 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>")
2337 ;; Unpredicated fma (%0 = (%1 * %2) + %3).
2338 (define_expand "fma<mode>4"
2339 [(set (match_operand:SVE_F 0 "register_operand")
2342 (fma:SVE_F (match_operand:SVE_F 1 "register_operand")
2343 (match_operand:SVE_F 2 "register_operand")
2344 (match_operand:SVE_F 3 "register_operand"))]
2345 UNSPEC_MERGE_PTRUE))]
2348 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2352 ;; fma predicated with a PTRUE.
2353 (define_insn "*fma<mode>4"
2354 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w")
2356 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2357 (fma:SVE_F (match_operand:SVE_F 3 "register_operand" "%0, w, w")
2358 (match_operand:SVE_F 4 "register_operand" "w, w, w")
2359 (match_operand:SVE_F 2 "register_operand" "w, 0, w"))]
2360 UNSPEC_MERGE_PTRUE))]
2363 fmad\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype>
2364 fmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
2365 movprfx\t%0, %2\;fmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
2366 [(set_attr "movprfx" "*,*,yes")]
2369 ;; Unpredicated fnma (%0 = (-%1 * %2) + %3).
2370 (define_expand "fnma<mode>4"
2371 [(set (match_operand:SVE_F 0 "register_operand")
2374 (fma:SVE_F (neg:SVE_F
2375 (match_operand:SVE_F 1 "register_operand"))
2376 (match_operand:SVE_F 2 "register_operand")
2377 (match_operand:SVE_F 3 "register_operand"))]
2378 UNSPEC_MERGE_PTRUE))]
2381 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2385 ;; fnma predicated with a PTRUE.
2386 (define_insn "*fnma<mode>4"
2387 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w")
2389 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2390 (fma:SVE_F (neg:SVE_F
2391 (match_operand:SVE_F 3 "register_operand" "%0, w, w"))
2392 (match_operand:SVE_F 4 "register_operand" "w, w, w")
2393 (match_operand:SVE_F 2 "register_operand" "w, 0, w"))]
2394 UNSPEC_MERGE_PTRUE))]
2397 fmsb\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype>
2398 fmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
2399 movprfx\t%0, %2\;fmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
2400 [(set_attr "movprfx" "*,*,yes")]
2403 ;; Unpredicated fms (%0 = (%1 * %2) - %3).
2404 (define_expand "fms<mode>4"
2405 [(set (match_operand:SVE_F 0 "register_operand")
2408 (fma:SVE_F (match_operand:SVE_F 1 "register_operand")
2409 (match_operand:SVE_F 2 "register_operand")
2411 (match_operand:SVE_F 3 "register_operand")))]
2412 UNSPEC_MERGE_PTRUE))]
2415 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2419 ;; fms predicated with a PTRUE.
2420 (define_insn "*fms<mode>4"
2421 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w")
2423 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2424 (fma:SVE_F (match_operand:SVE_F 3 "register_operand" "%0, w, w")
2425 (match_operand:SVE_F 4 "register_operand" "w, w, w")
2427 (match_operand:SVE_F 2 "register_operand" "w, 0, w")))]
2428 UNSPEC_MERGE_PTRUE))]
2431 fnmsb\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype>
2432 fnmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
2433 movprfx\t%0, %2\;fnmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
2434 [(set_attr "movprfx" "*,*,yes")]
2437 ;; Unpredicated fnms (%0 = (-%1 * %2) - %3).
2438 (define_expand "fnms<mode>4"
2439 [(set (match_operand:SVE_F 0 "register_operand")
2442 (fma:SVE_F (neg:SVE_F
2443 (match_operand:SVE_F 1 "register_operand"))
2444 (match_operand:SVE_F 2 "register_operand")
2446 (match_operand:SVE_F 3 "register_operand")))]
2447 UNSPEC_MERGE_PTRUE))]
2450 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2454 ;; fnms predicated with a PTRUE.
2455 (define_insn "*fnms<mode>4"
2456 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w")
2458 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2459 (fma:SVE_F (neg:SVE_F
2460 (match_operand:SVE_F 3 "register_operand" "%0, w, w"))
2461 (match_operand:SVE_F 4 "register_operand" "w, w, w")
2463 (match_operand:SVE_F 2 "register_operand" "w, 0, w")))]
2464 UNSPEC_MERGE_PTRUE))]
2467 fnmad\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype>
2468 fnmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
2469 movprfx\t%0, %2\;fnmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
2470 [(set_attr "movprfx" "*,*,yes")]
2473 ;; Unpredicated floating-point division.
2474 (define_expand "div<mode>3"
2475 [(set (match_operand:SVE_F 0 "register_operand")
2478 (div:SVE_F (match_operand:SVE_F 1 "register_operand")
2479 (match_operand:SVE_F 2 "register_operand"))]
2480 UNSPEC_MERGE_PTRUE))]
2483 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2487 ;; Floating-point division predicated with a PTRUE.
2488 (define_insn "*div<mode>3"
2489 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w")
2491 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2492 (div:SVE_F (match_operand:SVE_F 2 "register_operand" "0, w, w")
2493 (match_operand:SVE_F 3 "register_operand" "w, 0, w"))]
2494 UNSPEC_MERGE_PTRUE))]
2497 fdiv\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
2498 fdivr\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
2499 movprfx\t%0, %2\;fdiv\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
2500 [(set_attr "movprfx" "*,*,yes")]
2503 ;; Unpredicated FNEG, FABS and FSQRT.
2504 (define_expand "<optab><mode>2"
2505 [(set (match_operand:SVE_F 0 "register_operand")
2508 (SVE_FP_UNARY:SVE_F (match_operand:SVE_F 1 "register_operand"))]
2509 UNSPEC_MERGE_PTRUE))]
2512 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2516 ;; FNEG, FABS and FSQRT predicated with a PTRUE.
2517 (define_insn "*<optab><mode>2"
2518 [(set (match_operand:SVE_F 0 "register_operand" "=w")
2520 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2521 (SVE_FP_UNARY:SVE_F (match_operand:SVE_F 2 "register_operand" "w"))]
2522 UNSPEC_MERGE_PTRUE))]
2524 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2527 ;; Unpredicated FRINTy.
2528 (define_expand "<frint_pattern><mode>2"
2529 [(set (match_operand:SVE_F 0 "register_operand")
2532 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand")]
2534 UNSPEC_MERGE_PTRUE))]
2537 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2541 ;; FRINTy predicated with a PTRUE.
2542 (define_insn "*<frint_pattern><mode>2"
2543 [(set (match_operand:SVE_F 0 "register_operand" "=w")
2545 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2546 (unspec:SVE_F [(match_operand:SVE_F 2 "register_operand" "w")]
2548 UNSPEC_MERGE_PTRUE))]
2550 "frint<frint_suffix>\t%0.<Vetype>, %1/m, %2.<Vetype>"
2553 ;; Unpredicated conversion of floats to integers of the same size (HF to HI,
2554 ;; SF to SI or DF to DI).
2555 (define_expand "<fix_trunc_optab><mode><v_int_equiv>2"
2556 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
2557 (unspec:<V_INT_EQUIV>
2559 (FIXUORS:<V_INT_EQUIV>
2560 (match_operand:SVE_F 1 "register_operand"))]
2561 UNSPEC_MERGE_PTRUE))]
2564 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2568 ;; Conversion of SF to DI, SI or HI, predicated with a PTRUE.
2569 (define_insn "*<fix_trunc_optab>v16hsf<mode>2"
2570 [(set (match_operand:SVE_HSDI 0 "register_operand" "=w")
2572 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2574 (match_operand:VNx8HF 2 "register_operand" "w"))]
2575 UNSPEC_MERGE_PTRUE))]
2577 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.h"
2580 ;; Conversion of SF to DI or SI, predicated with a PTRUE.
2581 (define_insn "*<fix_trunc_optab>vnx4sf<mode>2"
2582 [(set (match_operand:SVE_SDI 0 "register_operand" "=w")
2584 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2586 (match_operand:VNx4SF 2 "register_operand" "w"))]
2587 UNSPEC_MERGE_PTRUE))]
2589 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.s"
2592 ;; Conversion of DF to DI or SI, predicated with a PTRUE.
2593 (define_insn "*<fix_trunc_optab>vnx2df<mode>2"
2594 [(set (match_operand:SVE_SDI 0 "register_operand" "=w")
2596 [(match_operand:VNx2BI 1 "register_operand" "Upl")
2598 (match_operand:VNx2DF 2 "register_operand" "w"))]
2599 UNSPEC_MERGE_PTRUE))]
2601 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.d"
2604 ;; Unpredicated conversion of integers to floats of the same size
2605 ;; (HI to HF, SI to SF or DI to DF).
2606 (define_expand "<optab><v_int_equiv><mode>2"
2607 [(set (match_operand:SVE_F 0 "register_operand")
2611 (match_operand:<V_INT_EQUIV> 1 "register_operand"))]
2612 UNSPEC_MERGE_PTRUE))]
2615 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
2619 ;; Conversion of DI, SI or HI to the same number of HFs, predicated
2621 (define_insn "*<optab><mode>vnx8hf2"
2622 [(set (match_operand:VNx8HF 0 "register_operand" "=w")
2624 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2626 (match_operand:SVE_HSDI 2 "register_operand" "w"))]
2627 UNSPEC_MERGE_PTRUE))]
2629 "<su_optab>cvtf\t%0.h, %1/m, %2.<Vetype>"
2632 ;; Conversion of DI or SI to the same number of SFs, predicated with a PTRUE.
2633 (define_insn "*<optab><mode>vnx4sf2"
2634 [(set (match_operand:VNx4SF 0 "register_operand" "=w")
2636 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2638 (match_operand:SVE_SDI 2 "register_operand" "w"))]
2639 UNSPEC_MERGE_PTRUE))]
2641 "<su_optab>cvtf\t%0.s, %1/m, %2.<Vetype>"
2644 ;; Conversion of DI or SI to DF, predicated with a PTRUE.
2645 (define_insn "aarch64_sve_<optab><mode>vnx2df2"
2646 [(set (match_operand:VNx2DF 0 "register_operand" "=w")
2648 [(match_operand:VNx2BI 1 "register_operand" "Upl")
2650 (match_operand:SVE_SDI 2 "register_operand" "w"))]
2651 UNSPEC_MERGE_PTRUE))]
2653 "<su_optab>cvtf\t%0.d, %1/m, %2.<Vetype>"
2656 ;; Conversion of DFs to the same number of SFs, or SFs to the same number
2658 (define_insn "*trunc<Vwide><mode>2"
2659 [(set (match_operand:SVE_HSF 0 "register_operand" "=w")
2661 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl")
2663 [(match_operand:<VWIDE> 2 "register_operand" "w")]
2664 UNSPEC_FLOAT_CONVERT)]
2665 UNSPEC_MERGE_PTRUE))]
2667 "fcvt\t%0.<Vetype>, %1/m, %2.<Vewtype>"
2670 ;; Conversion of SFs to the same number of DFs, or HFs to the same number
2672 (define_insn "aarch64_sve_extend<mode><Vwide>2"
2673 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2675 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl")
2677 [(match_operand:SVE_HSF 2 "register_operand" "w")]
2678 UNSPEC_FLOAT_CONVERT)]
2679 UNSPEC_MERGE_PTRUE))]
2681 "fcvt\t%0.<Vewtype>, %1/m, %2.<Vetype>"
2684 ;; Unpack the low or high half of a predicate, where "high" refers to
2685 ;; the low-numbered lanes for big-endian and the high-numbered lanes
2686 ;; for little-endian.
2687 (define_expand "vec_unpack<su>_<perm_hilo>_<mode>"
2688 [(match_operand:<VWIDE> 0 "register_operand")
2689 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand")]
2693 emit_insn ((<hi_lanes_optab>
2694 ? gen_aarch64_sve_punpkhi_<PRED_BHS:mode>
2695 : gen_aarch64_sve_punpklo_<PRED_BHS:mode>)
2696 (operands[0], operands[1]));
2701 ;; PUNPKHI and PUNPKLO.
2702 (define_insn "aarch64_sve_punpk<perm_hilo>_<mode>"
2703 [(set (match_operand:<VWIDE> 0 "register_operand" "=Upa")
2704 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand" "Upa")]
2707 "punpk<perm_hilo>\t%0.h, %1.b"
2710 ;; Unpack the low or high half of a vector, where "high" refers to
2711 ;; the low-numbered lanes for big-endian and the high-numbered lanes
2712 ;; for little-endian.
2713 (define_expand "vec_unpack<su>_<perm_hilo>_<SVE_BHSI:mode>"
2714 [(match_operand:<VWIDE> 0 "register_operand")
2715 (unspec:<VWIDE> [(match_operand:SVE_BHSI 1 "register_operand")] UNPACK)]
2718 emit_insn ((<hi_lanes_optab>
2719 ? gen_aarch64_sve_<su>unpkhi_<SVE_BHSI:mode>
2720 : gen_aarch64_sve_<su>unpklo_<SVE_BHSI:mode>)
2721 (operands[0], operands[1]));
2726 ;; SUNPKHI, UUNPKHI, SUNPKLO and UUNPKLO.
2727 (define_insn "aarch64_sve_<su>unpk<perm_hilo>_<SVE_BHSI:mode>"
2728 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2729 (unspec:<VWIDE> [(match_operand:SVE_BHSI 1 "register_operand" "w")]
2732 "<su>unpk<perm_hilo>\t%0.<Vewtype>, %1.<Vetype>"
2735 ;; Unpack one half of a VNx4SF to VNx2DF, or one half of a VNx8HF to VNx4SF.
2736 ;; First unpack the source without conversion, then float-convert the
2738 (define_expand "vec_unpacks_<perm_hilo>_<mode>"
2739 [(match_operand:<VWIDE> 0 "register_operand")
2740 (unspec:SVE_HSF [(match_operand:SVE_HSF 1 "register_operand")]
2744 /* Use ZIP to do the unpack, since we don't care about the upper halves
2745 and since it has the nice property of not needing any subregs.
2746 If using UUNPK* turns out to be preferable, we could model it as
2747 a ZIP whose first operand is zero. */
2748 rtx temp = gen_reg_rtx (<MODE>mode);
2749 emit_insn ((<hi_lanes_optab>
2750 ? gen_aarch64_sve_zip2<mode>
2751 : gen_aarch64_sve_zip1<mode>)
2752 (temp, operands[1], operands[1]));
2753 rtx ptrue = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode));
2754 emit_insn (gen_aarch64_sve_extend<mode><Vwide>2 (operands[0],
2760 ;; Unpack one half of a VNx4SI to VNx2DF. First unpack from VNx4SI
2761 ;; to VNx2DI, reinterpret the VNx2DI as a VNx4SI, then convert the
2762 ;; unpacked VNx4SI to VNx2DF.
2763 (define_expand "vec_unpack<su_optab>_float_<perm_hilo>_vnx4si"
2764 [(match_operand:VNx2DF 0 "register_operand")
2766 (unspec:VNx2DI [(match_operand:VNx4SI 1 "register_operand")]
2770 /* Use ZIP to do the unpack, since we don't care about the upper halves
2771 and since it has the nice property of not needing any subregs.
2772 If using UUNPK* turns out to be preferable, we could model it as
2773 a ZIP whose first operand is zero. */
2774 rtx temp = gen_reg_rtx (VNx4SImode);
2775 emit_insn ((<hi_lanes_optab>
2776 ? gen_aarch64_sve_zip2vnx4si
2777 : gen_aarch64_sve_zip1vnx4si)
2778 (temp, operands[1], operands[1]));
2779 rtx ptrue = force_reg (VNx2BImode, CONSTM1_RTX (VNx2BImode));
2780 emit_insn (gen_aarch64_sve_<FLOATUORS:optab>vnx4sivnx2df2 (operands[0],
2786 ;; Predicate pack. Use UZP1 on the narrower type, which discards
2787 ;; the high part of each wide element.
2788 (define_insn "vec_pack_trunc_<Vwide>"
2789 [(set (match_operand:PRED_BHS 0 "register_operand" "=Upa")
2791 [(match_operand:<VWIDE> 1 "register_operand" "Upa")
2792 (match_operand:<VWIDE> 2 "register_operand" "Upa")]
2795 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
2798 ;; Integer pack. Use UZP1 on the narrower type, which discards
2799 ;; the high part of each wide element.
2800 (define_insn "vec_pack_trunc_<Vwide>"
2801 [(set (match_operand:SVE_BHSI 0 "register_operand" "=w")
2803 [(match_operand:<VWIDE> 1 "register_operand" "w")
2804 (match_operand:<VWIDE> 2 "register_operand" "w")]
2807 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
2810 ;; Convert two vectors of DF to SF, or two vectors of SF to HF, and pack
2811 ;; the results into a single vector.
2812 (define_expand "vec_pack_trunc_<Vwide>"
2816 (unspec:SVE_HSF [(match_operand:<VWIDE> 1 "register_operand")]
2817 UNSPEC_FLOAT_CONVERT)]
2818 UNSPEC_MERGE_PTRUE))
2822 (unspec:SVE_HSF [(match_operand:<VWIDE> 2 "register_operand")]
2823 UNSPEC_FLOAT_CONVERT)]
2824 UNSPEC_MERGE_PTRUE))
2825 (set (match_operand:SVE_HSF 0 "register_operand")
2826 (unspec:SVE_HSF [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
2829 operands[3] = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode));
2830 operands[4] = gen_reg_rtx (<MODE>mode);
2831 operands[5] = gen_reg_rtx (<MODE>mode);
2835 ;; Convert two vectors of DF to SI and pack the results into a single vector.
2836 (define_expand "vec_pack_<su>fix_trunc_vnx2df"
2840 (FIXUORS:VNx4SI (match_operand:VNx2DF 1 "register_operand"))]
2841 UNSPEC_MERGE_PTRUE))
2845 (FIXUORS:VNx4SI (match_operand:VNx2DF 2 "register_operand"))]
2846 UNSPEC_MERGE_PTRUE))
2847 (set (match_operand:VNx4SI 0 "register_operand")
2848 (unspec:VNx4SI [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
2851 operands[3] = force_reg (VNx2BImode, CONSTM1_RTX (VNx2BImode));
2852 operands[4] = gen_reg_rtx (VNx4SImode);
2853 operands[5] = gen_reg_rtx (VNx4SImode);
2857 ;; Predicated floating-point operations with select.
2858 (define_expand "cond_<optab><mode>"
2859 [(set (match_operand:SVE_F 0 "register_operand")
2861 [(match_operand:<VPRED> 1 "register_operand")
2863 [(match_operand:SVE_F 2 "register_operand")
2864 (match_operand:SVE_F 3 "register_operand")]
2866 (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
2871 ;; Predicated floating-point operations with select matching output.
2872 (define_insn "*cond_<optab><mode>_0"
2873 [(set (match_operand:SVE_F 0 "register_operand" "+w, w, ?&w")
2875 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
2877 [(match_operand:SVE_F 2 "register_operand" "0, w, w")
2878 (match_operand:SVE_F 3 "register_operand" "w, 0, w")]
2884 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
2885 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
2886 movprfx\t%0, %1/m, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
2887 [(set_attr "movprfx" "*,*,yes")]
2890 ;; Predicated floating-point operations with select matching first operand.
2891 (define_insn "*cond_<optab><mode>_2"
2892 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
2894 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2896 [(match_operand:SVE_F 2 "register_operand" "0, w")
2897 (match_operand:SVE_F 3 "register_operand" "w, w")]
2903 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
2904 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
2905 [(set_attr "movprfx" "*,yes")]
2908 ;; Predicated floating-point operations with select matching second operand.
2909 (define_insn "*cond_<optab><mode>_3"
2910 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
2912 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2914 [(match_operand:SVE_F 2 "register_operand" "w, w")
2915 (match_operand:SVE_F 3 "register_operand" "0, w")]
2921 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>
2922 movprfx\t%0, %3\;<sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
2923 [(set_attr "movprfx" "*,yes")]
2926 ;; Predicated floating-point operations with select matching zero.
2927 (define_insn "*cond_<optab><mode>_z"
2928 [(set (match_operand:SVE_F 0 "register_operand" "=&w")
2930 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2932 [(match_operand:SVE_F 2 "register_operand" "w")
2933 (match_operand:SVE_F 3 "register_operand" "w")]
2935 (match_operand:SVE_F 4 "aarch64_simd_imm_zero")]
2938 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
2939 [(set_attr "movprfx" "yes")]
2942 ;; Synthetic predication of floating-point operations with select unmatched.
2943 (define_insn_and_split "*cond_<optab><mode>_any"
2944 [(set (match_operand:SVE_F 0 "register_operand" "=&w")
2946 [(match_operand:<VPRED> 1 "register_operand" "Upl")
2948 [(match_operand:SVE_F 2 "register_operand" "w")
2949 (match_operand:SVE_F 3 "register_operand" "w")]
2951 (match_operand:SVE_F 4 "register_operand" "w")]
2955 "&& reload_completed
2956 && !(rtx_equal_p (operands[0], operands[4])
2957 || rtx_equal_p (operands[2], operands[4])
2958 || rtx_equal_p (operands[3], operands[4]))"
2959 ; Not matchable by any one insn or movprfx insn. We need a separate select.
2961 (unspec:SVE_F [(match_dup 1) (match_dup 2) (match_dup 4)] UNSPEC_SEL))
2965 (unspec:SVE_F [(match_dup 0) (match_dup 3)] SVE_COND_FP_BINARY)
2970 ;; Predicated floating-point ternary operations with select.
2971 (define_expand "cond_<optab><mode>"
2972 [(set (match_operand:SVE_F 0 "register_operand")
2974 [(match_operand:<VPRED> 1 "register_operand")
2976 [(match_operand:SVE_F 2 "register_operand")
2977 (match_operand:SVE_F 3 "register_operand")
2978 (match_operand:SVE_F 4 "register_operand")]
2979 SVE_COND_FP_TERNARY)
2980 (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero")]
2984 /* Swap the multiplication operands if the fallback value is the
2985 second of the two. */
2986 if (rtx_equal_p (operands[3], operands[5]))
2987 std::swap (operands[2], operands[3]);
2990 ;; Predicated floating-point ternary operations using the FMAD-like form.
2991 (define_insn "*cond_<optab><mode>_2"
2992 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
2994 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
2996 [(match_operand:SVE_F 2 "register_operand" "0, w")
2997 (match_operand:SVE_F 3 "register_operand" "w, w")
2998 (match_operand:SVE_F 4 "register_operand" "w, w")]
2999 SVE_COND_FP_TERNARY)
3004 <sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
3005 movprfx\t%0, %2\;<sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>"
3006 [(set_attr "movprfx" "*,yes")]
3009 ;; Predicated floating-point ternary operations using the FMLA-like form.
3010 (define_insn "*cond_<optab><mode>_4"
3011 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w")
3013 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
3015 [(match_operand:SVE_F 2 "register_operand" "w, w")
3016 (match_operand:SVE_F 3 "register_operand" "w, w")
3017 (match_operand:SVE_F 4 "register_operand" "0, w")]
3018 SVE_COND_FP_TERNARY)
3023 <sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
3024 movprfx\t%0, %4\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
3025 [(set_attr "movprfx" "*,yes")]
3028 ;; Predicated floating-point ternary operations in which the value for
3029 ;; inactive lanes is distinct from the other inputs.
3030 (define_insn_and_split "*cond_<optab><mode>_any"
3031 [(set (match_operand:SVE_F 0 "register_operand" "=&w, &w, ?&w")
3033 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
3035 [(match_operand:SVE_F 2 "register_operand" "w, w, w")
3036 (match_operand:SVE_F 3 "register_operand" "w, w, w")
3037 (match_operand:SVE_F 4 "register_operand" "w, w, w")]
3038 SVE_COND_FP_TERNARY)
3039 (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero" "Dz, 0, w")]
3042 && !rtx_equal_p (operands[2], operands[5])
3043 && !rtx_equal_p (operands[3], operands[5])
3044 && !rtx_equal_p (operands[4], operands[5])"
3046 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
3047 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>
3049 "&& reload_completed
3050 && !CONSTANT_P (operands[5])
3051 && !rtx_equal_p (operands[0], operands[5])"
3053 (unspec:SVE_F [(match_dup 1) (match_dup 4) (match_dup 5)] UNSPEC_SEL))
3057 (unspec:SVE_F [(match_dup 2) (match_dup 3) (match_dup 0)]
3058 SVE_COND_FP_TERNARY)
3062 [(set_attr "movprfx" "yes")]
3065 ;; Shift an SVE vector left and insert a scalar into element 0.
3066 (define_insn "vec_shl_insert_<mode>"
3067 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w")
3069 [(match_operand:SVE_ALL 1 "register_operand" "0, 0")
3070 (match_operand:<VEL> 2 "register_operand" "rZ, w")]
3074 insr\t%0.<Vetype>, %<vwcore>2
3075 insr\t%0.<Vetype>, %<Vetype>2"