1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2020 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
91 ;; Defined only to make the DWARF description simpler.
109 (LAST_SAVED_REGNUM 83)
111 ;; "FFR token": a fake register used for representing the scheduling
112 ;; restrictions on FFR-related operations.
114 ;; Scratch register used by stack clash protection to calculate
115 ;; SVE CFA offsets during probing.
116 (STACK_CLASH_SVE_CFA_REGNUM 11)
117 ;; Scratch registers for prologue/epilogue use.
120 ;; A couple of call-clobbered registers that we need to reserve when
121 ;; tracking speculation this is not ABI, so is subject to change.
122 (SPECULATION_SCRATCH_REGNUM 14)
123 (SPECULATION_TRACKER_REGNUM 15)
124 ;; Scratch registers used in frame layout.
132 (define_c_enum "unspec" [
166 UNSPEC_GOTSMALLPIC28K
264 UNSPEC_SPECULATION_TRACKER
266 UNSPEC_TTEST ; Represent transaction test.
271 ;; Represents an SVE-style lane index, in which the indexing applies
272 ;; within the containing 128-bit block.
273 UNSPEC_SVE_LANE_SELECT
276 UNSPEC_SVE_PREFETCH_GATHER
279 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag.
280 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag.
281 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space.
284 (define_c_enum "unspecv" [
285 UNSPECV_EH_RETURN ; Represent EH_RETURN
286 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
287 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
288 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
289 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
290 UNSPECV_BLOCKAGE ; Represent a blockage
291 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
292 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
293 UNSPECV_BTI_NOARG ; Represent BTI.
294 UNSPECV_BTI_C ; Represent BTI c.
295 UNSPECV_BTI_J ; Represent BTI j.
296 UNSPECV_BTI_JC ; Represent BTI jc.
297 UNSPECV_TSTART ; Represent transaction start.
298 UNSPECV_TCOMMIT ; Represent transaction commit.
299 UNSPECV_TCANCEL ; Represent transaction cancel.
300 UNSPEC_RNDR ; Represent RNDR
301 UNSPEC_RNDRRS ; Represent RNDRRS
305 ;; These constants are used as a const_int in various SVE unspecs
306 ;; to indicate whether the governing predicate is known to be a PTRUE.
308 [; Indicates that the predicate might not be a PTRUE.
309 (SVE_MAYBE_NOT_PTRUE 0)
311 ; Indicates that the predicate is known to be a PTRUE.
312 (SVE_KNOWN_PTRUE 1)])
314 ;; These constants are used as a const_int in predicated SVE FP arithmetic
315 ;; to indicate whether the operation is allowed to make additional lanes
316 ;; active without worrying about the effect on faulting behavior.
318 [; Indicates either that all lanes are active or that the instruction may
319 ; operate on inactive inputs even if doing so could induce a fault.
322 ; Indicates that some lanes might be inactive and that the instruction
323 ; must not operate on inactive inputs if doing so could induce a fault.
326 ;; If further include files are added the defintion of MD_INCLUDES
329 (include "constraints.md")
330 (include "predicates.md")
331 (include "iterators.md")
333 ;; -------------------------------------------------------------------
334 ;; Instruction types and attributes
335 ;; -------------------------------------------------------------------
337 ; The "type" attribute is included here from AArch32 backend to be able
338 ; to share pipeline descriptions.
339 (include "../arm/types.md")
341 ;; It is important to set the fp or simd attributes to yes when a pattern
342 ;; alternative uses the FP or SIMD register files, usually signified by use of
343 ;; the 'w' constraint. This will ensure that the alternative will be
344 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
345 ;; architecture extensions. If all the alternatives in a pattern use the
346 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
349 ;; Attributes of the architecture required to support the instruction (or
350 ;; alternative). This attribute is used to compute attribute "enabled", use type
351 ;; "any" to enable an alternative in all cases.
353 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
355 (define_enum_attr "arch" "arches" (const_string "any"))
357 ;; [For compatibility with Arm in pipeline models]
358 ;; Attribute that specifies whether or not the instruction touches fp
360 ;; Note that this attribute is not used anywhere in either the arm or aarch64
361 ;; backends except in the scheduling description for xgene1. In that
362 ;; scheduling description this attribute is used to subclass the load_4 and
364 (define_attr "fp" "no,yes"
366 (eq_attr "arch" "fp")
368 (const_string "no")))
370 (define_attr "arch_enabled" "no,yes"
373 (eq_attr "arch" "any")
375 (and (eq_attr "arch" "rcpc8_4")
376 (match_test "AARCH64_ISA_RCPC8_4"))
378 (and (eq_attr "arch" "fp")
379 (match_test "TARGET_FLOAT"))
381 (and (eq_attr "arch" "simd")
382 (match_test "TARGET_SIMD"))
384 (and (eq_attr "arch" "fp16")
385 (match_test "TARGET_FP_F16INST"))
387 (and (eq_attr "arch" "sve")
388 (match_test "TARGET_SVE")))
390 (const_string "no")))
392 ;; Attribute that controls whether an alternative is enabled or not.
393 ;; Currently it is only used to disable alternatives which touch fp or simd
394 ;; registers when -mgeneral-regs-only is specified or to require a special
395 ;; architecture support.
396 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
398 ;; Attribute that specifies whether we are dealing with a branch to a
399 ;; label that is far away, i.e. further away than the maximum/minimum
400 ;; representable in a signed 21-bits number.
403 (define_attr "far_branch" "" (const_int 0))
405 ;; Attribute that specifies whether the alternative uses MOVPRFX.
406 (define_attr "movprfx" "no,yes" (const_string "no"))
408 (define_attr "length" ""
409 (cond [(eq_attr "movprfx" "yes")
413 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
414 ;; no predicated insns.
415 (define_attr "predicated" "yes,no" (const_string "no"))
417 ;; Set to true on an insn that requires the speculation tracking state to be
418 ;; in the tracking register before the insn issues. Otherwise the compiler
419 ;; may chose to hold the tracking state encoded in SP.
420 (define_attr "speculation_barrier" "true,false" (const_string "false"))
422 ;; -------------------------------------------------------------------
423 ;; Pipeline descriptions and scheduling
424 ;; -------------------------------------------------------------------
427 (include "aarch64-tune.md")
430 (include "../arm/cortex-a53.md")
431 (include "../arm/cortex-a57.md")
432 (include "../arm/exynos-m1.md")
433 (include "falkor.md")
434 (include "saphira.md")
435 (include "thunderx.md")
436 (include "../arm/xgene1.md")
437 (include "thunderx2t99.md")
438 (include "tsv110.md")
440 ;; -------------------------------------------------------------------
441 ;; Jumps and other miscellaneous insns
442 ;; -------------------------------------------------------------------
444 (define_insn "indirect_jump"
445 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
448 [(set_attr "type" "branch")]
452 [(set (pc) (label_ref (match_operand 0 "" "")))]
455 [(set_attr "type" "branch")]
458 (define_expand "cbranch<mode>4"
459 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
460 [(match_operand:GPI 1 "register_operand")
461 (match_operand:GPI 2 "aarch64_plus_operand")])
462 (label_ref (match_operand 3 "" ""))
466 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
468 operands[2] = const0_rtx;
472 (define_expand "cbranch<mode>4"
473 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
474 [(match_operand:GPF 1 "register_operand")
475 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
476 (label_ref (match_operand 3 "" ""))
480 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
482 operands[2] = const0_rtx;
486 (define_expand "cbranchcc4"
487 [(set (pc) (if_then_else
488 (match_operator 0 "aarch64_comparison_operator"
489 [(match_operand 1 "cc_register")
490 (match_operand 2 "const0_operand")])
491 (label_ref (match_operand 3 "" ""))
496 (define_insn "ccmp<mode>"
497 [(set (match_operand:CC 1 "cc_register" "")
499 (match_operator 4 "aarch64_comparison_operator"
500 [(match_operand 0 "cc_register" "")
503 (match_operand:GPI 2 "register_operand" "r,r,r")
504 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
505 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
508 ccmp\\t%<w>2, %<w>3, %k5, %m4
509 ccmp\\t%<w>2, %3, %k5, %m4
510 ccmn\\t%<w>2, #%n3, %k5, %m4"
511 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
514 (define_insn "fccmp<mode>"
515 [(set (match_operand:CCFP 1 "cc_register" "")
517 (match_operator 4 "aarch64_comparison_operator"
518 [(match_operand 0 "cc_register" "")
521 (match_operand:GPF 2 "register_operand" "w")
522 (match_operand:GPF 3 "register_operand" "w"))
523 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
525 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
526 [(set_attr "type" "fccmp<s>")]
529 (define_insn "fccmpe<mode>"
530 [(set (match_operand:CCFPE 1 "cc_register" "")
532 (match_operator 4 "aarch64_comparison_operator"
533 [(match_operand 0 "cc_register" "")
536 (match_operand:GPF 2 "register_operand" "w")
537 (match_operand:GPF 3 "register_operand" "w"))
538 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
540 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
541 [(set_attr "type" "fccmp<s>")]
544 ;; Expansion of signed mod by a power of 2 using CSNEG.
545 ;; For x0 % n where n is a power of 2 produce:
547 ;; and x0, x0, #(n - 1)
548 ;; and x1, x1, #(n - 1)
549 ;; csneg x0, x0, x1, mi
551 (define_expand "mod<mode>3"
552 [(match_operand:GPI 0 "register_operand")
553 (match_operand:GPI 1 "register_operand")
554 (match_operand:GPI 2 "const_int_operand")]
557 HOST_WIDE_INT val = INTVAL (operands[2]);
560 || exact_log2 (val) <= 0
561 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
564 rtx mask = GEN_INT (val - 1);
566 /* In the special case of x0 % 2 we can do the even shorter:
572 rtx masked = gen_reg_rtx (<MODE>mode);
573 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
574 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
575 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
576 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
580 rtx neg_op = gen_reg_rtx (<MODE>mode);
581 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
583 /* Extract the condition register and mode. */
584 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
585 rtx cc_reg = SET_DEST (cmp);
586 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
588 rtx masked_pos = gen_reg_rtx (<MODE>mode);
589 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
591 rtx masked_neg = gen_reg_rtx (<MODE>mode);
592 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
594 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
595 masked_neg, masked_pos));
600 (define_insn "condjump"
601 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
602 [(match_operand 1 "cc_register" "") (const_int 0)])
603 (label_ref (match_operand 2 "" ""))
607 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
608 but the "." is required for SVE conditions. */
609 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
610 if (get_attr_length (insn) == 8)
611 return aarch64_gen_far_branch (operands, 2, "Lbcond",
612 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
614 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
616 [(set_attr "type" "branch")
618 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
619 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
622 (set (attr "far_branch")
623 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
624 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
629 ;; For a 24-bit immediate CST we can optimize the compare for equality
630 ;; and branch sequence from:
632 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
636 ;; sub x0, x1, #(CST & 0xfff000)
637 ;; subs x0, x0, #(CST & 0x000fff)
639 (define_insn_and_split "*compare_condjump<GPI:mode>"
640 [(set (pc) (if_then_else (EQL
641 (match_operand:GPI 0 "register_operand" "r")
642 (match_operand:GPI 1 "aarch64_imm24" "n"))
643 (label_ref:P (match_operand 2 "" ""))
645 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
646 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
647 && !reload_completed"
652 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
653 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
654 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
655 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
656 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
657 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
658 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
660 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
665 (define_expand "casesi"
666 [(match_operand:SI 0 "register_operand") ; Index
667 (match_operand:SI 1 "const_int_operand") ; Lower bound
668 (match_operand:SI 2 "const_int_operand") ; Total range
669 (match_operand:DI 3 "" "") ; Table label
670 (match_operand:DI 4 "" "")] ; Out of range label
673 if (operands[1] != const0_rtx)
675 rtx reg = gen_reg_rtx (SImode);
677 /* Canonical RTL says that if you have:
681 then this should be emitted as:
685 The use of trunc_int_for_mode ensures that the resulting
686 constant can be represented in SImode, this is important
687 for the corner case where operand[1] is INT_MIN. */
689 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
691 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
692 (operands[1], SImode))
693 operands[1] = force_reg (SImode, operands[1]);
694 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
698 if (!aarch64_plus_operand (operands[2], SImode))
699 operands[2] = force_reg (SImode, operands[2]);
700 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
702 operands[0], operands[2], operands[4]));
704 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
706 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
708 operands[2] = gen_rtx_MEM (DImode, operands[2]);
709 MEM_READONLY_P (operands[2]) = 1;
710 MEM_NOTRAP_P (operands[2]) = 1;
711 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
716 (define_expand "casesi_dispatch"
718 [(set (pc) (match_operand:DI 0 ""))
719 (clobber (reg:CC CC_REGNUM))
720 (clobber (match_scratch:DI 2))
721 (clobber (match_scratch:DI 3))
722 (use (label_ref:DI (match_operand 1 "")))])]
725 (define_insn "*casesi_dispatch"
728 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
729 (match_operand:SI 1 "register_operand" "r")]
731 (clobber (reg:CC CC_REGNUM))
732 (clobber (match_scratch:DI 3 "=r"))
733 (clobber (match_scratch:DI 4 "=r"))
734 (use (label_ref:DI (match_operand 2 "" "")))])]
737 return aarch64_output_casesi (operands);
739 [(set_attr "length" "16")
740 (set_attr "type" "branch")]
744 [(unspec[(const_int 0)] UNSPEC_NOP)]
747 [(set_attr "type" "no_insn")]
750 (define_insn "prefetch"
751 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
752 (match_operand:QI 1 "const_int_operand" "")
753 (match_operand:QI 2 "const_int_operand" ""))]
756 const char * pftype[2][4] =
758 {"prfm\\tPLDL1STRM, %0",
759 "prfm\\tPLDL3KEEP, %0",
760 "prfm\\tPLDL2KEEP, %0",
761 "prfm\\tPLDL1KEEP, %0"},
762 {"prfm\\tPSTL1STRM, %0",
763 "prfm\\tPSTL3KEEP, %0",
764 "prfm\\tPSTL2KEEP, %0",
765 "prfm\\tPSTL1KEEP, %0"},
768 int locality = INTVAL (operands[2]);
770 gcc_assert (IN_RANGE (locality, 0, 3));
772 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
773 the address into a DImode MEM so that aarch64_print_operand knows
775 operands[0] = gen_rtx_MEM (DImode, operands[0]);
776 return pftype[INTVAL(operands[1])][locality];
778 [(set_attr "type" "load_4")]
782 [(trap_if (const_int 1) (const_int 8))]
785 [(set_attr "type" "trap")])
787 (define_expand "prologue"
788 [(clobber (const_int 0))]
791 aarch64_expand_prologue ();
796 (define_expand "epilogue"
797 [(clobber (const_int 0))]
800 aarch64_expand_epilogue (false);
805 (define_expand "sibcall_epilogue"
806 [(clobber (const_int 0))]
809 aarch64_expand_epilogue (true);
814 (define_insn "*do_return"
818 if (aarch64_return_address_signing_enabled ()
820 && !crtl->calls_eh_return)
822 if (aarch64_ra_sign_key == AARCH64_KEY_B)
829 [(set_attr "type" "branch")]
832 (define_expand "return"
834 "aarch64_use_return_insn_p ()"
838 (define_insn "simple_return"
842 [(set_attr "type" "branch")]
845 (define_insn "*cb<optab><mode>1"
846 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
848 (label_ref (match_operand 1 "" ""))
850 "!aarch64_track_speculation"
852 if (get_attr_length (insn) == 8)
853 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
855 return "<cbz>\\t%<w>0, %l1";
857 [(set_attr "type" "branch")
859 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
860 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
863 (set (attr "far_branch")
864 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
865 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
870 (define_insn "*tb<optab><mode>1"
871 [(set (pc) (if_then_else
872 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
875 "aarch64_simd_shift_imm_<mode>" "n"))
877 (label_ref (match_operand 2 "" ""))
879 (clobber (reg:CC CC_REGNUM))]
880 "!aarch64_track_speculation"
882 if (get_attr_length (insn) == 8)
884 if (get_attr_far_branch (insn) == 1)
885 return aarch64_gen_far_branch (operands, 2, "Ltb",
886 "<inv_tb>\\t%<w>0, %1, ");
889 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
890 return "tst\t%<w>0, %1\;<bcond>\t%l2";
894 return "<tbz>\t%<w>0, %1, %l2";
896 [(set_attr "type" "branch")
898 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
899 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
902 (set (attr "far_branch")
903 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
904 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
910 (define_insn "*cb<optab><mode>1"
911 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
913 (label_ref (match_operand 1 "" ""))
915 (clobber (reg:CC CC_REGNUM))]
916 "!aarch64_track_speculation"
918 if (get_attr_length (insn) == 8)
920 if (get_attr_far_branch (insn) == 1)
921 return aarch64_gen_far_branch (operands, 1, "Ltb",
922 "<inv_tb>\\t%<w>0, <sizem1>, ");
926 uint64_t val = ((uint64_t) 1)
927 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
928 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
929 output_asm_insn (buf, operands);
930 return "<bcond>\t%l1";
934 return "<tbz>\t%<w>0, <sizem1>, %l1";
936 [(set_attr "type" "branch")
938 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
939 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
942 (set (attr "far_branch")
943 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
944 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
949 ;; -------------------------------------------------------------------
950 ;; Subroutine calls and sibcalls
951 ;; -------------------------------------------------------------------
953 (define_expand "call"
955 [(call (match_operand 0 "memory_operand")
956 (match_operand 1 "general_operand"))
957 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
958 (clobber (reg:DI LR_REGNUM))])]
962 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
967 (define_insn "*call_insn"
968 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
969 (match_operand 1 "" ""))
970 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
971 (clobber (reg:DI LR_REGNUM))]
976 [(set_attr "type" "call, call")]
979 (define_expand "call_value"
981 [(set (match_operand 0 "")
982 (call (match_operand 1 "memory_operand")
983 (match_operand 2 "general_operand")))
984 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
985 (clobber (reg:DI LR_REGNUM))])]
989 aarch64_expand_call (operands[0], operands[1], operands[3], false);
994 (define_insn "*call_value_insn"
995 [(set (match_operand 0 "" "")
996 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
997 (match_operand 2 "" "")))
998 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
999 (clobber (reg:DI LR_REGNUM))]
1004 [(set_attr "type" "call, call")]
1007 (define_expand "sibcall"
1009 [(call (match_operand 0 "memory_operand")
1010 (match_operand 1 "general_operand"))
1011 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1015 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
1020 (define_expand "sibcall_value"
1022 [(set (match_operand 0 "")
1023 (call (match_operand 1 "memory_operand")
1024 (match_operand 2 "general_operand")))
1025 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1029 aarch64_expand_call (operands[0], operands[1], operands[3], true);
1034 (define_insn "*sibcall_insn"
1035 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
1036 (match_operand 1 ""))
1037 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1039 "SIBLING_CALL_P (insn)"
1043 [(set_attr "type" "branch, branch")]
1046 (define_insn "*sibcall_value_insn"
1047 [(set (match_operand 0 "")
1049 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1050 (match_operand 2 "")))
1051 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1053 "SIBLING_CALL_P (insn)"
1057 [(set_attr "type" "branch, branch")]
1060 ;; Call subroutine returning any type.
1062 (define_expand "untyped_call"
1063 [(parallel [(call (match_operand 0 "")
1065 (match_operand 1 "")
1066 (match_operand 2 "")])]
1071 /* Untyped calls always use the default ABI. It's only possible to use
1072 ABI variants if we know the type of the target function. */
1073 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
1075 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1077 rtx set = XVECEXP (operands[2], 0, i);
1078 emit_move_insn (SET_DEST (set), SET_SRC (set));
1081 /* The optimizer does not know that the call sets the function value
1082 registers we stored in the result block. We avoid problems by
1083 claiming that all hard registers are used and clobbered at this
1085 emit_insn (gen_blockage ());
1089 ;; -------------------------------------------------------------------
1091 ;; -------------------------------------------------------------------
1093 (define_expand "mov<mode>"
1094 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1095 (match_operand:SHORT 1 "general_operand"))]
1098 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1099 operands[1] = force_reg (<MODE>mode, operands[1]);
1101 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1103 aarch64_expand_mov_immediate (operands[0], operands[1]);
1109 (define_insn "*mov<mode>_aarch64"
1110 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1111 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1112 "(register_operand (operands[0], <MODE>mode)
1113 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1115 switch (which_alternative)
1118 return "mov\t%w0, %w1";
1120 return "mov\t%w0, %1";
1122 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1125 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1127 return "ldr<size>\t%w0, %1";
1129 return "ldr\t%<size>0, %1";
1131 return "str<size>\t%w1, %0";
1133 return "str\t%<size>1, %0";
1135 return "umov\t%w0, %1.<v>[0]";
1137 return "dup\t%0.<Vallxd>, %w1";
1139 return "dup\t%<Vetype>0, %1.<v>[0]";
1144 ;; The "mov_imm" type for CNT is just a placeholder.
1145 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1146 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1147 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1150 (define_expand "mov<mode>"
1151 [(set (match_operand:GPI 0 "nonimmediate_operand")
1152 (match_operand:GPI 1 "general_operand"))]
1155 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1156 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
1157 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1160 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1161 operands[1] = force_reg (<MODE>mode, operands[1]);
1163 /* FIXME: RR we still need to fix up what we are doing with
1164 symbol_refs and other types of constants. */
1165 if (CONSTANT_P (operands[1])
1166 && !CONST_INT_P (operands[1]))
1168 aarch64_expand_mov_immediate (operands[0], operands[1]);
1174 (define_insn_and_split "*movsi_aarch64"
1175 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
1176 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1177 "(register_operand (operands[0], SImode)
1178 || aarch64_reg_or_zero (operands[1], SImode))"
1185 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1195 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1196 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1197 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1200 aarch64_expand_mov_immediate (operands[0], operands[1]);
1203 ;; The "mov_imm" type for CNT is just a placeholder.
1204 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1205 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1206 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1209 (define_insn_and_split "*movdi_aarch64"
1210 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1211 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1212 "(register_operand (operands[0], DImode)
1213 || aarch64_reg_or_zero (operands[1], DImode))"
1221 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1231 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1232 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1233 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1236 aarch64_expand_mov_immediate (operands[0], operands[1]);
1239 ;; The "mov_imm" type for CNTD is just a placeholder.
1240 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1241 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1243 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1246 (define_insn "insv_imm<mode>"
1247 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1249 (match_operand:GPI 1 "const_int_operand" "n"))
1250 (match_operand:GPI 2 "const_int_operand" "n"))]
1251 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1252 && UINTVAL (operands[1]) % 16 == 0"
1253 "movk\\t%<w>0, %X2, lsl %1"
1254 [(set_attr "type" "mov_imm")]
1257 (define_expand "movti"
1258 [(set (match_operand:TI 0 "nonimmediate_operand")
1259 (match_operand:TI 1 "general_operand"))]
1262 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1263 operands[1] = force_reg (TImode, operands[1]);
1265 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1267 emit_move_insn (gen_lowpart (DImode, operands[0]),
1268 gen_lowpart (DImode, operands[1]));
1269 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1275 (define_insn "*movti_aarch64"
1276 [(set (match_operand:TI 0
1277 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1279 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1280 "(register_operand (operands[0], TImode)
1281 || aarch64_reg_or_zero (operands[1], TImode))"
1286 mov\\t%0.16b, %1.16b
1292 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1293 load_16,store_16,store_16,\
1295 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1296 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1299 ;; Split a TImode register-register or register-immediate move into
1300 ;; its component DImode pieces, taking care to handle overlapping
1301 ;; source and dest registers.
1303 [(set (match_operand:TI 0 "register_operand" "")
1304 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1305 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1308 aarch64_split_128bit_move (operands[0], operands[1]);
1312 (define_expand "mov<mode>"
1313 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1314 (match_operand:GPF_TF_F16 1 "general_operand"))]
1319 aarch64_err_no_fpadvsimd (<MODE>mode);
1323 if (GET_CODE (operands[0]) == MEM
1324 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1325 && aarch64_float_const_zero_rtx_p (operands[1])))
1326 operands[1] = force_reg (<MODE>mode, operands[1]);
1330 (define_insn "*movhf_aarch64"
1331 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1332 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1333 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1334 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1340 mov\\t%0.h[0], %1.h[0]
1342 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1348 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1349 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1350 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1353 (define_insn "*movsf_aarch64"
1354 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1355 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1356 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1357 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1364 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1371 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1372 f_loads,f_stores,load_4,store_4,mov_reg,\
1374 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1377 (define_insn "*movdf_aarch64"
1378 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1379 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1380 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1381 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1388 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1395 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1396 f_loadd,f_stored,load_8,store_8,mov_reg,\
1398 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1402 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1403 (match_operand:GPF_HF 1 "general_operand"))]
1404 "can_create_pseudo_p ()
1405 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1406 && !aarch64_float_const_representable_p (operands[1])
1407 && aarch64_float_const_rtx_p (operands[1])"
1410 unsigned HOST_WIDE_INT ival;
1411 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1414 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1415 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1416 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1421 (define_insn "*movtf_aarch64"
1422 [(set (match_operand:TF 0
1423 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1425 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1426 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1427 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1429 mov\\t%0.16b, %1.16b
1440 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1441 f_loadd,f_stored,load_16,store_16,store_16")
1442 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1443 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1447 [(set (match_operand:TF 0 "register_operand" "")
1448 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1449 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1452 aarch64_split_128bit_move (operands[0], operands[1]);
1459 ;; 2 is size of copy in bytes
1462 (define_expand "cpymemdi"
1463 [(match_operand:BLK 0 "memory_operand")
1464 (match_operand:BLK 1 "memory_operand")
1465 (match_operand:DI 2 "immediate_operand")
1466 (match_operand:DI 3 "immediate_operand")]
1469 if (aarch64_expand_cpymem (operands))
1475 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1476 ;; fairly lax checking on the second memory operation.
1477 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1478 [(set (match_operand:SX 0 "register_operand" "=r,w")
1479 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1480 (set (match_operand:SX2 2 "register_operand" "=r,w")
1481 (match_operand:SX2 3 "memory_operand" "m,m"))]
1482 "rtx_equal_p (XEXP (operands[3], 0),
1483 plus_constant (Pmode,
1484 XEXP (operands[1], 0),
1485 GET_MODE_SIZE (<SX:MODE>mode)))"
1489 [(set_attr "type" "load_8,neon_load1_2reg")
1490 (set_attr "arch" "*,fp")]
1493 ;; Storing different modes that can still be merged
1494 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1495 [(set (match_operand:DX 0 "register_operand" "=r,w")
1496 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1497 (set (match_operand:DX2 2 "register_operand" "=r,w")
1498 (match_operand:DX2 3 "memory_operand" "m,m"))]
1499 "rtx_equal_p (XEXP (operands[3], 0),
1500 plus_constant (Pmode,
1501 XEXP (operands[1], 0),
1502 GET_MODE_SIZE (<DX:MODE>mode)))"
1506 [(set_attr "type" "load_16,neon_load1_2reg")
1507 (set_attr "arch" "*,fp")]
1510 (define_insn "load_pair_dw_tftf"
1511 [(set (match_operand:TF 0 "register_operand" "=w")
1512 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1513 (set (match_operand:TF 2 "register_operand" "=w")
1514 (match_operand:TF 3 "memory_operand" "m"))]
1516 && rtx_equal_p (XEXP (operands[3], 0),
1517 plus_constant (Pmode,
1518 XEXP (operands[1], 0),
1519 GET_MODE_SIZE (TFmode)))"
1520 "ldp\\t%q0, %q2, %1"
1521 [(set_attr "type" "neon_ldp_q")
1522 (set_attr "fp" "yes")]
1525 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1526 ;; fairly lax checking on the second memory operation.
1527 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1528 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1529 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1530 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1531 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1532 "rtx_equal_p (XEXP (operands[2], 0),
1533 plus_constant (Pmode,
1534 XEXP (operands[0], 0),
1535 GET_MODE_SIZE (<SX:MODE>mode)))"
1539 [(set_attr "type" "store_8,neon_store1_2reg")
1540 (set_attr "arch" "*,fp")]
1543 ;; Storing different modes that can still be merged
1544 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1545 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1546 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1547 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1548 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1549 "rtx_equal_p (XEXP (operands[2], 0),
1550 plus_constant (Pmode,
1551 XEXP (operands[0], 0),
1552 GET_MODE_SIZE (<DX:MODE>mode)))"
1556 [(set_attr "type" "store_16,neon_store1_2reg")
1557 (set_attr "arch" "*,fp")]
1560 (define_insn "store_pair_dw_tftf"
1561 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1562 (match_operand:TF 1 "register_operand" "w"))
1563 (set (match_operand:TF 2 "memory_operand" "=m")
1564 (match_operand:TF 3 "register_operand" "w"))]
1566 rtx_equal_p (XEXP (operands[2], 0),
1567 plus_constant (Pmode,
1568 XEXP (operands[0], 0),
1569 GET_MODE_SIZE (TFmode)))"
1570 "stp\\t%q1, %q3, %0"
1571 [(set_attr "type" "neon_stp_q")
1572 (set_attr "fp" "yes")]
1575 ;; Load pair with post-index writeback. This is primarily used in function
1577 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1579 [(set (match_operand:P 0 "register_operand" "=k")
1580 (plus:P (match_operand:P 1 "register_operand" "0")
1581 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1582 (set (match_operand:GPI 2 "register_operand" "=r")
1583 (mem:GPI (match_dup 1)))
1584 (set (match_operand:GPI 3 "register_operand" "=r")
1585 (mem:GPI (plus:P (match_dup 1)
1586 (match_operand:P 5 "const_int_operand" "n"))))])]
1587 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1588 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1589 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1592 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1594 [(set (match_operand:P 0 "register_operand" "=k")
1595 (plus:P (match_operand:P 1 "register_operand" "0")
1596 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1597 (set (match_operand:GPF 2 "register_operand" "=w")
1598 (mem:GPF (match_dup 1)))
1599 (set (match_operand:GPF 3 "register_operand" "=w")
1600 (mem:GPF (plus:P (match_dup 1)
1601 (match_operand:P 5 "const_int_operand" "n"))))])]
1602 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1603 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1604 [(set_attr "type" "neon_load1_2reg")]
1607 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1609 [(set (match_operand:P 0 "register_operand" "=k")
1610 (plus:P (match_operand:P 1 "register_operand" "0")
1611 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1612 (set (match_operand:TX 2 "register_operand" "=w")
1613 (mem:TX (match_dup 1)))
1614 (set (match_operand:TX 3 "register_operand" "=w")
1615 (mem:TX (plus:P (match_dup 1)
1616 (match_operand:P 5 "const_int_operand" "n"))))])]
1617 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1618 "ldp\\t%q2, %q3, [%1], %4"
1619 [(set_attr "type" "neon_ldp_q")]
1622 ;; Store pair with pre-index writeback. This is primarily used in function
1624 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1626 [(set (match_operand:P 0 "register_operand" "=&k")
1627 (plus:P (match_operand:P 1 "register_operand" "0")
1628 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1629 (set (mem:GPI (plus:P (match_dup 0)
1631 (match_operand:GPI 2 "register_operand" "r"))
1632 (set (mem:GPI (plus:P (match_dup 0)
1633 (match_operand:P 5 "const_int_operand" "n")))
1634 (match_operand:GPI 3 "register_operand" "r"))])]
1635 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1636 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1637 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1640 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1642 [(set (match_operand:P 0 "register_operand" "=&k")
1643 (plus:P (match_operand:P 1 "register_operand" "0")
1644 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1645 (set (mem:GPF (plus:P (match_dup 0)
1647 (match_operand:GPF 2 "register_operand" "w"))
1648 (set (mem:GPF (plus:P (match_dup 0)
1649 (match_operand:P 5 "const_int_operand" "n")))
1650 (match_operand:GPF 3 "register_operand" "w"))])]
1651 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1652 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1653 [(set_attr "type" "neon_store1_2reg<q>")]
1656 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1658 [(set (match_operand:P 0 "register_operand" "=&k")
1659 (plus:P (match_operand:P 1 "register_operand" "0")
1660 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1661 (set (mem:TX (plus:P (match_dup 0)
1663 (match_operand:TX 2 "register_operand" "w"))
1664 (set (mem:TX (plus:P (match_dup 0)
1665 (match_operand:P 5 "const_int_operand" "n")))
1666 (match_operand:TX 3 "register_operand" "w"))])]
1668 && INTVAL (operands[5])
1669 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1670 "stp\\t%q2, %q3, [%0, %4]!"
1671 [(set_attr "type" "neon_stp_q")]
1674 ;; -------------------------------------------------------------------
1675 ;; Sign/Zero extension
1676 ;; -------------------------------------------------------------------
1678 (define_expand "<optab>sidi2"
1679 [(set (match_operand:DI 0 "register_operand")
1680 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1684 (define_insn "*extendsidi2_aarch64"
1685 [(set (match_operand:DI 0 "register_operand" "=r,r")
1686 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1691 [(set_attr "type" "extend,load_4")]
1694 (define_insn "*load_pair_extendsidi2_aarch64"
1695 [(set (match_operand:DI 0 "register_operand" "=r")
1696 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1697 (set (match_operand:DI 2 "register_operand" "=r")
1698 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1699 "rtx_equal_p (XEXP (operands[3], 0),
1700 plus_constant (Pmode,
1701 XEXP (operands[1], 0),
1702 GET_MODE_SIZE (SImode)))"
1703 "ldpsw\\t%0, %2, %1"
1704 [(set_attr "type" "load_8")]
1707 (define_insn "*zero_extendsidi2_aarch64"
1708 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1709 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1718 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1719 (set_attr "arch" "*,*,fp,fp,fp,fp")]
1722 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1723 [(set (match_operand:DI 0 "register_operand" "=r,w")
1724 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1725 (set (match_operand:DI 2 "register_operand" "=r,w")
1726 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1727 "rtx_equal_p (XEXP (operands[3], 0),
1728 plus_constant (Pmode,
1729 XEXP (operands[1], 0),
1730 GET_MODE_SIZE (SImode)))"
1734 [(set_attr "type" "load_8,neon_load1_2reg")
1735 (set_attr "arch" "*,fp")]
1738 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1739 [(set (match_operand:GPI 0 "register_operand")
1740 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1744 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1745 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1746 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1749 sxt<SHORT:size>\t%<GPI:w>0, %w1
1750 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1751 [(set_attr "type" "extend,load_4")]
1754 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1755 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1756 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1759 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1760 ldr<SHORT:size>\t%w0, %1
1761 ldr\t%<SHORT:size>0, %1"
1762 [(set_attr "type" "logic_imm,load_4,f_loads")
1763 (set_attr "arch" "*,*,fp")]
1766 (define_expand "<optab>qihi2"
1767 [(set (match_operand:HI 0 "register_operand")
1768 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1772 (define_insn "*extendqihi2_aarch64"
1773 [(set (match_operand:HI 0 "register_operand" "=r,r")
1774 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1779 [(set_attr "type" "extend,load_4")]
1782 (define_insn "*zero_extendqihi2_aarch64"
1783 [(set (match_operand:HI 0 "register_operand" "=r,r")
1784 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1789 [(set_attr "type" "logic_imm,load_4")]
1792 ;; -------------------------------------------------------------------
1793 ;; Simple arithmetic
1794 ;; -------------------------------------------------------------------
1796 (define_expand "add<mode>3"
1798 (match_operand:GPI 0 "register_operand")
1799 (plus:GPI (match_operand:GPI 1 "register_operand")
1800 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1803 /* If operands[1] is a subreg extract the inner RTX. */
1804 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1806 /* If the constant is too large for a single instruction and isn't frame
1807 based, split off the immediate so it is available for CSE. */
1808 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1809 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1810 && can_create_pseudo_p ()
1812 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1813 operands[2] = force_reg (<MODE>mode, operands[2]);
1814 /* Expand polynomial additions now if the destination is the stack
1815 pointer, since we don't want to use that as a temporary. */
1816 else if (operands[0] == stack_pointer_rtx
1817 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1819 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1820 operands[2], NULL_RTX, NULL_RTX);
1825 (define_insn "*add<mode>3_aarch64"
1827 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1829 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1830 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1833 add\\t%<w>0, %<w>1, %2
1834 add\\t%<w>0, %<w>1, %<w>2
1835 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1836 sub\\t%<w>0, %<w>1, #%n2
1838 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1839 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1840 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1841 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1842 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1845 ;; zero_extend version of above
1846 (define_insn "*addsi3_aarch64_uxtw"
1848 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1850 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1851 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1856 sub\\t%w0, %w1, #%n2
1858 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1861 ;; If there's a free register, and we can load the constant with a
1862 ;; single instruction, do so. This has a chance to improve scheduling.
1864 [(match_scratch:GPI 3 "r")
1865 (set (match_operand:GPI 0 "register_operand")
1867 (match_operand:GPI 1 "register_operand")
1868 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1869 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1870 [(set (match_dup 3) (match_dup 2))
1871 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1875 [(match_scratch:SI 3 "r")
1876 (set (match_operand:DI 0 "register_operand")
1879 (match_operand:SI 1 "register_operand")
1880 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1881 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1882 [(set (match_dup 3) (match_dup 2))
1883 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1886 ;; After peephole2 has had a chance to run, split any remaining long
1887 ;; additions into two add immediates.
1889 [(set (match_operand:GPI 0 "register_operand")
1891 (match_operand:GPI 1 "register_operand")
1892 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1893 "epilogue_completed"
1894 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1895 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1897 HOST_WIDE_INT i = INTVAL (operands[2]);
1898 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1899 operands[3] = GEN_INT (i - s);
1900 operands[4] = GEN_INT (s);
1904 ;; Match addition of polynomial offsets that require one temporary, for which
1905 ;; we can use the early-clobbered destination register. This is a separate
1906 ;; pattern so that the early clobber doesn't affect register allocation
1907 ;; for other forms of addition. However, we still need to provide an
1908 ;; all-register alternative, in case the offset goes out of range after
1909 ;; elimination. For completeness we might as well provide all GPR-based
1910 ;; alternatives from the main pattern.
1912 ;; We don't have a pattern for additions requiring two temporaries since at
1913 ;; present LRA doesn't allow new scratches to be added during elimination.
1914 ;; Such offsets should be rare anyway.
1916 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1917 ;; here would go away. We could just handle all polynomial constants in
1919 (define_insn_and_split "*add<mode>3_poly_1"
1921 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1923 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1924 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1925 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1927 add\\t%<w>0, %<w>1, %2
1928 add\\t%<w>0, %<w>1, %<w>2
1929 sub\\t%<w>0, %<w>1, #%n2
1931 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1932 * return aarch64_output_sve_addvl_addpl (operands[2]);
1934 "&& epilogue_completed
1935 && !reg_overlap_mentioned_p (operands[0], operands[1])
1936 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1939 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1940 operands[2], operands[0], NULL_RTX);
1943 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1944 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1948 [(set (match_operand:DI 0 "register_operand")
1951 (match_operand:SI 1 "register_operand")
1952 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1953 "epilogue_completed"
1954 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1955 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1957 HOST_WIDE_INT i = INTVAL (operands[2]);
1958 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1959 operands[3] = GEN_INT (i - s);
1960 operands[4] = GEN_INT (s);
1961 operands[5] = gen_lowpart (SImode, operands[0]);
1965 (define_expand "addv<mode>4"
1966 [(match_operand:GPI 0 "register_operand")
1967 (match_operand:GPI 1 "register_operand")
1968 (match_operand:GPI 2 "aarch64_plus_operand")
1969 (label_ref (match_operand 3 "" ""))]
1972 if (CONST_INT_P (operands[2]))
1973 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1976 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1977 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1982 (define_expand "uaddv<mode>4"
1983 [(match_operand:GPI 0 "register_operand")
1984 (match_operand:GPI 1 "register_operand")
1985 (match_operand:GPI 2 "register_operand")
1986 (label_ref (match_operand 3 "" ""))]
1989 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1990 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1995 (define_expand "addti3"
1996 [(set (match_operand:TI 0 "register_operand")
1997 (plus:TI (match_operand:TI 1 "register_operand")
1998 (match_operand:TI 2 "aarch64_reg_or_imm")))]
2001 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2003 aarch64_addti_scratch_regs (operands[1], operands[2],
2004 &low_dest, &op1_low, &op2_low,
2005 &high_dest, &op1_high, &op2_high);
2007 if (op2_low == const0_rtx)
2010 if (!aarch64_pluslong_operand (op2_high, DImode))
2011 op2_high = force_reg (DImode, op2_high);
2012 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2016 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2017 force_reg (DImode, op2_low)));
2018 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2019 force_reg (DImode, op2_high)));
2022 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2023 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2028 (define_expand "addvti4"
2029 [(match_operand:TI 0 "register_operand")
2030 (match_operand:TI 1 "register_operand")
2031 (match_operand:TI 2 "aarch64_reg_or_imm")
2032 (label_ref (match_operand 3 "" ""))]
2035 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2037 aarch64_addti_scratch_regs (operands[1], operands[2],
2038 &low_dest, &op1_low, &op2_low,
2039 &high_dest, &op1_high, &op2_high);
2041 if (op2_low == const0_rtx)
2044 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2045 force_reg (DImode, op2_high)));
2049 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2050 force_reg (DImode, op2_low)));
2051 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2052 force_reg (DImode, op2_high)));
2055 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2056 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2058 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2062 (define_expand "uaddvti4"
2063 [(match_operand:TI 0 "register_operand")
2064 (match_operand:TI 1 "register_operand")
2065 (match_operand:TI 2 "aarch64_reg_or_imm")
2066 (label_ref (match_operand 3 "" ""))]
2069 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2071 aarch64_addti_scratch_regs (operands[1], operands[2],
2072 &low_dest, &op1_low, &op2_low,
2073 &high_dest, &op1_high, &op2_high);
2075 if (op2_low == const0_rtx)
2078 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2079 force_reg (DImode, op2_high)));
2083 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2084 force_reg (DImode, op2_low)));
2085 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2086 force_reg (DImode, op2_high)));
2089 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2090 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2092 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2096 (define_insn "add<mode>3_compare0"
2097 [(set (reg:CC_NZ CC_REGNUM)
2099 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2100 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2102 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2103 (plus:GPI (match_dup 1) (match_dup 2)))]
2106 adds\\t%<w>0, %<w>1, %<w>2
2107 adds\\t%<w>0, %<w>1, %2
2108 subs\\t%<w>0, %<w>1, #%n2"
2109 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2112 ;; zero_extend version of above
2113 (define_insn "*addsi3_compare0_uxtw"
2114 [(set (reg:CC_NZ CC_REGNUM)
2116 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2117 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2119 (set (match_operand:DI 0 "register_operand" "=r,r,r")
2120 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2123 adds\\t%w0, %w1, %w2
2125 subs\\t%w0, %w1, #%n2"
2126 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2129 (define_insn "*add<mode>3_compareC_cconly"
2130 [(set (reg:CC_C CC_REGNUM)
2133 (match_operand:GPI 0 "register_operand" "r,r,r")
2134 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2141 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2144 (define_insn "add<mode>3_compareC"
2145 [(set (reg:CC_C CC_REGNUM)
2148 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2149 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2151 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2152 (plus:GPI (match_dup 1) (match_dup 2)))]
2155 adds\\t%<w>0, %<w>1, %<w>2
2156 adds\\t%<w>0, %<w>1, %2
2157 subs\\t%<w>0, %<w>1, #%n2"
2158 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2161 (define_insn "*add<mode>3_compareV_cconly_imm"
2162 [(set (reg:CC_V CC_REGNUM)
2165 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2166 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2170 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2171 "INTVAL (operands[1]) == INTVAL (operands[2])"
2175 [(set_attr "type" "alus_imm")]
2178 (define_insn "*add<mode>3_compareV_cconly"
2179 [(set (reg:CC_V CC_REGNUM)
2182 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2183 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2184 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2186 "cmn\\t%<w>0, %<w>1"
2187 [(set_attr "type" "alus_sreg")]
2190 (define_insn "add<mode>3_compareV_imm"
2191 [(set (reg:CC_V CC_REGNUM)
2195 (match_operand:GPI 1 "register_operand" "rk,rk"))
2196 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2198 (plus:GPI (match_dup 1) (match_dup 2)))))
2199 (set (match_operand:GPI 0 "register_operand" "=r,r")
2200 (plus:GPI (match_dup 1) (match_dup 2)))]
2203 adds\\t%<w>0, %<w>1, %<w>2
2204 subs\\t%<w>0, %<w>1, #%n2"
2205 [(set_attr "type" "alus_imm,alus_imm")]
2208 (define_insn "add<mode>3_compareV"
2209 [(set (reg:CC_V CC_REGNUM)
2212 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2213 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2214 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2215 (set (match_operand:GPI 0 "register_operand" "=r")
2216 (plus:GPI (match_dup 1) (match_dup 2)))]
2218 "adds\\t%<w>0, %<w>1, %<w>2"
2219 [(set_attr "type" "alus_sreg")]
2222 (define_insn "*adds_shift_imm_<mode>"
2223 [(set (reg:CC_NZ CC_REGNUM)
2225 (plus:GPI (ASHIFT:GPI
2226 (match_operand:GPI 1 "register_operand" "r")
2227 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2228 (match_operand:GPI 3 "register_operand" "r"))
2230 (set (match_operand:GPI 0 "register_operand" "=r")
2231 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2234 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2235 [(set_attr "type" "alus_shift_imm")]
2238 (define_insn "*subs_shift_imm_<mode>"
2239 [(set (reg:CC_NZ CC_REGNUM)
2241 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2243 (match_operand:GPI 2 "register_operand" "r")
2244 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2246 (set (match_operand:GPI 0 "register_operand" "=r")
2247 (minus:GPI (match_dup 1)
2248 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2250 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2251 [(set_attr "type" "alus_shift_imm")]
2254 (define_insn "*adds_mul_imm_<mode>"
2255 [(set (reg:CC_NZ CC_REGNUM)
2258 (match_operand:GPI 1 "register_operand" "r")
2259 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2260 (match_operand:GPI 3 "register_operand" "r"))
2262 (set (match_operand:GPI 0 "register_operand" "=r")
2263 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2266 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2267 [(set_attr "type" "alus_shift_imm")]
2270 (define_insn "*subs_mul_imm_<mode>"
2271 [(set (reg:CC_NZ CC_REGNUM)
2273 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2275 (match_operand:GPI 2 "register_operand" "r")
2276 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2278 (set (match_operand:GPI 0 "register_operand" "=r")
2279 (minus:GPI (match_dup 1)
2280 (mult:GPI (match_dup 2) (match_dup 3))))]
2282 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2283 [(set_attr "type" "alus_shift_imm")]
2286 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2287 [(set (reg:CC_NZ CC_REGNUM)
2290 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2291 (match_operand:GPI 2 "register_operand" "rk"))
2293 (set (match_operand:GPI 0 "register_operand" "=r")
2294 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2296 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2297 [(set_attr "type" "alus_ext")]
2300 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2301 [(set (reg:CC_NZ CC_REGNUM)
2303 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2305 (match_operand:ALLX 2 "register_operand" "r")))
2307 (set (match_operand:GPI 0 "register_operand" "=r")
2308 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2310 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2311 [(set_attr "type" "alus_ext")]
2314 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2315 [(set (reg:CC_NZ CC_REGNUM)
2317 (plus:GPI (ashift:GPI
2319 (match_operand:ALLX 1 "register_operand" "r"))
2320 (match_operand 2 "aarch64_imm3" "Ui3"))
2321 (match_operand:GPI 3 "register_operand" "rk"))
2323 (set (match_operand:GPI 0 "register_operand" "=rk")
2324 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2328 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2329 [(set_attr "type" "alus_ext")]
2332 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2333 [(set (reg:CC_NZ CC_REGNUM)
2335 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2338 (match_operand:ALLX 2 "register_operand" "r"))
2339 (match_operand 3 "aarch64_imm3" "Ui3")))
2341 (set (match_operand:GPI 0 "register_operand" "=rk")
2342 (minus:GPI (match_dup 1)
2343 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2346 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2347 [(set_attr "type" "alus_ext")]
2350 (define_insn "*adds_<optab><mode>_multp2"
2351 [(set (reg:CC_NZ CC_REGNUM)
2353 (plus:GPI (ANY_EXTRACT:GPI
2354 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2355 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2356 (match_operand 3 "const_int_operand" "n")
2358 (match_operand:GPI 4 "register_operand" "rk"))
2360 (set (match_operand:GPI 0 "register_operand" "=r")
2361 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2365 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2366 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2367 [(set_attr "type" "alus_ext")]
2370 (define_insn "*subs_<optab><mode>_multp2"
2371 [(set (reg:CC_NZ CC_REGNUM)
2373 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2375 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2376 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2377 (match_operand 3 "const_int_operand" "n")
2380 (set (match_operand:GPI 0 "register_operand" "=r")
2381 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2382 (mult:GPI (match_dup 1) (match_dup 2))
2385 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2386 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2387 [(set_attr "type" "alus_ext")]
2390 (define_insn "*add<mode>3nr_compare0"
2391 [(set (reg:CC_NZ CC_REGNUM)
2393 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2394 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2401 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2404 (define_insn "aarch64_sub<mode>_compare0"
2405 [(set (reg:CC_NZ CC_REGNUM)
2407 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2408 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2411 "cmp\\t%<w>0, %<w>1"
2412 [(set_attr "type" "alus_sreg")]
2415 (define_insn "*compare_neg<mode>"
2416 [(set (reg:CC_Z CC_REGNUM)
2418 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2419 (match_operand:GPI 1 "register_operand" "r")))]
2421 "cmn\\t%<w>1, %<w>0"
2422 [(set_attr "type" "alus_sreg")]
2425 (define_insn "*add_<shift>_<mode>"
2426 [(set (match_operand:GPI 0 "register_operand" "=r")
2427 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2428 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2429 (match_operand:GPI 3 "register_operand" "r")))]
2431 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2432 [(set_attr "type" "alu_shift_imm")]
2435 ;; zero_extend version of above
2436 (define_insn "*add_<shift>_si_uxtw"
2437 [(set (match_operand:DI 0 "register_operand" "=r")
2439 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2440 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2441 (match_operand:SI 3 "register_operand" "r"))))]
2443 "add\\t%w0, %w3, %w1, <shift> %2"
2444 [(set_attr "type" "alu_shift_imm")]
2447 (define_insn "*add_mul_imm_<mode>"
2448 [(set (match_operand:GPI 0 "register_operand" "=r")
2449 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2450 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2451 (match_operand:GPI 3 "register_operand" "r")))]
2453 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2454 [(set_attr "type" "alu_shift_imm")]
2457 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2458 [(set (match_operand:GPI 0 "register_operand" "=rk")
2459 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2460 (match_operand:GPI 2 "register_operand" "r")))]
2462 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2463 [(set_attr "type" "alu_ext")]
2466 ;; zero_extend version of above
2467 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2468 [(set (match_operand:DI 0 "register_operand" "=rk")
2470 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2471 (match_operand:GPI 2 "register_operand" "r"))))]
2473 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2474 [(set_attr "type" "alu_ext")]
2477 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2478 [(set (match_operand:GPI 0 "register_operand" "=rk")
2479 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2480 (match_operand:ALLX 1 "register_operand" "r"))
2481 (match_operand 2 "aarch64_imm3" "Ui3"))
2482 (match_operand:GPI 3 "register_operand" "r")))]
2484 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2485 [(set_attr "type" "alu_ext")]
2488 ;; zero_extend version of above
2489 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2490 [(set (match_operand:DI 0 "register_operand" "=rk")
2492 (plus:SI (ashift:SI (ANY_EXTEND:SI
2493 (match_operand:SHORT 1 "register_operand" "r"))
2494 (match_operand 2 "aarch64_imm3" "Ui3"))
2495 (match_operand:SI 3 "register_operand" "r"))))]
2497 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2498 [(set_attr "type" "alu_ext")]
2501 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2502 [(set (match_operand:GPI 0 "register_operand" "=rk")
2503 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2504 (match_operand:ALLX 1 "register_operand" "r"))
2505 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2506 (match_operand:GPI 3 "register_operand" "r")))]
2508 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2509 [(set_attr "type" "alu_ext")]
2512 ;; zero_extend version of above
2513 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2514 [(set (match_operand:DI 0 "register_operand" "=rk")
2515 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2516 (match_operand:SHORT 1 "register_operand" "r"))
2517 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2518 (match_operand:SI 3 "register_operand" "r"))))]
2520 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2521 [(set_attr "type" "alu_ext")]
2524 (define_insn "*add_<optab><mode>_multp2"
2525 [(set (match_operand:GPI 0 "register_operand" "=rk")
2526 (plus:GPI (ANY_EXTRACT:GPI
2527 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2528 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2529 (match_operand 3 "const_int_operand" "n")
2531 (match_operand:GPI 4 "register_operand" "r")))]
2532 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2533 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2534 [(set_attr "type" "alu_ext")]
2537 ;; zero_extend version of above
2538 (define_insn "*add_<optab>si_multp2_uxtw"
2539 [(set (match_operand:DI 0 "register_operand" "=rk")
2541 (plus:SI (ANY_EXTRACT:SI
2542 (mult:SI (match_operand:SI 1 "register_operand" "r")
2543 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2544 (match_operand 3 "const_int_operand" "n")
2546 (match_operand:SI 4 "register_operand" "r"))))]
2547 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2548 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2549 [(set_attr "type" "alu_ext")]
2552 (define_expand "add<mode>3_carryin"
2553 [(set (match_operand:GPI 0 "register_operand")
2556 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2557 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2558 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2563 ;; Note that add with carry with two zero inputs is matched by cset,
2564 ;; and that add with carry with one zero input is matched by cinc.
2566 (define_insn "*add<mode>3_carryin"
2567 [(set (match_operand:GPI 0 "register_operand" "=r")
2570 (match_operand:GPI 3 "aarch64_carry_operation" "")
2571 (match_operand:GPI 1 "register_operand" "r"))
2572 (match_operand:GPI 2 "register_operand" "r")))]
2574 "adc\\t%<w>0, %<w>1, %<w>2"
2575 [(set_attr "type" "adc_reg")]
2578 ;; zero_extend version of above
2579 (define_insn "*addsi3_carryin_uxtw"
2580 [(set (match_operand:DI 0 "register_operand" "=r")
2584 (match_operand:SI 3 "aarch64_carry_operation" "")
2585 (match_operand:SI 1 "register_operand" "r"))
2586 (match_operand:SI 2 "register_operand" "r"))))]
2588 "adc\\t%w0, %w1, %w2"
2589 [(set_attr "type" "adc_reg")]
2592 (define_expand "add<mode>3_carryinC"
2600 (match_operand:GPI 1 "register_operand")))
2602 (match_operand:GPI 2 "register_operand")))
2604 (set (match_operand:GPI 0 "register_operand")
2606 (plus:GPI (match_dup 5) (match_dup 1))
2610 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2611 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2612 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2613 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2614 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2615 << GET_MODE_BITSIZE (<MODE>mode),
2619 (define_insn "*add<mode>3_carryinC_zero"
2620 [(set (reg:CC_ADC CC_REGNUM)
2623 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2624 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2625 (match_operand 4 "const_scalar_int_operand" "")))
2626 (set (match_operand:GPI 0 "register_operand" "=r")
2627 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2629 "rtx_mode_t (operands[4], <DWI>mode)
2630 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2631 "adcs\\t%<w>0, %<w>1, <w>zr"
2632 [(set_attr "type" "adc_reg")]
2635 (define_insn "*add<mode>3_carryinC"
2636 [(set (reg:CC_ADC CC_REGNUM)
2640 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2641 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2642 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2643 (match_operand 5 "const_scalar_int_operand" "")))
2644 (set (match_operand:GPI 0 "register_operand" "=r")
2646 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2649 "rtx_mode_t (operands[5], <DWI>mode)
2650 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2651 "adcs\\t%<w>0, %<w>1, %<w>2"
2652 [(set_attr "type" "adc_reg")]
2655 (define_expand "add<mode>3_carryinV"
2657 [(set (reg:CC_V CC_REGNUM)
2663 (match_operand:GPI 1 "register_operand")))
2665 (match_operand:GPI 2 "register_operand")))
2668 (plus:GPI (match_dup 4) (match_dup 1))
2670 (set (match_operand:GPI 0 "register_operand")
2672 (plus:GPI (match_dup 4) (match_dup 1))
2676 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2677 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2678 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2681 (define_insn "*add<mode>3_carryinV_zero"
2682 [(set (reg:CC_V CC_REGNUM)
2685 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2686 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2689 (match_operand:GPI 3 "aarch64_carry_operation" "")
2691 (set (match_operand:GPI 0 "register_operand" "=r")
2692 (plus:GPI (match_dup 3) (match_dup 1)))]
2694 "adcs\\t%<w>0, %<w>1, <w>zr"
2695 [(set_attr "type" "adc_reg")]
2698 (define_insn "*add<mode>3_carryinV"
2699 [(set (reg:CC_V CC_REGNUM)
2703 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2704 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2705 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2709 (match_operand:GPI 4 "aarch64_carry_operation" "")
2712 (set (match_operand:GPI 0 "register_operand" "=r")
2714 (plus:GPI (match_dup 4) (match_dup 1))
2717 "adcs\\t%<w>0, %<w>1, %<w>2"
2718 [(set_attr "type" "adc_reg")]
2721 (define_insn "*add_uxt<mode>_shift2"
2722 [(set (match_operand:GPI 0 "register_operand" "=rk")
2724 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2725 (match_operand 2 "aarch64_imm3" "Ui3"))
2726 (match_operand 3 "const_int_operand" "n"))
2727 (match_operand:GPI 4 "register_operand" "r")))]
2728 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2730 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2731 INTVAL (operands[3])));
2732 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2733 [(set_attr "type" "alu_ext")]
2736 ;; zero_extend version of above
2737 (define_insn "*add_uxtsi_shift2_uxtw"
2738 [(set (match_operand:DI 0 "register_operand" "=rk")
2741 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2742 (match_operand 2 "aarch64_imm3" "Ui3"))
2743 (match_operand 3 "const_int_operand" "n"))
2744 (match_operand:SI 4 "register_operand" "r"))))]
2745 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2747 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2748 INTVAL (operands[3])));
2749 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2750 [(set_attr "type" "alu_ext")]
2753 (define_insn "*add_uxt<mode>_multp2"
2754 [(set (match_operand:GPI 0 "register_operand" "=rk")
2756 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2757 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2758 (match_operand 3 "const_int_operand" "n"))
2759 (match_operand:GPI 4 "register_operand" "r")))]
2760 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2762 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2763 INTVAL (operands[3])));
2764 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2765 [(set_attr "type" "alu_ext")]
2768 ;; zero_extend version of above
2769 (define_insn "*add_uxtsi_multp2_uxtw"
2770 [(set (match_operand:DI 0 "register_operand" "=rk")
2773 (mult:SI (match_operand:SI 1 "register_operand" "r")
2774 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2775 (match_operand 3 "const_int_operand" "n"))
2776 (match_operand:SI 4 "register_operand" "r"))))]
2777 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2779 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2780 INTVAL (operands[3])));
2781 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2782 [(set_attr "type" "alu_ext")]
2785 (define_insn "subsi3"
2786 [(set (match_operand:SI 0 "register_operand" "=rk")
2787 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2788 (match_operand:SI 2 "register_operand" "r")))]
2790 "sub\\t%w0, %w1, %w2"
2791 [(set_attr "type" "alu_sreg")]
2794 ;; zero_extend version of above
2795 (define_insn "*subsi3_uxtw"
2796 [(set (match_operand:DI 0 "register_operand" "=rk")
2798 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2799 (match_operand:SI 2 "register_operand" "r"))))]
2801 "sub\\t%w0, %w1, %w2"
2802 [(set_attr "type" "alu_sreg")]
2805 (define_insn "subdi3"
2806 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2807 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2808 (match_operand:DI 2 "register_operand" "r,w")))]
2812 sub\\t%d0, %d1, %d2"
2813 [(set_attr "type" "alu_sreg, neon_sub")
2814 (set_attr "arch" "*,simd")]
2817 (define_expand "subv<GPI:mode>4"
2818 [(match_operand:GPI 0 "register_operand")
2819 (match_operand:GPI 1 "register_operand")
2820 (match_operand:GPI 2 "aarch64_plus_operand")
2821 (label_ref (match_operand 3 "" ""))]
2824 if (CONST_INT_P (operands[2]))
2825 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2827 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2828 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2833 (define_insn "subv<GPI:mode>_insn"
2834 [(set (reg:CC_V CC_REGNUM)
2838 (match_operand:GPI 1 "register_operand" "rk")
2839 (match_operand:GPI 2 "register_operand" "r")))
2840 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2841 (sign_extend:<DWI> (match_dup 2)))))
2842 (set (match_operand:GPI 0 "register_operand" "=r")
2843 (minus:GPI (match_dup 1) (match_dup 2)))]
2845 "subs\\t%<w>0, %<w>1, %<w>2"
2846 [(set_attr "type" "alus_sreg")]
2849 (define_insn "subv<GPI:mode>_imm"
2850 [(set (reg:CC_V CC_REGNUM)
2854 (match_operand:GPI 1 "register_operand" "rk,rk")
2855 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2856 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2858 (set (match_operand:GPI 0 "register_operand" "=r,r")
2859 (minus:GPI (match_dup 1) (match_dup 2)))]
2862 subs\\t%<w>0, %<w>1, %2
2863 adds\\t%<w>0, %<w>1, #%n2"
2864 [(set_attr "type" "alus_sreg")]
2867 (define_expand "negv<GPI:mode>3"
2868 [(match_operand:GPI 0 "register_operand")
2869 (match_operand:GPI 1 "register_operand")
2870 (label_ref (match_operand 2 "" ""))]
2873 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2874 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2880 (define_insn "negv<GPI:mode>_insn"
2881 [(set (reg:CC_V CC_REGNUM)
2884 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2885 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2886 (set (match_operand:GPI 0 "register_operand" "=r")
2887 (neg:GPI (match_dup 1)))]
2889 "negs\\t%<w>0, %<w>1"
2890 [(set_attr "type" "alus_sreg")]
2893 (define_insn "negv<GPI:mode>_cmp_only"
2894 [(set (reg:CC_V CC_REGNUM)
2897 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2898 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2900 "negs\\t%<w>zr, %<w>0"
2901 [(set_attr "type" "alus_sreg")]
2904 (define_insn "*cmpv<GPI:mode>_insn"
2905 [(set (reg:CC_V CC_REGNUM)
2908 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2909 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2910 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2911 (sign_extend:<DWI> (match_dup 1)))))]
2917 [(set_attr "type" "alus_sreg")]
2920 (define_expand "usubv<mode>4"
2921 [(match_operand:GPI 0 "register_operand")
2922 (match_operand:GPI 1 "aarch64_reg_or_zero")
2923 (match_operand:GPI 2 "aarch64_reg_or_zero")
2924 (label_ref (match_operand 3 "" ""))]
2927 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2928 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2933 (define_expand "subti3"
2934 [(set (match_operand:TI 0 "register_operand")
2935 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2936 (match_operand:TI 2 "register_operand")))]
2939 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2941 aarch64_subvti_scratch_regs (operands[1], operands[2],
2942 &low_dest, &op1_low, &op2_low,
2943 &high_dest, &op1_high, &op2_high);
2945 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2946 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2948 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2949 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2953 (define_expand "subvti4"
2954 [(match_operand:TI 0 "register_operand")
2955 (match_operand:TI 1 "register_operand")
2956 (match_operand:TI 2 "aarch64_reg_or_imm")
2957 (label_ref (match_operand 3 "" ""))]
2960 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2962 aarch64_subvti_scratch_regs (operands[1], operands[2],
2963 &low_dest, &op1_low, &op2_low,
2964 &high_dest, &op1_high, &op2_high);
2965 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2966 high_dest, op1_high, op2_high, false);
2968 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2972 (define_expand "usubvti4"
2973 [(match_operand:TI 0 "register_operand")
2974 (match_operand:TI 1 "register_operand")
2975 (match_operand:TI 2 "aarch64_reg_or_imm")
2976 (label_ref (match_operand 3 "" ""))]
2979 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2981 aarch64_subvti_scratch_regs (operands[1], operands[2],
2982 &low_dest, &op1_low, &op2_low,
2983 &high_dest, &op1_high, &op2_high);
2984 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2985 high_dest, op1_high, op2_high, true);
2987 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2991 (define_expand "negvti3"
2992 [(match_operand:TI 0 "register_operand")
2993 (match_operand:TI 1 "register_operand")
2994 (label_ref (match_operand 2 "" ""))]
2997 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2998 gen_lowpart (DImode, operands[1])));
2999 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
3000 gen_highpart (DImode, operands[1])));
3001 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
3007 (define_insn "negdi_carryout"
3008 [(set (reg:CC CC_REGNUM)
3010 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3011 (set (match_operand:DI 0 "register_operand" "=r")
3012 (neg:DI (match_dup 1)))]
3015 [(set_attr "type" "alus_sreg")]
3018 (define_insn "negvdi_carryinV"
3019 [(set (reg:CC_V CC_REGNUM)
3022 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3023 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3025 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3027 (set (match_operand:DI 0 "register_operand" "=r")
3028 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3032 [(set_attr "type" "alus_sreg")]
3035 (define_insn "*sub<mode>3_compare0"
3036 [(set (reg:CC_NZ CC_REGNUM)
3037 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3038 (match_operand:GPI 2 "register_operand" "r"))
3040 (set (match_operand:GPI 0 "register_operand" "=r")
3041 (minus:GPI (match_dup 1) (match_dup 2)))]
3043 "subs\\t%<w>0, %<w>1, %<w>2"
3044 [(set_attr "type" "alus_sreg")]
3047 ;; zero_extend version of above
3048 (define_insn "*subsi3_compare0_uxtw"
3049 [(set (reg:CC_NZ CC_REGNUM)
3050 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3051 (match_operand:SI 2 "register_operand" "r"))
3053 (set (match_operand:DI 0 "register_operand" "=r")
3054 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3056 "subs\\t%w0, %w1, %w2"
3057 [(set_attr "type" "alus_sreg")]
3060 (define_insn "sub<mode>3_compare1_imm"
3061 [(set (reg:CC CC_REGNUM)
3063 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3064 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3065 (set (match_operand:GPI 0 "register_operand" "=r,r")
3068 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3069 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3071 subs\\t%<w>0, %<w>1, %2
3072 adds\\t%<w>0, %<w>1, #%n2"
3073 [(set_attr "type" "alus_imm")]
3076 (define_insn "sub<mode>3_compare1"
3077 [(set (reg:CC CC_REGNUM)
3079 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3080 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3081 (set (match_operand:GPI 0 "register_operand" "=r")
3082 (minus:GPI (match_dup 1) (match_dup 2)))]
3084 "subs\\t%<w>0, %<w>1, %<w>2"
3085 [(set_attr "type" "alus_sreg")]
3089 [(set (match_operand:GPI 0 "aarch64_general_reg")
3090 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3091 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3092 (set (reg:CC CC_REGNUM)
3096 "!reg_overlap_mentioned_p (operands[0], operands[1])
3097 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3100 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3106 ;; Same as the above peephole but with the compare and minus in
3107 ;; swapped order. The restriction on overlap between operand 0
3108 ;; and operands 1 and 2 doesn't apply here.
3110 [(set (reg:CC CC_REGNUM)
3112 (match_operand:GPI 1 "aarch64_reg_or_zero")
3113 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3114 (set (match_operand:GPI 0 "aarch64_general_reg")
3115 (minus:GPI (match_dup 1)
3120 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3127 [(set (match_operand:GPI 0 "aarch64_general_reg")
3128 (plus:GPI (match_operand:GPI 1 "register_operand")
3129 (match_operand:GPI 2 "aarch64_plus_immediate")))
3130 (set (reg:CC CC_REGNUM)
3133 (match_operand:GPI 3 "const_int_operand")))]
3134 "!reg_overlap_mentioned_p (operands[0], operands[1])
3135 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3138 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3139 operands[3], operands[2]));
3144 ;; Same as the above peephole but with the compare and minus in
3145 ;; swapped order. The restriction on overlap between operand 0
3146 ;; and operands 1 doesn't apply here.
3148 [(set (reg:CC CC_REGNUM)
3150 (match_operand:GPI 1 "register_operand")
3151 (match_operand:GPI 3 "const_int_operand")))
3152 (set (match_operand:GPI 0 "aarch64_general_reg")
3153 (plus:GPI (match_dup 1)
3154 (match_operand:GPI 2 "aarch64_plus_immediate")))]
3155 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3158 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3159 operands[3], operands[2]));
3164 (define_insn "*sub_<shift>_<mode>"
3165 [(set (match_operand:GPI 0 "register_operand" "=r")
3166 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3168 (match_operand:GPI 1 "register_operand" "r")
3169 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3171 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3172 [(set_attr "type" "alu_shift_imm")]
3175 ;; zero_extend version of above
3176 (define_insn "*sub_<shift>_si_uxtw"
3177 [(set (match_operand:DI 0 "register_operand" "=r")
3179 (minus:SI (match_operand:SI 3 "register_operand" "r")
3181 (match_operand:SI 1 "register_operand" "r")
3182 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3184 "sub\\t%w0, %w3, %w1, <shift> %2"
3185 [(set_attr "type" "alu_shift_imm")]
3188 (define_insn "*sub_mul_imm_<mode>"
3189 [(set (match_operand:GPI 0 "register_operand" "=r")
3190 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3192 (match_operand:GPI 1 "register_operand" "r")
3193 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3195 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3196 [(set_attr "type" "alu_shift_imm")]
3199 ;; zero_extend version of above
3200 (define_insn "*sub_mul_imm_si_uxtw"
3201 [(set (match_operand:DI 0 "register_operand" "=r")
3203 (minus:SI (match_operand:SI 3 "register_operand" "r")
3205 (match_operand:SI 1 "register_operand" "r")
3206 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3208 "sub\\t%w0, %w3, %w1, lsl %p2"
3209 [(set_attr "type" "alu_shift_imm")]
3212 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3213 [(set (match_operand:GPI 0 "register_operand" "=rk")
3214 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3216 (match_operand:ALLX 2 "register_operand" "r"))))]
3218 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3219 [(set_attr "type" "alu_ext")]
3222 ;; zero_extend version of above
3223 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3224 [(set (match_operand:DI 0 "register_operand" "=rk")
3226 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3228 (match_operand:SHORT 2 "register_operand" "r")))))]
3230 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3231 [(set_attr "type" "alu_ext")]
3234 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3235 [(set (match_operand:GPI 0 "register_operand" "=rk")
3236 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3237 (ashift:GPI (ANY_EXTEND:GPI
3238 (match_operand:ALLX 2 "register_operand" "r"))
3239 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3241 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3242 [(set_attr "type" "alu_ext")]
3245 ;; zero_extend version of above
3246 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3247 [(set (match_operand:DI 0 "register_operand" "=rk")
3249 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3250 (ashift:SI (ANY_EXTEND:SI
3251 (match_operand:SHORT 2 "register_operand" "r"))
3252 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3254 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3255 [(set_attr "type" "alu_ext")]
3258 (define_insn "*sub_<optab><mode>_multp2"
3259 [(set (match_operand:GPI 0 "register_operand" "=rk")
3260 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3262 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3263 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3264 (match_operand 3 "const_int_operand" "n")
3266 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3267 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3268 [(set_attr "type" "alu_ext")]
3271 ;; zero_extend version of above
3272 (define_insn "*sub_<optab>si_multp2_uxtw"
3273 [(set (match_operand:DI 0 "register_operand" "=rk")
3275 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3277 (mult:SI (match_operand:SI 1 "register_operand" "r")
3278 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3279 (match_operand 3 "const_int_operand" "n")
3281 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3282 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3283 [(set_attr "type" "alu_ext")]
3286 ;; The hardware description is op1 + ~op2 + C.
3287 ;; = op1 + (-op2 + 1) + (1 - !C)
3288 ;; = op1 - op2 - 1 + 1 - !C
3289 ;; = op1 - op2 - !C.
3290 ;; We describe the latter.
3292 (define_insn "*sub<mode>3_carryin0"
3293 [(set (match_operand:GPI 0 "register_operand" "=r")
3295 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3296 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3298 "sbc\\t%<w>0, %<w>1, <w>zr"
3299 [(set_attr "type" "adc_reg")]
3302 ;; zero_extend version of the above
3303 (define_insn "*subsi3_carryin_uxtw"
3304 [(set (match_operand:DI 0 "register_operand" "=r")
3307 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3308 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3310 "sbc\\t%w0, %w1, wzr"
3311 [(set_attr "type" "adc_reg")]
3314 (define_expand "sub<mode>3_carryin"
3315 [(set (match_operand:GPI 0 "register_operand")
3318 (match_operand:GPI 1 "aarch64_reg_or_zero")
3319 (match_operand:GPI 2 "register_operand"))
3320 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3325 (define_insn "*sub<mode>3_carryin"
3326 [(set (match_operand:GPI 0 "register_operand" "=r")
3329 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3330 (match_operand:GPI 2 "register_operand" "r"))
3331 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3334 "sbc\\t%<w>0, %<w>1, %<w>2"
3335 [(set_attr "type" "adc_reg")]
3338 ;; zero_extend version of the above
3339 (define_insn "*subsi3_carryin_uxtw"
3340 [(set (match_operand:DI 0 "register_operand" "=r")
3344 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3345 (match_operand:SI 2 "register_operand" "r"))
3346 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3349 "sbc\\t%w0, %w1, %w2"
3350 [(set_attr "type" "adc_reg")]
3353 (define_insn "*sub<mode>3_carryin_alt"
3354 [(set (match_operand:GPI 0 "register_operand" "=r")
3357 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3358 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3359 (match_operand:GPI 2 "register_operand" "r")))]
3361 "sbc\\t%<w>0, %<w>1, %<w>2"
3362 [(set_attr "type" "adc_reg")]
3365 ;; zero_extend version of the above
3366 (define_insn "*subsi3_carryin_alt_uxtw"
3367 [(set (match_operand:DI 0 "register_operand" "=r")
3371 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3372 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3373 (match_operand:SI 2 "register_operand" "r"))))]
3375 "sbc\\t%w0, %w1, %w2"
3376 [(set_attr "type" "adc_reg")]
3379 (define_expand "usub<GPI:mode>3_carryinC"
3381 [(set (reg:CC CC_REGNUM)
3384 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3387 (match_operand:GPI 2 "register_operand"))
3388 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3389 (set (match_operand:GPI 0 "register_operand")
3391 (minus:GPI (match_dup 1) (match_dup 2))
3392 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3396 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3397 [(set (reg:CC CC_REGNUM)
3402 (match_operand:GPI 1 "register_operand" "r"))
3403 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3404 (set (match_operand:GPI 0 "register_operand" "=r")
3406 (neg:GPI (match_dup 1))
3407 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3409 "sbcs\\t%<w>0, <w>zr, %<w>1"
3410 [(set_attr "type" "adc_reg")]
3413 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3414 [(set (reg:CC CC_REGNUM)
3417 (match_operand:GPI 1 "register_operand" "r"))
3418 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3419 (set (match_operand:GPI 0 "register_operand" "=r")
3422 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3424 "sbcs\\t%<w>0, %<w>1, <w>zr"
3425 [(set_attr "type" "adc_reg")]
3428 (define_insn "*usub<GPI:mode>3_carryinC"
3429 [(set (reg:CC CC_REGNUM)
3432 (match_operand:GPI 1 "register_operand" "r"))
3435 (match_operand:GPI 2 "register_operand" "r"))
3436 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3437 (set (match_operand:GPI 0 "register_operand" "=r")
3439 (minus:GPI (match_dup 1) (match_dup 2))
3440 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3442 "sbcs\\t%<w>0, %<w>1, %<w>2"
3443 [(set_attr "type" "adc_reg")]
3446 (define_expand "sub<GPI:mode>3_carryinV"
3448 [(set (reg:CC_V CC_REGNUM)
3452 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3455 (match_operand:GPI 2 "register_operand"))
3456 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3458 (minus:GPI (match_dup 1)
3459 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3461 (set (match_operand:GPI 0 "register_operand")
3463 (minus:GPI (match_dup 1) (match_dup 2))
3464 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3468 (define_insn "*sub<mode>3_carryinV_z2"
3469 [(set (reg:CC_V CC_REGNUM)
3472 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3473 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3475 (minus:GPI (match_dup 1)
3476 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3477 (set (match_operand:GPI 0 "register_operand" "=r")
3479 (match_dup 1) (match_dup 3)))]
3481 "sbcs\\t%<w>0, %<w>1, <w>zr"
3482 [(set_attr "type" "adc_reg")]
3485 (define_insn "*sub<mode>3_carryinV"
3486 [(set (reg:CC_V CC_REGNUM)
3490 (match_operand:GPI 1 "register_operand" "r"))
3493 (match_operand:GPI 2 "register_operand" "r"))
3494 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3498 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3500 (set (match_operand:GPI 0 "register_operand" "=r")
3502 (minus:GPI (match_dup 1) (match_dup 2))
3505 "sbcs\\t%<w>0, %<w>1, %<w>2"
3506 [(set_attr "type" "adc_reg")]
3509 (define_insn "*sub_uxt<mode>_shift2"
3510 [(set (match_operand:GPI 0 "register_operand" "=rk")
3511 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3513 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3514 (match_operand 2 "aarch64_imm3" "Ui3"))
3515 (match_operand 3 "const_int_operand" "n"))))]
3516 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3518 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3519 INTVAL (operands[3])));
3520 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3521 [(set_attr "type" "alu_ext")]
3524 ;; zero_extend version of above
3525 (define_insn "*sub_uxtsi_shift2_uxtw"
3526 [(set (match_operand:DI 0 "register_operand" "=rk")
3528 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3530 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3531 (match_operand 2 "aarch64_imm3" "Ui3"))
3532 (match_operand 3 "const_int_operand" "n")))))]
3533 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3535 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3536 INTVAL (operands[3])));
3537 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3538 [(set_attr "type" "alu_ext")]
3541 (define_insn "*sub_uxt<mode>_multp2"
3542 [(set (match_operand:GPI 0 "register_operand" "=rk")
3543 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3545 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3546 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3547 (match_operand 3 "const_int_operand" "n"))))]
3548 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3550 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3551 INTVAL (operands[3])));
3552 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3553 [(set_attr "type" "alu_ext")]
3556 ;; zero_extend version of above
3557 (define_insn "*sub_uxtsi_multp2_uxtw"
3558 [(set (match_operand:DI 0 "register_operand" "=rk")
3560 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3562 (mult:SI (match_operand:SI 1 "register_operand" "r")
3563 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3564 (match_operand 3 "const_int_operand" "n")))))]
3565 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3567 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3568 INTVAL (operands[3])));
3569 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3570 [(set_attr "type" "alu_ext")]
3573 (define_expand "abs<mode>2"
3574 [(match_operand:GPI 0 "register_operand")
3575 (match_operand:GPI 1 "register_operand")]
3578 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3579 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3580 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3585 (define_insn "neg<mode>2"
3586 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3587 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3591 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3592 [(set_attr "type" "alu_sreg, neon_neg<q>")
3593 (set_attr "arch" "*,simd")]
3596 ;; zero_extend version of above
3597 (define_insn "*negsi2_uxtw"
3598 [(set (match_operand:DI 0 "register_operand" "=r")
3599 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3602 [(set_attr "type" "alu_sreg")]
3605 (define_insn "*ngc<mode>"
3606 [(set (match_operand:GPI 0 "register_operand" "=r")
3608 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3609 (match_operand:GPI 1 "register_operand" "r")))]
3611 "ngc\\t%<w>0, %<w>1"
3612 [(set_attr "type" "adc_reg")]
3615 (define_insn "*ngcsi_uxtw"
3616 [(set (match_operand:DI 0 "register_operand" "=r")
3619 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3620 (match_operand:SI 1 "register_operand" "r"))))]
3623 [(set_attr "type" "adc_reg")]
3626 (define_insn "neg<mode>2_compare0"
3627 [(set (reg:CC_NZ CC_REGNUM)
3628 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3630 (set (match_operand:GPI 0 "register_operand" "=r")
3631 (neg:GPI (match_dup 1)))]
3633 "negs\\t%<w>0, %<w>1"
3634 [(set_attr "type" "alus_sreg")]
3637 ;; zero_extend version of above
3638 (define_insn "*negsi2_compare0_uxtw"
3639 [(set (reg:CC_NZ CC_REGNUM)
3640 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3642 (set (match_operand:DI 0 "register_operand" "=r")
3643 (zero_extend:DI (neg:SI (match_dup 1))))]
3646 [(set_attr "type" "alus_sreg")]
3649 (define_insn "*neg_<shift><mode>3_compare0"
3650 [(set (reg:CC_NZ CC_REGNUM)
3652 (neg:GPI (ASHIFT:GPI
3653 (match_operand:GPI 1 "register_operand" "r")
3654 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3656 (set (match_operand:GPI 0 "register_operand" "=r")
3657 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3659 "negs\\t%<w>0, %<w>1, <shift> %2"
3660 [(set_attr "type" "alus_shift_imm")]
3663 (define_insn "*neg_<shift>_<mode>2"
3664 [(set (match_operand:GPI 0 "register_operand" "=r")
3665 (neg:GPI (ASHIFT:GPI
3666 (match_operand:GPI 1 "register_operand" "r")
3667 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3669 "neg\\t%<w>0, %<w>1, <shift> %2"
3670 [(set_attr "type" "alu_shift_imm")]
3673 ;; zero_extend version of above
3674 (define_insn "*neg_<shift>_si2_uxtw"
3675 [(set (match_operand:DI 0 "register_operand" "=r")
3678 (match_operand:SI 1 "register_operand" "r")
3679 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3681 "neg\\t%w0, %w1, <shift> %2"
3682 [(set_attr "type" "alu_shift_imm")]
3685 (define_insn "*neg_mul_imm_<mode>2"
3686 [(set (match_operand:GPI 0 "register_operand" "=r")
3688 (match_operand:GPI 1 "register_operand" "r")
3689 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3691 "neg\\t%<w>0, %<w>1, lsl %p2"
3692 [(set_attr "type" "alu_shift_imm")]
3695 ;; zero_extend version of above
3696 (define_insn "*neg_mul_imm_si2_uxtw"
3697 [(set (match_operand:DI 0 "register_operand" "=r")
3700 (match_operand:SI 1 "register_operand" "r")
3701 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3703 "neg\\t%w0, %w1, lsl %p2"
3704 [(set_attr "type" "alu_shift_imm")]
3707 (define_insn "mul<mode>3"
3708 [(set (match_operand:GPI 0 "register_operand" "=r")
3709 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3710 (match_operand:GPI 2 "register_operand" "r")))]
3712 "mul\\t%<w>0, %<w>1, %<w>2"
3713 [(set_attr "type" "mul")]
3716 ;; zero_extend version of above
3717 (define_insn "*mulsi3_uxtw"
3718 [(set (match_operand:DI 0 "register_operand" "=r")
3720 (mult:SI (match_operand:SI 1 "register_operand" "r")
3721 (match_operand:SI 2 "register_operand" "r"))))]
3723 "mul\\t%w0, %w1, %w2"
3724 [(set_attr "type" "mul")]
3727 (define_insn "madd<mode>"
3728 [(set (match_operand:GPI 0 "register_operand" "=r")
3729 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3730 (match_operand:GPI 2 "register_operand" "r"))
3731 (match_operand:GPI 3 "register_operand" "r")))]
3733 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3734 [(set_attr "type" "mla")]
3737 ;; zero_extend version of above
3738 (define_insn "*maddsi_uxtw"
3739 [(set (match_operand:DI 0 "register_operand" "=r")
3741 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3742 (match_operand:SI 2 "register_operand" "r"))
3743 (match_operand:SI 3 "register_operand" "r"))))]
3745 "madd\\t%w0, %w1, %w2, %w3"
3746 [(set_attr "type" "mla")]
3749 (define_insn "*msub<mode>"
3750 [(set (match_operand:GPI 0 "register_operand" "=r")
3751 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3752 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3753 (match_operand:GPI 2 "register_operand" "r"))))]
3756 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3757 [(set_attr "type" "mla")]
3760 ;; zero_extend version of above
3761 (define_insn "*msubsi_uxtw"
3762 [(set (match_operand:DI 0 "register_operand" "=r")
3764 (minus:SI (match_operand:SI 3 "register_operand" "r")
3765 (mult:SI (match_operand:SI 1 "register_operand" "r")
3766 (match_operand:SI 2 "register_operand" "r")))))]
3769 "msub\\t%w0, %w1, %w2, %w3"
3770 [(set_attr "type" "mla")]
3773 (define_insn "*mul<mode>_neg"
3774 [(set (match_operand:GPI 0 "register_operand" "=r")
3775 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3776 (match_operand:GPI 2 "register_operand" "r")))]
3779 "mneg\\t%<w>0, %<w>1, %<w>2"
3780 [(set_attr "type" "mul")]
3783 ;; zero_extend version of above
3784 (define_insn "*mulsi_neg_uxtw"
3785 [(set (match_operand:DI 0 "register_operand" "=r")
3787 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3788 (match_operand:SI 2 "register_operand" "r"))))]
3791 "mneg\\t%w0, %w1, %w2"
3792 [(set_attr "type" "mul")]
3795 (define_insn "<su_optab>mulsidi3"
3796 [(set (match_operand:DI 0 "register_operand" "=r")
3797 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3798 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3800 "<su>mull\\t%0, %w1, %w2"
3801 [(set_attr "type" "<su>mull")]
3804 (define_insn "<su_optab>maddsidi4"
3805 [(set (match_operand:DI 0 "register_operand" "=r")
3807 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3808 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3809 (match_operand:DI 3 "register_operand" "r")))]
3811 "<su>maddl\\t%0, %w1, %w2, %3"
3812 [(set_attr "type" "<su>mlal")]
3815 (define_insn "<su_optab>msubsidi4"
3816 [(set (match_operand:DI 0 "register_operand" "=r")
3818 (match_operand:DI 3 "register_operand" "r")
3819 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3821 (match_operand:SI 2 "register_operand" "r")))))]
3823 "<su>msubl\\t%0, %w1, %w2, %3"
3824 [(set_attr "type" "<su>mlal")]
3827 (define_insn "*<su_optab>mulsidi_neg"
3828 [(set (match_operand:DI 0 "register_operand" "=r")
3830 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3831 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3833 "<su>mnegl\\t%0, %w1, %w2"
3834 [(set_attr "type" "<su>mull")]
3837 (define_expand "<su_optab>mulditi3"
3838 [(set (match_operand:TI 0 "register_operand")
3839 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3840 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3843 rtx low = gen_reg_rtx (DImode);
3844 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3846 rtx high = gen_reg_rtx (DImode);
3847 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3849 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3850 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3854 ;; The default expansion of multi3 using umuldi3_highpart will perform
3855 ;; the additions in an order that fails to combine into two madd insns.
3856 (define_expand "multi3"
3857 [(set (match_operand:TI 0 "register_operand")
3858 (mult:TI (match_operand:TI 1 "register_operand")
3859 (match_operand:TI 2 "register_operand")))]
3862 rtx l0 = gen_reg_rtx (DImode);
3863 rtx l1 = gen_lowpart (DImode, operands[1]);
3864 rtx l2 = gen_lowpart (DImode, operands[2]);
3865 rtx h0 = gen_reg_rtx (DImode);
3866 rtx h1 = gen_highpart (DImode, operands[1]);
3867 rtx h2 = gen_highpart (DImode, operands[2]);
3869 emit_insn (gen_muldi3 (l0, l1, l2));
3870 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3871 emit_insn (gen_madddi (h0, h1, l2, h0));
3872 emit_insn (gen_madddi (h0, l1, h2, h0));
3874 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3875 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3879 (define_insn "<su>muldi3_highpart"
3880 [(set (match_operand:DI 0 "register_operand" "=r")
3884 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3885 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3888 "<su>mulh\\t%0, %1, %2"
3889 [(set_attr "type" "<su>mull")]
3892 (define_insn "<su_optab>div<mode>3"
3893 [(set (match_operand:GPI 0 "register_operand" "=r")
3894 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3895 (match_operand:GPI 2 "register_operand" "r")))]
3897 "<su>div\\t%<w>0, %<w>1, %<w>2"
3898 [(set_attr "type" "<su>div")]
3901 ;; zero_extend version of above
3902 (define_insn "*<su_optab>divsi3_uxtw"
3903 [(set (match_operand:DI 0 "register_operand" "=r")
3905 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3906 (match_operand:SI 2 "register_operand" "r"))))]
3908 "<su>div\\t%w0, %w1, %w2"
3909 [(set_attr "type" "<su>div")]
3912 ;; -------------------------------------------------------------------
3914 ;; -------------------------------------------------------------------
3916 (define_insn "cmp<mode>"
3917 [(set (reg:CC CC_REGNUM)
3918 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3919 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3925 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3928 (define_insn "fcmp<mode>"
3929 [(set (reg:CCFP CC_REGNUM)
3930 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3931 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3935 fcmp\\t%<s>0, %<s>1"
3936 [(set_attr "type" "fcmp<s>")]
3939 (define_insn "fcmpe<mode>"
3940 [(set (reg:CCFPE CC_REGNUM)
3941 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3942 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3946 fcmpe\\t%<s>0, %<s>1"
3947 [(set_attr "type" "fcmp<s>")]
3950 (define_insn "*cmp_swp_<shift>_reg<mode>"
3951 [(set (reg:CC_SWP CC_REGNUM)
3952 (compare:CC_SWP (ASHIFT:GPI
3953 (match_operand:GPI 0 "register_operand" "r")
3954 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3955 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3957 "cmp\\t%<w>2, %<w>0, <shift> %1"
3958 [(set_attr "type" "alus_shift_imm")]
3961 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3962 [(set (reg:CC_SWP CC_REGNUM)
3963 (compare:CC_SWP (ANY_EXTEND:GPI
3964 (match_operand:ALLX 0 "register_operand" "r"))
3965 (match_operand:GPI 1 "register_operand" "r")))]
3967 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3968 [(set_attr "type" "alus_ext")]
3971 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3972 [(set (reg:CC_SWP CC_REGNUM)
3973 (compare:CC_SWP (ashift:GPI
3975 (match_operand:ALLX 0 "register_operand" "r"))
3976 (match_operand 1 "aarch64_imm3" "Ui3"))
3977 (match_operand:GPI 2 "register_operand" "r")))]
3979 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3980 [(set_attr "type" "alus_ext")]
3983 ;; -------------------------------------------------------------------
3984 ;; Store-flag and conditional select insns
3985 ;; -------------------------------------------------------------------
3987 (define_expand "cstore<mode>4"
3988 [(set (match_operand:SI 0 "register_operand")
3989 (match_operator:SI 1 "aarch64_comparison_operator"
3990 [(match_operand:GPI 2 "register_operand")
3991 (match_operand:GPI 3 "aarch64_plus_operand")]))]
3994 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3996 operands[3] = const0_rtx;
4000 (define_expand "cstorecc4"
4001 [(set (match_operand:SI 0 "register_operand")
4002 (match_operator 1 "aarch64_comparison_operator_mode"
4003 [(match_operand 2 "cc_register")
4004 (match_operand 3 "const0_operand")]))]
4007 emit_insn (gen_rtx_SET (operands[0], operands[1]));
4012 (define_expand "cstore<mode>4"
4013 [(set (match_operand:SI 0 "register_operand")
4014 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4015 [(match_operand:GPF 2 "register_operand")
4016 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
4019 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4021 operands[3] = const0_rtx;
4025 (define_insn "aarch64_cstore<mode>"
4026 [(set (match_operand:ALLI 0 "register_operand" "=r")
4027 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4028 [(match_operand 2 "cc_register" "") (const_int 0)]))]
4031 [(set_attr "type" "csel")]
4034 ;; For a 24-bit immediate CST we can optimize the compare for equality
4035 ;; and branch sequence from:
4037 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4040 ;; into the shorter:
4041 ;; sub x0, x1, #(CST & 0xfff000)
4042 ;; subs x0, x0, #(CST & 0x000fff)
4043 ;; cset x2, <ne, eq>.
4044 (define_insn_and_split "*compare_cstore<mode>_insn"
4045 [(set (match_operand:GPI 0 "register_operand" "=r")
4046 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
4047 (match_operand:GPI 2 "aarch64_imm24" "n")))
4048 (clobber (reg:CC CC_REGNUM))]
4049 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4050 && !aarch64_plus_operand (operands[2], <MODE>mode)
4051 && !reload_completed"
4056 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4057 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4058 rtx tmp = gen_reg_rtx (<MODE>mode);
4059 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4060 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4061 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4062 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4063 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4066 [(set_attr "type" "csel")]
4069 ;; zero_extend version of the above
4070 (define_insn "*cstoresi_insn_uxtw"
4071 [(set (match_operand:DI 0 "register_operand" "=r")
4073 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4074 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4077 [(set_attr "type" "csel")]
4080 (define_insn "cstore<mode>_neg"
4081 [(set (match_operand:ALLI 0 "register_operand" "=r")
4082 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4083 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4085 "csetm\\t%<w>0, %m1"
4086 [(set_attr "type" "csel")]
4089 ;; zero_extend version of the above
4090 (define_insn "*cstoresi_neg_uxtw"
4091 [(set (match_operand:DI 0 "register_operand" "=r")
4093 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4094 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4097 [(set_attr "type" "csel")]
4100 (define_expand "cmov<mode>6"
4101 [(set (match_operand:GPI 0 "register_operand")
4103 (match_operator 1 "aarch64_comparison_operator"
4104 [(match_operand:GPI 2 "register_operand")
4105 (match_operand:GPI 3 "aarch64_plus_operand")])
4106 (match_operand:GPI 4 "register_operand")
4107 (match_operand:GPI 5 "register_operand")))]
4110 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4112 operands[3] = const0_rtx;
4116 (define_expand "cmov<mode>6"
4117 [(set (match_operand:GPF 0 "register_operand")
4119 (match_operator 1 "aarch64_comparison_operator"
4120 [(match_operand:GPF 2 "register_operand")
4121 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4122 (match_operand:GPF 4 "register_operand")
4123 (match_operand:GPF 5 "register_operand")))]
4126 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4128 operands[3] = const0_rtx;
4132 (define_insn "*cmov<mode>_insn"
4133 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4135 (match_operator 1 "aarch64_comparison_operator"
4136 [(match_operand 2 "cc_register" "") (const_int 0)])
4137 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4138 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4139 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4140 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4141 ;; Final two alternatives should be unreachable, but included for completeness
4143 csel\\t%<w>0, %<w>3, %<w>4, %m1
4144 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4145 csinv\\t%<w>0, %<w>4, <w>zr, %M1
4146 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4147 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4150 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4153 ;; zero_extend version of above
4154 (define_insn "*cmovsi_insn_uxtw"
4155 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4158 (match_operator 1 "aarch64_comparison_operator"
4159 [(match_operand 2 "cc_register" "") (const_int 0)])
4160 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4161 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4162 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4163 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4164 ;; Final two alternatives should be unreachable, but included for completeness
4166 csel\\t%w0, %w3, %w4, %m1
4167 csinv\\t%w0, %w3, wzr, %m1
4168 csinv\\t%w0, %w4, wzr, %M1
4169 csinc\\t%w0, %w3, wzr, %m1
4170 csinc\\t%w0, %w4, wzr, %M1
4173 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4176 (define_insn "*cmovdi_insn_uxtw"
4177 [(set (match_operand:DI 0 "register_operand" "=r")
4179 (match_operator 1 "aarch64_comparison_operator"
4180 [(match_operand 2 "cc_register" "") (const_int 0)])
4181 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4182 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4184 "csel\\t%w0, %w3, %w4, %m1"
4185 [(set_attr "type" "csel")]
4188 (define_insn "*cmov<mode>_insn"
4189 [(set (match_operand:GPF 0 "register_operand" "=w")
4191 (match_operator 1 "aarch64_comparison_operator"
4192 [(match_operand 2 "cc_register" "") (const_int 0)])
4193 (match_operand:GPF 3 "register_operand" "w")
4194 (match_operand:GPF 4 "register_operand" "w")))]
4196 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4197 [(set_attr "type" "fcsel")]
4200 (define_expand "mov<mode>cc"
4201 [(set (match_operand:ALLI 0 "register_operand")
4202 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4203 (match_operand:ALLI 2 "register_operand")
4204 (match_operand:ALLI 3 "register_operand")))]
4208 enum rtx_code code = GET_CODE (operands[1]);
4210 if (code == UNEQ || code == LTGT)
4213 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4214 XEXP (operands[1], 1));
4215 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4219 (define_expand "mov<GPF:mode><GPI:mode>cc"
4220 [(set (match_operand:GPI 0 "register_operand")
4221 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4222 (match_operand:GPF 2 "register_operand")
4223 (match_operand:GPF 3 "register_operand")))]
4227 enum rtx_code code = GET_CODE (operands[1]);
4229 if (code == UNEQ || code == LTGT)
4232 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4233 XEXP (operands[1], 1));
4234 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4238 (define_expand "mov<mode>cc"
4239 [(set (match_operand:GPF 0 "register_operand")
4240 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4241 (match_operand:GPF 2 "register_operand")
4242 (match_operand:GPF 3 "register_operand")))]
4246 enum rtx_code code = GET_CODE (operands[1]);
4248 if (code == UNEQ || code == LTGT)
4251 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4252 XEXP (operands[1], 1));
4253 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4257 (define_expand "<neg_not_op><mode>cc"
4258 [(set (match_operand:GPI 0 "register_operand")
4259 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4260 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4261 (match_operand:GPI 3 "register_operand")))]
4265 enum rtx_code code = GET_CODE (operands[1]);
4267 if (code == UNEQ || code == LTGT)
4270 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4271 XEXP (operands[1], 1));
4272 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4276 ;; CRC32 instructions.
4277 (define_insn "aarch64_<crc_variant>"
4278 [(set (match_operand:SI 0 "register_operand" "=r")
4279 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4280 (match_operand:<crc_mode> 2 "register_operand" "r")]
4284 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4285 return "<crc_variant>\\t%w0, %w1, %x2";
4287 return "<crc_variant>\\t%w0, %w1, %w2";
4289 [(set_attr "type" "crc")]
4292 (define_insn "*csinc2<mode>_insn"
4293 [(set (match_operand:GPI 0 "register_operand" "=r")
4294 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4295 (match_operand:GPI 1 "register_operand" "r")))]
4297 "cinc\\t%<w>0, %<w>1, %m2"
4298 [(set_attr "type" "csel")]
4301 (define_insn "csinc3<mode>_insn"
4302 [(set (match_operand:GPI 0 "register_operand" "=r")
4304 (match_operand 1 "aarch64_comparison_operation" "")
4305 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4307 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4309 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4310 [(set_attr "type" "csel")]
4313 (define_insn "*csinv3<mode>_insn"
4314 [(set (match_operand:GPI 0 "register_operand" "=r")
4316 (match_operand 1 "aarch64_comparison_operation" "")
4317 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4318 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4320 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4321 [(set_attr "type" "csel")]
4324 (define_insn "csneg3_uxtw_insn"
4325 [(set (match_operand:DI 0 "register_operand" "=r")
4328 (match_operand 1 "aarch64_comparison_operation" "")
4329 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4330 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4332 "csneg\\t%w0, %w3, %w2, %M1"
4333 [(set_attr "type" "csel")]
4336 (define_insn "csneg3<mode>_insn"
4337 [(set (match_operand:GPI 0 "register_operand" "=r")
4339 (match_operand 1 "aarch64_comparison_operation" "")
4340 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4341 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4343 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4344 [(set_attr "type" "csel")]
4347 ;; If X can be loaded by a single CNT[BHWD] instruction,
4351 ;; is equivalent to:
4353 ;; TMP = UQDEC[BHWD] (B, X)
4356 ;; Defining the pattern this way means that:
4358 ;; A = UMAX (B, X) - X
4362 ;; TMP1 = UQDEC[BHWD] (B, X)
4366 ;; which combine can optimize to:
4368 ;; A = UQDEC[BHWD] (B, X)
4370 ;; We don't use match_operand predicates because the order of the operands
4371 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4372 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4373 (define_expand "umax<mode>3"
4374 [(set (match_operand:GPI 0 "register_operand")
4375 (umax:GPI (match_operand:GPI 1 "")
4376 (match_operand:GPI 2 "")))]
4379 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4380 std::swap (operands[1], operands[2]);
4381 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4383 rtx temp = gen_reg_rtx (<MODE>mode);
4384 operands[1] = force_reg (<MODE>mode, operands[1]);
4385 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4386 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4391 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4392 (define_insn "aarch64_uqdec<mode>"
4393 [(set (match_operand:GPI 0 "register_operand" "=r")
4395 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4396 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4400 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4404 ;; -------------------------------------------------------------------
4405 ;; Logical operations
4406 ;; -------------------------------------------------------------------
4409 (define_insn_and_split "*aarch64_and<mode>_imm2"
4410 [(set (match_operand:GPI 0 "register_operand" "=rk")
4411 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4412 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4418 HOST_WIDE_INT val = INTVAL (operands[2]);
4419 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4420 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4422 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4423 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4428 (define_insn "<optab><mode>3"
4429 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4430 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4431 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4434 <logical>\\t%<w>0, %<w>1, %<w>2
4435 <logical>\\t%<w>0, %<w>1, %2
4436 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4437 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4438 (set_attr "arch" "*,*,simd")]
4441 ;; zero_extend version of above
4442 (define_insn "*<optab>si3_uxtw"
4443 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4445 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4446 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4449 <logical>\\t%w0, %w1, %w2
4450 <logical>\\t%w0, %w1, %2"
4451 [(set_attr "type" "logic_reg,logic_imm")]
4454 (define_insn "*and<mode>3_compare0"
4455 [(set (reg:CC_NZ CC_REGNUM)
4457 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4458 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4460 (set (match_operand:GPI 0 "register_operand" "=r,r")
4461 (and:GPI (match_dup 1) (match_dup 2)))]
4464 ands\\t%<w>0, %<w>1, %<w>2
4465 ands\\t%<w>0, %<w>1, %2"
4466 [(set_attr "type" "logics_reg,logics_imm")]
4469 ;; zero_extend version of above
4470 (define_insn "*andsi3_compare0_uxtw"
4471 [(set (reg:CC_NZ CC_REGNUM)
4473 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4474 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4476 (set (match_operand:DI 0 "register_operand" "=r,r")
4477 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4480 ands\\t%w0, %w1, %w2
4481 ands\\t%w0, %w1, %2"
4482 [(set_attr "type" "logics_reg,logics_imm")]
4485 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4486 [(set (reg:CC_NZ CC_REGNUM)
4489 (match_operand:GPI 1 "register_operand" "r")
4490 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4491 (match_operand:GPI 3 "register_operand" "r"))
4493 (set (match_operand:GPI 0 "register_operand" "=r")
4494 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4496 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4497 [(set_attr "type" "logics_shift_imm")]
4500 ;; zero_extend version of above
4501 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4502 [(set (reg:CC_NZ CC_REGNUM)
4505 (match_operand:SI 1 "register_operand" "r")
4506 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4507 (match_operand:SI 3 "register_operand" "r"))
4509 (set (match_operand:DI 0 "register_operand" "=r")
4510 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4513 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4514 [(set_attr "type" "logics_shift_imm")]
4517 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4518 [(set (match_operand:GPI 0 "register_operand" "=r")
4519 (LOGICAL:GPI (SHIFT:GPI
4520 (match_operand:GPI 1 "register_operand" "r")
4521 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4522 (match_operand:GPI 3 "register_operand" "r")))]
4524 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4525 [(set_attr "type" "logic_shift_imm")]
4528 (define_insn "*<optab>_rol<mode>3"
4529 [(set (match_operand:GPI 0 "register_operand" "=r")
4530 (LOGICAL:GPI (rotate:GPI
4531 (match_operand:GPI 1 "register_operand" "r")
4532 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4533 (match_operand:GPI 3 "register_operand" "r")))]
4535 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4536 [(set_attr "type" "logic_shift_imm")]
4539 ;; zero_extend versions of above
4540 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4541 [(set (match_operand:DI 0 "register_operand" "=r")
4543 (LOGICAL:SI (SHIFT:SI
4544 (match_operand:SI 1 "register_operand" "r")
4545 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4546 (match_operand:SI 3 "register_operand" "r"))))]
4548 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4549 [(set_attr "type" "logic_shift_imm")]
4552 (define_insn "*<optab>_rolsi3_uxtw"
4553 [(set (match_operand:DI 0 "register_operand" "=r")
4555 (LOGICAL:SI (rotate:SI
4556 (match_operand:SI 1 "register_operand" "r")
4557 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4558 (match_operand:SI 3 "register_operand" "r"))))]
4560 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4561 [(set_attr "type" "logic_shift_imm")]
4564 (define_insn "one_cmpl<mode>2"
4565 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4566 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4571 [(set_attr "type" "logic_reg,neon_logic")
4572 (set_attr "arch" "*,simd")]
4575 (define_insn "*one_cmpl_<optab><mode>2"
4576 [(set (match_operand:GPI 0 "register_operand" "=r")
4577 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4578 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4580 "mvn\\t%<w>0, %<w>1, <shift> %2"
4581 [(set_attr "type" "logic_shift_imm")]
4584 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4586 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4587 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4588 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4589 (match_operand:GPI 2 "register_operand" "r,w")))]
4592 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4593 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4594 [(set_attr "type" "logic_reg,neon_logic")
4595 (set_attr "arch" "*,simd")]
4598 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4599 [(set (match_operand:DI 0 "register_operand" "=r")
4601 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4602 (match_operand:SI 2 "register_operand" "r"))))]
4604 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4605 [(set_attr "type" "logic_reg")]
4608 (define_insn "*xor_one_cmplsidi3_ze"
4609 [(set (match_operand:DI 0 "register_operand" "=r")
4611 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4612 (match_operand:SI 2 "register_operand" "r")))))]
4614 "eon\\t%w0, %w1, %w2"
4615 [(set_attr "type" "logic_reg")]
4618 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4619 ;; eon does not operate on SIMD registers so the vector variant must be split.
4620 (define_insn_and_split "*xor_one_cmpl<mode>3"
4621 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4622 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4623 (match_operand:GPI 2 "register_operand" "r,w"))))]
4626 eon\\t%<w>0, %<w>1, %<w>2
4628 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4629 [(set (match_operand:GPI 0 "register_operand" "=w")
4630 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4631 (match_operand:GPI 2 "register_operand" "w")))
4632 (set (match_dup 0) (not:GPI (match_dup 0)))]
4634 [(set_attr "type" "logic_reg,multiple")
4635 (set_attr "arch" "*,simd")]
4638 (define_insn "*and_one_cmpl<mode>3_compare0"
4639 [(set (reg:CC_NZ CC_REGNUM)
4642 (match_operand:GPI 1 "register_operand" "r"))
4643 (match_operand:GPI 2 "register_operand" "r"))
4645 (set (match_operand:GPI 0 "register_operand" "=r")
4646 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4648 "bics\\t%<w>0, %<w>2, %<w>1"
4649 [(set_attr "type" "logics_reg")]
4652 ;; zero_extend version of above
4653 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4654 [(set (reg:CC_NZ CC_REGNUM)
4657 (match_operand:SI 1 "register_operand" "r"))
4658 (match_operand:SI 2 "register_operand" "r"))
4660 (set (match_operand:DI 0 "register_operand" "=r")
4661 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4663 "bics\\t%w0, %w2, %w1"
4664 [(set_attr "type" "logics_reg")]
4667 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4668 [(set (reg:CC_NZ CC_REGNUM)
4671 (match_operand:GPI 0 "register_operand" "r"))
4672 (match_operand:GPI 1 "register_operand" "r"))
4675 "bics\\t<w>zr, %<w>1, %<w>0"
4676 [(set_attr "type" "logics_reg")]
4679 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4680 [(set (match_operand:GPI 0 "register_operand" "=r")
4681 (LOGICAL:GPI (not:GPI
4683 (match_operand:GPI 1 "register_operand" "r")
4684 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4685 (match_operand:GPI 3 "register_operand" "r")))]
4687 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4688 [(set_attr "type" "logic_shift_imm")]
4691 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4692 [(set (match_operand:GPI 0 "register_operand" "=r")
4695 (match_operand:GPI 1 "register_operand" "r")
4696 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4697 (match_operand:GPI 3 "register_operand" "r"))))]
4699 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4700 [(set_attr "type" "logic_shift_imm")]
4703 ;; Zero-extend version of the above.
4704 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4705 [(set (match_operand:DI 0 "register_operand" "=r")
4709 (match_operand:SI 1 "register_operand" "r")
4710 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4711 (match_operand:SI 3 "register_operand" "r")))))]
4713 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4714 [(set_attr "type" "logic_shift_imm")]
4717 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4718 [(set (reg:CC_NZ CC_REGNUM)
4722 (match_operand:GPI 1 "register_operand" "r")
4723 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4724 (match_operand:GPI 3 "register_operand" "r"))
4726 (set (match_operand:GPI 0 "register_operand" "=r")
4729 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4731 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4732 [(set_attr "type" "logics_shift_imm")]
4735 ;; zero_extend version of above
4736 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4737 [(set (reg:CC_NZ CC_REGNUM)
4741 (match_operand:SI 1 "register_operand" "r")
4742 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4743 (match_operand:SI 3 "register_operand" "r"))
4745 (set (match_operand:DI 0 "register_operand" "=r")
4746 (zero_extend:DI (and:SI
4748 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4750 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4751 [(set_attr "type" "logics_shift_imm")]
4754 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4755 [(set (reg:CC_NZ CC_REGNUM)
4759 (match_operand:GPI 0 "register_operand" "r")
4760 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4761 (match_operand:GPI 2 "register_operand" "r"))
4764 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4765 [(set_attr "type" "logics_shift_imm")]
4768 (define_insn "clz<mode>2"
4769 [(set (match_operand:GPI 0 "register_operand" "=r")
4770 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4772 "clz\\t%<w>0, %<w>1"
4773 [(set_attr "type" "clz")]
4776 (define_expand "ffs<mode>2"
4777 [(match_operand:GPI 0 "register_operand")
4778 (match_operand:GPI 1 "register_operand")]
4781 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4782 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4784 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4785 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4786 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4791 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4798 (define_expand "popcount<mode>2"
4799 [(match_operand:GPI 0 "register_operand")
4800 (match_operand:GPI 1 "register_operand")]
4803 rtx v = gen_reg_rtx (V8QImode);
4804 rtx v1 = gen_reg_rtx (V8QImode);
4805 rtx r = gen_reg_rtx (QImode);
4806 rtx in = operands[1];
4807 rtx out = operands[0];
4808 if(<MODE>mode == SImode)
4811 tmp = gen_reg_rtx (DImode);
4812 /* If we have SImode, zero extend to DImode, pop count does
4813 not change if we have extra zeros. */
4814 emit_insn (gen_zero_extendsidi2 (tmp, in));
4817 emit_move_insn (v, gen_lowpart (V8QImode, in));
4818 emit_insn (gen_popcountv8qi2 (v1, v));
4819 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4820 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4824 (define_insn "clrsb<mode>2"
4825 [(set (match_operand:GPI 0 "register_operand" "=r")
4826 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4828 "cls\\t%<w>0, %<w>1"
4829 [(set_attr "type" "clz")]
4832 (define_insn "rbit<mode>2"
4833 [(set (match_operand:GPI 0 "register_operand" "=r")
4834 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4836 "rbit\\t%<w>0, %<w>1"
4837 [(set_attr "type" "rbit")]
4840 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4841 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4842 ;; expression and split after reload to enable scheduling them apart if
4845 (define_insn_and_split "ctz<mode>2"
4846 [(set (match_operand:GPI 0 "register_operand" "=r")
4847 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4853 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4854 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4858 (define_insn "*and<mode>_compare0"
4859 [(set (reg:CC_NZ CC_REGNUM)
4861 (match_operand:SHORT 0 "register_operand" "r")
4864 "tst\\t%<w>0, <short_mask>"
4865 [(set_attr "type" "alus_imm")]
4868 (define_insn "*ands<GPI:mode>_compare0"
4869 [(set (reg:CC_NZ CC_REGNUM)
4871 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4873 (set (match_operand:GPI 0 "register_operand" "=r")
4874 (zero_extend:GPI (match_dup 1)))]
4876 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4877 [(set_attr "type" "alus_imm")]
4880 (define_insn "*and<mode>3nr_compare0"
4881 [(set (reg:CC_NZ CC_REGNUM)
4883 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4884 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4890 [(set_attr "type" "logics_reg,logics_imm")]
4894 [(set (reg:CC_NZ CC_REGNUM)
4896 (and:GPI (match_operand:GPI 0 "register_operand")
4897 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4899 (clobber (match_operand:SI 2 "register_operand"))]
4901 [(set (match_dup 2) (match_dup 1))
4902 (set (reg:CC_NZ CC_REGNUM)
4904 (and:GPI (match_dup 0)
4909 (define_insn "*and<mode>3nr_compare0_zextract"
4910 [(set (reg:CC_NZ CC_REGNUM)
4912 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4913 (match_operand:GPI 1 "const_int_operand" "n")
4914 (match_operand:GPI 2 "const_int_operand" "n"))
4916 "INTVAL (operands[1]) > 0
4917 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4918 <= GET_MODE_BITSIZE (<MODE>mode))
4919 && aarch64_bitmask_imm (
4920 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4925 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4926 return "tst\\t%<w>0, %1";
4928 [(set_attr "type" "logics_shift_imm")]
4931 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4932 [(set (reg:CC_NZ CC_REGNUM)
4935 (match_operand:GPI 0 "register_operand" "r")
4936 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4937 (match_operand:GPI 2 "register_operand" "r"))
4940 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4941 [(set_attr "type" "logics_shift_imm")]
4945 [(set (reg:CC_NZ CC_REGNUM)
4948 (match_operand:GPI 0 "register_operand")
4949 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4950 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4952 (clobber (match_operand:SI 3 "register_operand"))]
4954 [(set (match_dup 3) (match_dup 2))
4955 (set (reg:CC_NZ CC_REGNUM)
4964 ;; -------------------------------------------------------------------
4966 ;; -------------------------------------------------------------------
4968 (define_expand "<optab><mode>3"
4969 [(set (match_operand:GPI 0 "register_operand")
4970 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4971 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4974 if (CONST_INT_P (operands[2]))
4976 operands[2] = GEN_INT (INTVAL (operands[2])
4977 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4979 if (operands[2] == const0_rtx)
4981 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4988 (define_expand "ashl<mode>3"
4989 [(set (match_operand:SHORT 0 "register_operand")
4990 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4991 (match_operand:QI 2 "const_int_operand")))]
4994 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4996 if (operands[2] == const0_rtx)
4998 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5004 (define_expand "rotr<mode>3"
5005 [(set (match_operand:GPI 0 "register_operand")
5006 (rotatert:GPI (match_operand:GPI 1 "register_operand")
5007 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5010 if (CONST_INT_P (operands[2]))
5012 operands[2] = GEN_INT (INTVAL (operands[2])
5013 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5015 if (operands[2] == const0_rtx)
5017 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5024 (define_expand "rotl<mode>3"
5025 [(set (match_operand:GPI 0 "register_operand")
5026 (rotatert:GPI (match_operand:GPI 1 "register_operand")
5027 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5030 /* (SZ - cnt) % SZ == -cnt % SZ */
5031 if (CONST_INT_P (operands[2]))
5033 operands[2] = GEN_INT ((-INTVAL (operands[2]))
5034 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5035 if (operands[2] == const0_rtx)
5037 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5042 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5047 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5048 ;; they truncate the shift/rotate amount by the size of the registers they
5049 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5050 ;; such redundant masking instructions. GCC can do that automatically when
5051 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5052 ;; because some of the SISD shift alternatives don't perform this truncations.
5053 ;; So this pattern exists to catch such cases.
5055 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5056 [(set (match_operand:GPI 0 "register_operand" "=r")
5058 (match_operand:GPI 1 "register_operand" "r")
5059 (match_operator 4 "subreg_lowpart_operator"
5060 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5061 (match_operand 3 "const_int_operand" "n"))])))]
5062 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5063 "<shift>\t%<w>0, %<w>1, %<w>2"
5064 [(set_attr "type" "shift_reg")]
5067 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5068 [(set (match_operand:GPI 0 "register_operand" "=&r")
5070 (match_operand:GPI 1 "register_operand" "r")
5071 (match_operator 4 "subreg_lowpart_operator"
5072 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5073 (match_operand 3 "const_int_operand" "n")))])))]
5074 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5079 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5080 : lowpart_subreg (SImode, operands[0], <MODE>mode));
5081 emit_insn (gen_negsi2 (tmp, operands[2]));
5083 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5084 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5085 SUBREG_BYTE (operands[4]));
5086 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5091 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5092 [(set (match_operand:GPI 0 "register_operand" "=&r")
5094 (match_operand:GPI 1 "register_operand" "r")
5095 (minus:QI (match_operand 2 "const_int_operand" "n")
5096 (match_operator 5 "subreg_lowpart_operator"
5097 [(and:SI (match_operand:SI 3 "register_operand" "r")
5098 (match_operand 4 "const_int_operand" "n"))]))))]
5099 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5100 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5105 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5108 emit_insn (gen_negsi2 (tmp, operands[3]));
5110 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5111 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5112 SUBREG_BYTE (operands[5]));
5114 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5119 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5120 [(set (match_operand:DI 0 "register_operand" "=r")
5122 (match_operand:DI 1 "register_operand" "r")
5123 (match_operator 4 "subreg_lowpart_operator"
5124 [(and:SI (match_operand:SI 2 "register_operand" "r")
5125 (match_operand 3 "const_int_operand" "n"))])))]
5126 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5129 xop[0] = operands[0];
5130 xop[1] = operands[1];
5131 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5132 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5135 [(set_attr "type" "shift_reg")]
5138 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5139 [(set (match_operand:GPI 0 "register_operand" "=&r")
5141 (match_operand:GPI 1 "register_operand" "r")
5142 (minus:QI (match_operand 2 "const_int_operand" "n")
5143 (match_operand:QI 3 "register_operand" "r"))))]
5144 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5149 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5151 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5152 : gen_lowpart (SImode, operands[0]));
5154 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5156 rtx and_op = gen_rtx_AND (SImode, tmp,
5157 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5159 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5161 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5164 [(set_attr "length" "8")]
5167 ;; Logical left shift using SISD or Integer instruction
5168 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5169 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5171 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5172 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5175 lsl\t%<w>0, %<w>1, %2
5176 lsl\t%<w>0, %<w>1, %<w>2
5177 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5178 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5179 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5180 (set_attr "arch" "*,*,simd,simd")]
5183 ;; Logical right shift using SISD or Integer instruction
5184 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5185 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5187 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5188 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5189 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5192 lsr\t%<w>0, %<w>1, %2
5193 lsr\t%<w>0, %<w>1, %<w>2
5194 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5197 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5198 (set_attr "arch" "*,*,simd,simd,simd")]
5202 [(set (match_operand:DI 0 "aarch64_simd_register")
5204 (match_operand:DI 1 "aarch64_simd_register")
5205 (match_operand:QI 2 "aarch64_simd_register")))]
5206 "TARGET_SIMD && reload_completed"
5208 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5210 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5212 operands[3] = gen_lowpart (QImode, operands[0]);
5217 [(set (match_operand:SI 0 "aarch64_simd_register")
5219 (match_operand:SI 1 "aarch64_simd_register")
5220 (match_operand:QI 2 "aarch64_simd_register")))]
5221 "TARGET_SIMD && reload_completed"
5223 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5225 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5227 operands[3] = gen_lowpart (QImode, operands[0]);
5231 ;; Arithmetic right shift using SISD or Integer instruction
5232 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5233 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5235 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5236 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5237 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5240 asr\t%<w>0, %<w>1, %2
5241 asr\t%<w>0, %<w>1, %<w>2
5242 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5245 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5246 (set_attr "arch" "*,*,simd,simd,simd")]
5250 [(set (match_operand:DI 0 "aarch64_simd_register")
5252 (match_operand:DI 1 "aarch64_simd_register")
5253 (match_operand:QI 2 "aarch64_simd_register")))]
5254 "TARGET_SIMD && reload_completed"
5256 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5258 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5260 operands[3] = gen_lowpart (QImode, operands[0]);
5265 [(set (match_operand:SI 0 "aarch64_simd_register")
5267 (match_operand:SI 1 "aarch64_simd_register")
5268 (match_operand:QI 2 "aarch64_simd_register")))]
5269 "TARGET_SIMD && reload_completed"
5271 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5273 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5275 operands[3] = gen_lowpart (QImode, operands[0]);
5279 (define_insn "*aarch64_sisd_ushl"
5280 [(set (match_operand:DI 0 "register_operand" "=w")
5281 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5282 (match_operand:QI 2 "register_operand" "w")]
5285 "ushl\t%d0, %d1, %d2"
5286 [(set_attr "type" "neon_shift_reg")]
5289 (define_insn "*aarch64_ushl_2s"
5290 [(set (match_operand:SI 0 "register_operand" "=w")
5291 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5292 (match_operand:QI 2 "register_operand" "w")]
5295 "ushl\t%0.2s, %1.2s, %2.2s"
5296 [(set_attr "type" "neon_shift_reg")]
5299 (define_insn "*aarch64_sisd_sshl"
5300 [(set (match_operand:DI 0 "register_operand" "=w")
5301 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5302 (match_operand:QI 2 "register_operand" "w")]
5305 "sshl\t%d0, %d1, %d2"
5306 [(set_attr "type" "neon_shift_reg")]
5309 (define_insn "*aarch64_sshl_2s"
5310 [(set (match_operand:SI 0 "register_operand" "=w")
5311 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5312 (match_operand:QI 2 "register_operand" "w")]
5315 "sshl\t%0.2s, %1.2s, %2.2s"
5316 [(set_attr "type" "neon_shift_reg")]
5319 (define_insn "*aarch64_sisd_neg_qi"
5320 [(set (match_operand:QI 0 "register_operand" "=w")
5321 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5325 [(set_attr "type" "neon_neg")]
5329 (define_insn "*ror<mode>3_insn"
5330 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5332 (match_operand:GPI 1 "register_operand" "r,r")
5333 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5336 ror\\t%<w>0, %<w>1, %2
5337 ror\\t%<w>0, %<w>1, %<w>2"
5338 [(set_attr "type" "rotate_imm,shift_reg")]
5341 ;; zero_extend version of above
5342 (define_insn "*<optab>si3_insn_uxtw"
5343 [(set (match_operand:DI 0 "register_operand" "=r,r")
5344 (zero_extend:DI (SHIFT:SI
5345 (match_operand:SI 1 "register_operand" "r,r")
5346 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5349 <shift>\\t%w0, %w1, %2
5350 <shift>\\t%w0, %w1, %w2"
5351 [(set_attr "type" "bfx,shift_reg")]
5354 (define_insn "*<optab><mode>3_insn"
5355 [(set (match_operand:SHORT 0 "register_operand" "=r")
5356 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5357 (match_operand 2 "const_int_operand" "n")))]
5358 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5360 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5361 return "<bfshift>\t%w0, %w1, %2, %3";
5363 [(set_attr "type" "bfx")]
5366 (define_insn "*extr<mode>5_insn"
5367 [(set (match_operand:GPI 0 "register_operand" "=r")
5368 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5369 (match_operand 3 "const_int_operand" "n"))
5370 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5371 (match_operand 4 "const_int_operand" "n"))))]
5372 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5373 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5374 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5375 [(set_attr "type" "rotate_imm")]
5378 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5379 ;; so we have to match both orderings.
5380 (define_insn "*extr<mode>5_insn_alt"
5381 [(set (match_operand:GPI 0 "register_operand" "=r")
5382 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5383 (match_operand 4 "const_int_operand" "n"))
5384 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5385 (match_operand 3 "const_int_operand" "n"))))]
5386 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5387 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5388 == GET_MODE_BITSIZE (<MODE>mode))"
5389 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5390 [(set_attr "type" "rotate_imm")]
5393 ;; zero_extend version of the above
5394 (define_insn "*extrsi5_insn_uxtw"
5395 [(set (match_operand:DI 0 "register_operand" "=r")
5397 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5398 (match_operand 3 "const_int_operand" "n"))
5399 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5400 (match_operand 4 "const_int_operand" "n")))))]
5401 "UINTVAL (operands[3]) < 32 &&
5402 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5403 "extr\\t%w0, %w1, %w2, %4"
5404 [(set_attr "type" "rotate_imm")]
5407 (define_insn "*extrsi5_insn_uxtw_alt"
5408 [(set (match_operand:DI 0 "register_operand" "=r")
5410 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5411 (match_operand 4 "const_int_operand" "n"))
5412 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5413 (match_operand 3 "const_int_operand" "n")))))]
5414 "UINTVAL (operands[3]) < 32 &&
5415 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5416 "extr\\t%w0, %w1, %w2, %4"
5417 [(set_attr "type" "rotate_imm")]
5420 (define_insn "*ror<mode>3_insn"
5421 [(set (match_operand:GPI 0 "register_operand" "=r")
5422 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5423 (match_operand 2 "const_int_operand" "n")))]
5424 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5426 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5427 return "ror\\t%<w>0, %<w>1, %3";
5429 [(set_attr "type" "rotate_imm")]
5432 ;; zero_extend version of the above
5433 (define_insn "*rorsi3_insn_uxtw"
5434 [(set (match_operand:DI 0 "register_operand" "=r")
5436 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5437 (match_operand 2 "const_int_operand" "n"))))]
5438 "UINTVAL (operands[2]) < 32"
5440 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5441 return "ror\\t%w0, %w1, %3";
5443 [(set_attr "type" "rotate_imm")]
5446 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5447 [(set (match_operand:GPI 0 "register_operand" "=r")
5449 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5450 (match_operand 2 "const_int_operand" "n"))))]
5451 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5453 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5454 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5456 [(set_attr "type" "bfx")]
5459 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5460 [(set (match_operand:GPI 0 "register_operand" "=r")
5462 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5463 (match_operand 2 "const_int_operand" "n"))))]
5464 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5466 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5467 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5469 [(set_attr "type" "bfx")]
5472 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5473 [(set (match_operand:GPI 0 "register_operand" "=r")
5475 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5476 (match_operand 2 "const_int_operand" "n"))))]
5477 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5479 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5480 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5482 [(set_attr "type" "bfx")]
5485 ;; -------------------------------------------------------------------
5487 ;; -------------------------------------------------------------------
5489 (define_expand "<optab>"
5490 [(set (match_operand:DI 0 "register_operand")
5491 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5493 "aarch64_simd_shift_imm_offset_di")
5494 (match_operand 3 "aarch64_simd_shift_imm_di")))]
5497 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5498 1, GET_MODE_BITSIZE (DImode) - 1))
5504 (define_insn "*<optab><mode>"
5505 [(set (match_operand:GPI 0 "register_operand" "=r")
5506 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5508 "aarch64_simd_shift_imm_offset_<mode>" "n")
5510 "aarch64_simd_shift_imm_<mode>" "n")))]
5511 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5512 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5513 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5514 [(set_attr "type" "bfx")]
5517 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5518 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5520 [(set (match_operand:DI 0 "register_operand")
5521 (zero_extract:DI (match_operand:DI 1 "register_operand")
5523 "aarch64_simd_shift_imm_offset_di")
5525 "aarch64_simd_shift_imm_di")))]
5526 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5527 GET_MODE_BITSIZE (DImode) - 1)
5528 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5529 == GET_MODE_BITSIZE (SImode)"
5531 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5533 operands[4] = gen_lowpart (SImode, operands[1]);
5537 ;; Bitfield Insert (insv)
5538 (define_expand "insv<mode>"
5539 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5540 (match_operand 1 "const_int_operand")
5541 (match_operand 2 "const_int_operand"))
5542 (match_operand:GPI 3 "general_operand"))]
5545 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5546 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5547 rtx value = operands[3];
5549 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5552 if (CONST_INT_P (value))
5554 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5556 /* Prefer AND/OR for inserting all zeros or all ones. */
5557 if ((UINTVAL (value) & mask) == 0
5558 || (UINTVAL (value) & mask) == mask)
5561 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5562 if (width == 16 && (pos % 16) == 0)
5565 operands[3] = force_reg (<MODE>mode, value);
5568 (define_insn "*insv_reg<mode>"
5569 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5570 (match_operand 1 "const_int_operand" "n")
5571 (match_operand 2 "const_int_operand" "n"))
5572 (match_operand:GPI 3 "register_operand" "r"))]
5573 "!(UINTVAL (operands[1]) == 0
5574 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5575 > GET_MODE_BITSIZE (<MODE>mode)))"
5576 "bfi\\t%<w>0, %<w>3, %2, %1"
5577 [(set_attr "type" "bfm")]
5580 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5581 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5582 (match_operand 1 "const_int_operand" "n")
5583 (match_operand 2 "const_int_operand" "n"))
5584 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5585 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5586 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5587 [(set_attr "type" "bfm")]
5590 ;; Match a bfi instruction where the shift of OP3 means that we are
5591 ;; actually copying the least significant bits of OP3 into OP0 by way
5592 ;; of the AND masks and the IOR instruction. A similar instruction
5593 ;; with the two parts of the IOR swapped around was never triggered
5594 ;; in a bootstrap build and test of GCC so it was not included.
5596 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5597 [(set (match_operand:GPI 0 "register_operand" "=r")
5598 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5599 (match_operand:GPI 2 "const_int_operand" "n"))
5600 (and:GPI (ashift:GPI
5601 (match_operand:GPI 3 "register_operand" "r")
5602 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5603 (match_operand:GPI 5 "const_int_operand" "n"))))]
5604 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5605 UINTVAL (operands[4]),
5606 UINTVAL(operands[5]))"
5607 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5608 [(set_attr "type" "bfm")]
5611 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5612 [(set (match_operand:GPI 0 "register_operand" "=r")
5613 (ior:GPI (and:GPI (ashift:GPI
5614 (match_operand:GPI 1 "register_operand" "r")
5615 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5616 (match_operand:GPI 3 "const_int_operand" "n"))
5617 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5618 (match_operand:GPI 5 "const_int_operand" "n"))))]
5619 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5620 UINTVAL (operands[2]),
5621 UINTVAL(operands[3]))"
5622 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5623 [(set_attr "type" "bfm")]
5626 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5627 ;; the shift is large enough to remove the need for an AND instruction.
5629 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5630 [(set (match_operand:GPI 0 "register_operand" "=r")
5631 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5632 (match_operand:GPI 2 "const_int_operand" "n"))
5634 (match_operand:GPI 3 "register_operand" "r")
5635 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5636 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5637 UINTVAL (operands[4]),
5638 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5640 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5641 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5643 [(set_attr "type" "bfm")]
5646 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5647 [(set (match_operand:GPI 0 "register_operand" "=r")
5648 (ior:GPI (ashift:GPI
5649 (match_operand:GPI 1 "register_operand" "r")
5650 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5651 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5652 (match_operand:GPI 4 "const_int_operand" "n"))))]
5653 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5654 UINTVAL (operands[2]),
5655 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5657 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5658 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5660 [(set_attr "type" "bfm")]
5663 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5664 ;; copying the least significant bits of OP3 to OP0. We need two versions
5665 ;; of the instruction to handle different checks on the constant values.
5667 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5668 [(set (match_operand:GPI 0 "register_operand" "=r")
5669 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5670 (match_operand:GPI 2 "const_int_operand" "n"))
5671 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5672 (match_operand:GPI 4 "const_int_operand" "n"))))]
5673 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5674 UINTVAL (operands[4]))"
5675 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5676 [(set_attr "type" "bfm")]
5679 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5680 [(set (match_operand:GPI 0 "register_operand" "=r")
5681 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5682 (match_operand:GPI 4 "const_int_operand" "n"))
5683 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5684 (match_operand:GPI 2 "const_int_operand" "n"))))]
5685 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5686 UINTVAL (operands[4]))"
5687 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5688 [(set_attr "type" "bfm")]
5691 (define_insn "*extr_insv_lower_reg<mode>"
5692 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5693 (match_operand 1 "const_int_operand" "n")
5695 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5697 (match_operand 3 "const_int_operand" "n")))]
5698 "!(UINTVAL (operands[1]) == 0
5699 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5700 > GET_MODE_BITSIZE (<MODE>mode)))"
5701 "bfxil\\t%<w>0, %<w>2, %3, %1"
5702 [(set_attr "type" "bfm")]
5705 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5706 [(set (match_operand:GPI 0 "register_operand" "=r")
5707 (ashift:GPI (ANY_EXTEND:GPI
5708 (match_operand:ALLX 1 "register_operand" "r"))
5709 (match_operand 2 "const_int_operand" "n")))]
5710 "UINTVAL (operands[2]) < <GPI:sizen>"
5712 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5713 ? GEN_INT (<ALLX:sizen>)
5714 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5715 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5717 [(set_attr "type" "bfx")]
5720 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5722 (define_insn "*andim_ashift<mode>_bfiz"
5723 [(set (match_operand:GPI 0 "register_operand" "=r")
5724 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5725 (match_operand 2 "const_int_operand" "n"))
5726 (match_operand 3 "const_int_operand" "n")))]
5727 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5728 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5729 [(set_attr "type" "bfx")]
5732 ;; Match sbfiz pattern in a shift left + shift right operation.
5734 (define_insn "*ashift<mode>_extv_bfiz"
5735 [(set (match_operand:GPI 0 "register_operand" "=r")
5736 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5737 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5739 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5740 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5741 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5742 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5743 [(set_attr "type" "bfx")]
5746 ;; When the bit position and width of the equivalent extraction add up to 32
5747 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5748 ;; zero-extension of the X-reg.
5750 [(set (match_operand:DI 0 "register_operand")
5751 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5752 (match_operand 2 "const_int_operand"))
5753 (match_operand 3 "const_int_operand")))]
5754 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5755 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5756 == GET_MODE_BITSIZE (SImode)"
5758 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5760 operands[4] = gen_lowpart (SImode, operands[1]);
5764 (define_insn "bswap<mode>2"
5765 [(set (match_operand:GPI 0 "register_operand" "=r")
5766 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5768 "rev\\t%<w>0, %<w>1"
5769 [(set_attr "type" "rev")]
5772 (define_insn "bswaphi2"
5773 [(set (match_operand:HI 0 "register_operand" "=r")
5774 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5777 [(set_attr "type" "rev")]
5780 (define_insn "*aarch64_bfxil<mode>"
5781 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5782 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5783 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5784 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5785 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5786 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5787 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5788 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5790 switch (which_alternative)
5793 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5794 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5796 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5797 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5802 [(set_attr "type" "bfm")]
5805 ; Zero-extended version of above (aarch64_bfxil)
5806 (define_insn "*aarch64_bfxilsi_uxtw"
5807 [(set (match_operand:DI 0 "register_operand" "=r,r")
5808 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5810 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5811 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5812 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5813 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5814 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5815 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5817 switch (which_alternative)
5820 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5821 return "bfxil\\t%0, %1, 0, %3";
5823 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5824 return "bfxil\\t%0, %2, 0, %3";
5829 [(set_attr "type" "bfm")]
5832 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5833 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5834 ;; each valid permutation.
5836 (define_insn "rev16<mode>2"
5837 [(set (match_operand:GPI 0 "register_operand" "=r")
5838 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5840 (match_operand:GPI 3 "const_int_operand" "n"))
5841 (and:GPI (lshiftrt:GPI (match_dup 1)
5843 (match_operand:GPI 2 "const_int_operand" "n"))))]
5844 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5845 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5846 "rev16\\t%<w>0, %<w>1"
5847 [(set_attr "type" "rev")]
5850 (define_insn "rev16<mode>2_alt"
5851 [(set (match_operand:GPI 0 "register_operand" "=r")
5852 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5854 (match_operand:GPI 2 "const_int_operand" "n"))
5855 (and:GPI (ashift:GPI (match_dup 1)
5857 (match_operand:GPI 3 "const_int_operand" "n"))))]
5858 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5859 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5860 "rev16\\t%<w>0, %<w>1"
5861 [(set_attr "type" "rev")]
5864 ;; zero_extend version of above
5865 (define_insn "*bswapsi2_uxtw"
5866 [(set (match_operand:DI 0 "register_operand" "=r")
5867 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5870 [(set_attr "type" "rev")]
5873 ;; -------------------------------------------------------------------
5874 ;; Floating-point intrinsics
5875 ;; -------------------------------------------------------------------
5877 ;; frint floating-point round to integral standard patterns.
5878 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5880 (define_insn "<frint_pattern><mode>2"
5881 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5882 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5885 "frint<frint_suffix>\\t%<s>0, %<s>1"
5886 [(set_attr "type" "f_rint<stype>")]
5889 ;; frcvt floating-point round to integer and convert standard patterns.
5890 ;; Expands to lbtrunc, lceil, lfloor, lround.
5891 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5892 [(set (match_operand:GPI 0 "register_operand" "=r")
5894 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5897 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5898 [(set_attr "type" "f_cvtf2i")]
5901 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5902 [(set (match_operand:GPI 0 "register_operand" "=r")
5905 (match_operand:GPF 1 "register_operand" "w")
5906 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5908 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5909 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5911 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5913 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5914 output_asm_insn (buf, operands);
5917 [(set_attr "type" "f_cvtf2i")]
5920 ;; fma - expand fma into patterns with the accumulator operand first since
5921 ;; reusing the accumulator results in better register allocation.
5922 ;; The register allocator considers copy preferences in operand order,
5923 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5925 (define_expand "fma<mode>4"
5926 [(set (match_operand:GPF_F16 0 "register_operand")
5927 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5928 (match_operand:GPF_F16 2 "register_operand")
5929 (match_operand:GPF_F16 3 "register_operand")))]
5933 (define_insn "*aarch64_fma<mode>4"
5934 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5935 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5936 (match_operand:GPF_F16 3 "register_operand" "w")
5937 (match_operand:GPF_F16 1 "register_operand" "w")))]
5939 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5940 [(set_attr "type" "fmac<stype>")]
5943 (define_expand "fnma<mode>4"
5944 [(set (match_operand:GPF_F16 0 "register_operand")
5946 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5947 (match_operand:GPF_F16 2 "register_operand")
5948 (match_operand:GPF_F16 3 "register_operand")))]
5952 (define_insn "*aarch64_fnma<mode>4"
5953 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5955 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5956 (match_operand:GPF_F16 3 "register_operand" "w")
5957 (match_operand:GPF_F16 1 "register_operand" "w")))]
5959 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5960 [(set_attr "type" "fmac<stype>")]
5964 (define_expand "fms<mode>4"
5965 [(set (match_operand:GPF 0 "register_operand")
5966 (fma:GPF (match_operand:GPF 1 "register_operand")
5967 (match_operand:GPF 2 "register_operand")
5968 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5972 (define_insn "*aarch64_fms<mode>4"
5973 [(set (match_operand:GPF 0 "register_operand" "=w")
5974 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5975 (match_operand:GPF 3 "register_operand" "w")
5976 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5978 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5979 [(set_attr "type" "fmac<s>")]
5982 (define_expand "fnms<mode>4"
5983 [(set (match_operand:GPF 0 "register_operand")
5984 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5985 (match_operand:GPF 2 "register_operand")
5986 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5990 (define_insn "*aarch64_fnms<mode>4"
5991 [(set (match_operand:GPF 0 "register_operand" "=w")
5992 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5993 (match_operand:GPF 3 "register_operand" "w")
5994 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5996 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5997 [(set_attr "type" "fmac<s>")]
6000 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
6001 (define_insn "*aarch64_fnmadd<mode>4"
6002 [(set (match_operand:GPF 0 "register_operand" "=w")
6003 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6004 (match_operand:GPF 3 "register_operand" "w")
6005 (match_operand:GPF 1 "register_operand" "w"))))]
6006 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
6007 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6008 [(set_attr "type" "fmac<s>")]
6011 ;; -------------------------------------------------------------------
6012 ;; Floating-point conversions
6013 ;; -------------------------------------------------------------------
6015 (define_insn "extendsfdf2"
6016 [(set (match_operand:DF 0 "register_operand" "=w")
6017 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6020 [(set_attr "type" "f_cvt")]
6023 (define_insn "extendhfsf2"
6024 [(set (match_operand:SF 0 "register_operand" "=w")
6025 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6028 [(set_attr "type" "f_cvt")]
6031 (define_insn "extendhfdf2"
6032 [(set (match_operand:DF 0 "register_operand" "=w")
6033 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6036 [(set_attr "type" "f_cvt")]
6039 (define_insn "truncdfsf2"
6040 [(set (match_operand:SF 0 "register_operand" "=w")
6041 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6044 [(set_attr "type" "f_cvt")]
6047 (define_insn "truncsfhf2"
6048 [(set (match_operand:HF 0 "register_operand" "=w")
6049 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6052 [(set_attr "type" "f_cvt")]
6055 (define_insn "truncdfhf2"
6056 [(set (match_operand:HF 0 "register_operand" "=w")
6057 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6060 [(set_attr "type" "f_cvt")]
6063 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6064 ;; and making r = w more expensive
6066 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6067 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6068 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6071 fcvtz<su>\t%<s>0, %<s>1
6072 fcvtz<su>\t%<w>0, %<s>1"
6073 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6076 ;; Convert HF -> SI or DI
6078 (define_insn "<optab>_trunchf<GPI:mode>2"
6079 [(set (match_operand:GPI 0 "register_operand" "=r")
6080 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6082 "fcvtz<su>\t%<w>0, %h1"
6083 [(set_attr "type" "f_cvtf2i")]
6086 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6087 ;; input in a fp register and output in a integer register
6089 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6090 [(set (match_operand:GPI 0 "register_operand" "=r")
6091 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6093 "fcvtz<su>\t%<w>0, %<fpw>1"
6094 [(set_attr "type" "f_cvtf2i")]
6097 (define_insn "*fix_to_zero_extend<mode>di2"
6098 [(set (match_operand:DI 0 "register_operand" "=r")
6101 (match_operand:GPF 1 "register_operand" "w"))))]
6103 "fcvtzu\t%w0, %<s>1"
6104 [(set_attr "type" "f_cvtf2i")]
6107 ;; Equal width integer to fp and multiply combine.
6108 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6109 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6110 (mult:GPF (FLOATUORS:GPF
6111 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6112 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6115 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6116 switch (which_alternative)
6119 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6121 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6126 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6127 (set_attr "arch" "simd,fp")]
6130 ;; Unequal width integer to fp and multiply combine.
6131 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6132 [(set (match_operand:GPF 0 "register_operand" "=w")
6133 (mult:GPF (FLOATUORS:GPF
6134 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6135 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6138 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6139 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6141 [(set_attr "type" "f_cvti2f")]
6144 ;; Equal width integer to fp conversion.
6145 (define_insn "<optab><fcvt_target><GPF:mode>2"
6146 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6147 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6150 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6151 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6152 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6153 (set_attr "arch" "simd,fp")]
6156 ;; Unequal width integer to fp conversions.
6157 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6158 [(set (match_operand:GPF 0 "register_operand" "=w")
6159 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6161 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6162 [(set_attr "type" "f_cvti2f")]
6165 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6166 ;; midend will arrange for an SImode conversion to HFmode to first go
6167 ;; through DFmode, then to HFmode. But first it will try converting
6168 ;; to DImode then down, which would match our DImode pattern below and
6169 ;; give very poor code-generation. So, we must provide our own emulation
6170 ;; of the mid-end logic.
6172 (define_insn "aarch64_fp16_<optab><mode>hf2"
6173 [(set (match_operand:HF 0 "register_operand" "=w")
6174 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6176 "<su_optab>cvtf\t%h0, %<w>1"
6177 [(set_attr "type" "f_cvti2f")]
6180 (define_expand "<optab>sihf2"
6181 [(set (match_operand:HF 0 "register_operand")
6182 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6185 if (TARGET_FP_F16INST)
6186 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6189 rtx convert_target = gen_reg_rtx (DFmode);
6190 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6191 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6197 ;; For DImode there is no wide enough floating-point mode that we
6198 ;; can convert through natively (TFmode would work, but requires a library
6199 ;; call). However, we know that any value >= 65504 will be rounded
6200 ;; to infinity on conversion. This is well within the range of SImode, so
6202 ;; Saturate to SImode.
6203 ;; Convert from that to DFmode
6204 ;; Convert from that to HFmode (phew!).
6205 ;; Note that the saturation to SImode requires the SIMD extensions. If
6206 ;; we ever need to provide this pattern where the SIMD extensions are not
6207 ;; available, we would need a different approach.
6209 (define_expand "<optab>dihf2"
6210 [(set (match_operand:HF 0 "register_operand")
6211 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6212 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6214 if (TARGET_FP_F16INST)
6215 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6218 rtx sat_target = gen_reg_rtx (SImode);
6219 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6220 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6227 ;; Convert between fixed-point and floating-point (scalar modes)
6229 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6230 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6231 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6232 (match_operand:SI 2 "immediate_operand" "i, i")]
6236 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6237 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6238 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6239 (set_attr "arch" "fp,simd")]
6242 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6243 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6244 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6245 (match_operand:SI 2 "immediate_operand" "i, i")]
6249 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6250 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6251 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6252 (set_attr "arch" "fp,simd")]
6255 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6256 [(set (match_operand:GPI 0 "register_operand" "=r")
6257 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6258 (match_operand:SI 2 "immediate_operand" "i")]
6261 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6262 [(set_attr "type" "f_cvtf2i")]
6265 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6266 [(set (match_operand:HF 0 "register_operand" "=w")
6267 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6268 (match_operand:SI 2 "immediate_operand" "i")]
6271 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6272 [(set_attr "type" "f_cvti2f")]
6275 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6276 [(set (match_operand:HI 0 "register_operand" "=w")
6277 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6278 (match_operand:SI 2 "immediate_operand" "i")]
6281 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6282 [(set_attr "type" "neon_fp_to_int_s")]
6285 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6286 [(set (match_operand:HF 0 "register_operand" "=w")
6287 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6288 (match_operand:SI 2 "immediate_operand" "i")]
6291 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6292 [(set_attr "type" "neon_int_to_fp_s")]
6295 ;; -------------------------------------------------------------------
6296 ;; Floating-point arithmetic
6297 ;; -------------------------------------------------------------------
6299 (define_insn "add<mode>3"
6300 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6302 (match_operand:GPF_F16 1 "register_operand" "w")
6303 (match_operand:GPF_F16 2 "register_operand" "w")))]
6305 "fadd\\t%<s>0, %<s>1, %<s>2"
6306 [(set_attr "type" "fadd<stype>")]
6309 (define_insn "sub<mode>3"
6310 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6312 (match_operand:GPF_F16 1 "register_operand" "w")
6313 (match_operand:GPF_F16 2 "register_operand" "w")))]
6315 "fsub\\t%<s>0, %<s>1, %<s>2"
6316 [(set_attr "type" "fadd<stype>")]
6319 (define_insn "mul<mode>3"
6320 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6322 (match_operand:GPF_F16 1 "register_operand" "w")
6323 (match_operand:GPF_F16 2 "register_operand" "w")))]
6325 "fmul\\t%<s>0, %<s>1, %<s>2"
6326 [(set_attr "type" "fmul<stype>")]
6329 (define_insn "*fnmul<mode>3"
6330 [(set (match_operand:GPF 0 "register_operand" "=w")
6332 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6333 (match_operand:GPF 2 "register_operand" "w")))]
6334 "TARGET_FLOAT && !flag_rounding_math"
6335 "fnmul\\t%<s>0, %<s>1, %<s>2"
6336 [(set_attr "type" "fmul<s>")]
6339 (define_insn "*fnmul<mode>3"
6340 [(set (match_operand:GPF 0 "register_operand" "=w")
6342 (match_operand:GPF 1 "register_operand" "w")
6343 (match_operand:GPF 2 "register_operand" "w"))))]
6345 "fnmul\\t%<s>0, %<s>1, %<s>2"
6346 [(set_attr "type" "fmul<s>")]
6349 (define_expand "div<mode>3"
6350 [(set (match_operand:GPF_F16 0 "register_operand")
6351 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6352 (match_operand:GPF_F16 2 "register_operand")))]
6355 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6358 operands[1] = force_reg (<MODE>mode, operands[1]);
6361 (define_insn "*div<mode>3"
6362 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6363 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6364 (match_operand:GPF_F16 2 "register_operand" "w")))]
6366 "fdiv\\t%<s>0, %<s>1, %<s>2"
6367 [(set_attr "type" "fdiv<stype>")]
6370 (define_insn "neg<mode>2"
6371 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6372 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6374 "fneg\\t%<s>0, %<s>1"
6375 [(set_attr "type" "ffarith<stype>")]
6378 (define_expand "sqrt<mode>2"
6379 [(set (match_operand:GPF_F16 0 "register_operand")
6380 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6383 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6387 (define_insn "*sqrt<mode>2"
6388 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6389 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6391 "fsqrt\\t%<s>0, %<s>1"
6392 [(set_attr "type" "fsqrt<stype>")]
6395 (define_insn "abs<mode>2"
6396 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6397 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6399 "fabs\\t%<s>0, %<s>1"
6400 [(set_attr "type" "ffarith<stype>")]
6403 ;; Given that smax/smin do not specify the result when either input is NaN,
6404 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6407 (define_insn "smax<mode>3"
6408 [(set (match_operand:GPF 0 "register_operand" "=w")
6409 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6410 (match_operand:GPF 2 "register_operand" "w")))]
6412 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6413 [(set_attr "type" "f_minmax<s>")]
6416 (define_insn "smin<mode>3"
6417 [(set (match_operand:GPF 0 "register_operand" "=w")
6418 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6419 (match_operand:GPF 2 "register_operand" "w")))]
6421 "fminnm\\t%<s>0, %<s>1, %<s>2"
6422 [(set_attr "type" "f_minmax<s>")]
6425 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6426 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6427 ;; which implement the IEEE fmax ()/fmin () functions.
6428 (define_insn "<maxmin_uns><mode>3"
6429 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6430 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6431 (match_operand:GPF_F16 2 "register_operand" "w")]
6434 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6435 [(set_attr "type" "f_minmax<stype>")]
6438 (define_expand "lrint<GPF:mode><GPI:mode>2"
6439 [(match_operand:GPI 0 "register_operand")
6440 (match_operand:GPF 1 "register_operand")]
6442 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6443 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6445 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6446 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6447 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6452 ;; For copysign (x, y), we want to generate:
6454 ;; LDR d2, #(1 << 63)
6455 ;; BSL v2.8b, [y], [x]
6457 ;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6458 ;; we expect these operations to nearly always operate on
6459 ;; floating-point values, we do not want the operation to be
6460 ;; simplified into a bit-field insert operation that operates on the
6461 ;; integer side, since typically that would involve three inter-bank
6462 ;; register copies. As we do not expect copysign to be followed by
6463 ;; other logical operations on the result, it seems preferable to keep
6464 ;; this as an unspec operation, rather than exposing the underlying
6465 ;; logic to the compiler.
6467 (define_expand "copysign<GPF:mode>3"
6468 [(match_operand:GPF 0 "register_operand")
6469 (match_operand:GPF 1 "register_operand")
6470 (match_operand:GPF 2 "register_operand")]
6471 "TARGET_FLOAT && TARGET_SIMD"
6473 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6474 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6475 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6476 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6482 (define_insn "copysign<GPF:mode>3_insn"
6483 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6484 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6485 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6486 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6488 "TARGET_FLOAT && TARGET_SIMD"
6490 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6491 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6492 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6493 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6494 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6498 ;; For xorsign (x, y), we want to generate:
6501 ;; AND v3.8B, v1.8B, v2.8B
6502 ;; EOR v0.8B, v0.8B, v3.8B
6505 (define_expand "xorsign<mode>3"
6506 [(match_operand:GPF 0 "register_operand")
6507 (match_operand:GPF 1 "register_operand")
6508 (match_operand:GPF 2 "register_operand")]
6509 "TARGET_FLOAT && TARGET_SIMD"
6512 machine_mode imode = <V_INT_EQUIV>mode;
6513 rtx mask = gen_reg_rtx (imode);
6514 rtx op1x = gen_reg_rtx (imode);
6515 rtx op2x = gen_reg_rtx (imode);
6517 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6518 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6521 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6522 lowpart_subreg (imode, operands[2],
6524 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6525 lowpart_subreg (imode, operands[1],
6528 emit_move_insn (operands[0],
6529 lowpart_subreg (<MODE>mode, op1x, imode));
6534 ;; -------------------------------------------------------------------
6536 ;; -------------------------------------------------------------------
6537 ;; Reload Scalar Floating point modes from constant pool.
6538 ;; The AArch64 port doesn't have __int128 constant move support.
6539 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6540 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6541 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6542 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6543 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6546 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6547 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6552 ;; Reload Vector modes from constant pool.
6553 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6554 [(set (match_operand:VALL 0 "register_operand" "=w")
6555 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6556 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6559 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6560 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6565 (define_expand "@aarch64_reload_mov<mode>"
6566 [(set (match_operand:TX 0 "register_operand" "=w")
6567 (match_operand:TX 1 "register_operand" "w"))
6568 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6572 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6573 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6574 gen_aarch64_movtilow_tilow (op0, op1);
6575 gen_aarch64_movdi_tihigh (operands[2], op1);
6576 gen_aarch64_movtihigh_di (op0, operands[2]);
6581 ;; The following secondary reload helpers patterns are invoked
6582 ;; after or during reload as we don't want these patterns to start
6583 ;; kicking in during the combiner.
6585 (define_insn "@aarch64_movdi_<mode>low"
6586 [(set (match_operand:DI 0 "register_operand" "=r")
6587 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6588 (const_int 64) (const_int 0)))]
6589 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6591 [(set_attr "type" "f_mrc")
6592 (set_attr "length" "4")
6595 (define_insn "@aarch64_movdi_<mode>high"
6596 [(set (match_operand:DI 0 "register_operand" "=r")
6597 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6598 (const_int 64) (const_int 64)))]
6599 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6600 "fmov\\t%x0, %1.d[1]"
6601 [(set_attr "type" "f_mrc")
6602 (set_attr "length" "4")
6605 (define_insn "@aarch64_mov<mode>high_di"
6606 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6607 (const_int 64) (const_int 64))
6608 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6609 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6610 "fmov\\t%0.d[1], %x1"
6611 [(set_attr "type" "f_mcr")
6612 (set_attr "length" "4")
6615 (define_insn "@aarch64_mov<mode>low_di"
6616 [(set (match_operand:TX 0 "register_operand" "=w")
6617 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6618 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6620 [(set_attr "type" "f_mcr")
6621 (set_attr "length" "4")
6624 (define_insn "aarch64_movtilow_tilow"
6625 [(set (match_operand:TI 0 "register_operand" "=w")
6627 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6628 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6630 [(set_attr "type" "fmov")
6631 (set_attr "length" "4")
6634 ;; There is a deliberate reason why the parameters of high and lo_sum's
6635 ;; don't have modes for ADRP and ADD instructions. This is to allow high
6636 ;; and lo_sum's to be used with the labels defining the jump tables in
6639 (define_expand "add_losym"
6640 [(set (match_operand 0 "register_operand")
6641 (lo_sum (match_operand 1 "register_operand")
6642 (match_operand 2 "aarch64_valid_symref")))]
6645 machine_mode mode = GET_MODE (operands[0]);
6647 emit_insn ((mode == DImode
6649 : gen_add_losym_si) (operands[0],
6655 (define_insn "add_losym_<mode>"
6656 [(set (match_operand:P 0 "register_operand" "=r")
6657 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6658 (match_operand 2 "aarch64_valid_symref" "S")))]
6660 "add\\t%<w>0, %<w>1, :lo12:%c2"
6661 [(set_attr "type" "alu_imm")]
6664 (define_insn "ldr_got_small_<mode>"
6665 [(set (match_operand:PTR 0 "register_operand" "=r")
6666 (unspec:PTR [(mem:PTR (lo_sum:PTR
6667 (match_operand:PTR 1 "register_operand" "r")
6668 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6669 UNSPEC_GOTSMALLPIC))]
6671 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6672 [(set_attr "type" "load_<ldst_sz>")]
6675 (define_insn "ldr_got_small_sidi"
6676 [(set (match_operand:DI 0 "register_operand" "=r")
6678 (unspec:SI [(mem:SI (lo_sum:DI
6679 (match_operand:DI 1 "register_operand" "r")
6680 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6681 UNSPEC_GOTSMALLPIC)))]
6683 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6684 [(set_attr "type" "load_4")]
6687 (define_insn "ldr_got_small_28k_<mode>"
6688 [(set (match_operand:PTR 0 "register_operand" "=r")
6689 (unspec:PTR [(mem:PTR (lo_sum:PTR
6690 (match_operand:PTR 1 "register_operand" "r")
6691 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6692 UNSPEC_GOTSMALLPIC28K))]
6694 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6695 [(set_attr "type" "load_<ldst_sz>")]
6698 (define_insn "ldr_got_small_28k_sidi"
6699 [(set (match_operand:DI 0 "register_operand" "=r")
6701 (unspec:SI [(mem:SI (lo_sum:DI
6702 (match_operand:DI 1 "register_operand" "r")
6703 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6704 UNSPEC_GOTSMALLPIC28K)))]
6706 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6707 [(set_attr "type" "load_4")]
6710 (define_insn "ldr_got_tiny"
6711 [(set (match_operand:DI 0 "register_operand" "=r")
6712 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6713 UNSPEC_GOTTINYPIC))]
6716 [(set_attr "type" "load_8")]
6719 (define_insn "aarch64_load_tp_hard"
6720 [(set (match_operand:DI 0 "register_operand" "=r")
6721 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6723 "mrs\\t%0, tpidr_el0"
6724 [(set_attr "type" "mrs")]
6727 ;; The TLS ABI specifically requires that the compiler does not schedule
6728 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6729 ;; Therefore we treat the stubs as an atomic sequence.
6730 (define_expand "tlsgd_small_<mode>"
6731 [(parallel [(set (match_operand 0 "register_operand")
6732 (call (mem:DI (match_dup 2)) (const_int 1)))
6733 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6734 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6735 (clobber (reg:DI LR_REGNUM))])]
6738 operands[2] = aarch64_tls_get_addr ();
6741 (define_insn "*tlsgd_small_<mode>"
6742 [(set (match_operand 0 "register_operand" "")
6743 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6744 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6745 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6746 (clobber (reg:DI LR_REGNUM))
6749 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6750 [(set_attr "type" "call")
6751 (set_attr "length" "16")])
6753 (define_insn "tlsie_small_<mode>"
6754 [(set (match_operand:PTR 0 "register_operand" "=r")
6755 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6756 UNSPEC_GOTSMALLTLS))]
6758 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6759 [(set_attr "type" "load_4")
6760 (set_attr "length" "8")]
6763 (define_insn "tlsie_small_sidi"
6764 [(set (match_operand:DI 0 "register_operand" "=r")
6766 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6767 UNSPEC_GOTSMALLTLS)))]
6769 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6770 [(set_attr "type" "load_4")
6771 (set_attr "length" "8")]
6774 (define_insn "tlsie_tiny_<mode>"
6775 [(set (match_operand:PTR 0 "register_operand" "=&r")
6776 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6777 (match_operand:PTR 2 "register_operand" "r")]
6778 UNSPEC_GOTTINYTLS))]
6780 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6781 [(set_attr "type" "multiple")
6782 (set_attr "length" "8")]
6785 (define_insn "tlsie_tiny_sidi"
6786 [(set (match_operand:DI 0 "register_operand" "=&r")
6788 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6789 (match_operand:DI 2 "register_operand" "r")
6791 UNSPEC_GOTTINYTLS)))]
6793 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6794 [(set_attr "type" "multiple")
6795 (set_attr "length" "8")]
6798 (define_insn "tlsle12_<mode>"
6799 [(set (match_operand:P 0 "register_operand" "=r")
6800 (unspec:P [(match_operand:P 1 "register_operand" "r")
6801 (match_operand 2 "aarch64_tls_le_symref" "S")]
6804 "add\\t%<w>0, %<w>1, #%L2";
6805 [(set_attr "type" "alu_sreg")
6806 (set_attr "length" "4")]
6809 (define_insn "tlsle24_<mode>"
6810 [(set (match_operand:P 0 "register_operand" "=r")
6811 (unspec:P [(match_operand:P 1 "register_operand" "r")
6812 (match_operand 2 "aarch64_tls_le_symref" "S")]
6815 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6816 [(set_attr "type" "multiple")
6817 (set_attr "length" "8")]
6820 (define_insn "tlsle32_<mode>"
6821 [(set (match_operand:P 0 "register_operand" "=r")
6822 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6825 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6826 [(set_attr "type" "multiple")
6827 (set_attr "length" "8")]
6830 (define_insn "tlsle48_<mode>"
6831 [(set (match_operand:P 0 "register_operand" "=r")
6832 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6835 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6836 [(set_attr "type" "multiple")
6837 (set_attr "length" "12")]
6840 (define_expand "tlsdesc_small_<mode>"
6841 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6846 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6848 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6849 RTL_CONST_CALL_P (call) = 1;
6852 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6857 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6859 (define_insn "tlsdesc_small_advsimd_<mode>"
6860 [(set (reg:PTR R0_REGNUM)
6861 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6863 (clobber (reg:DI LR_REGNUM))
6864 (clobber (reg:CC CC_REGNUM))
6865 (clobber (match_scratch:DI 1 "=r"))
6866 (use (reg:DI FP_REGNUM))]
6867 "TARGET_TLS_DESC && !TARGET_SVE"
6868 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6869 [(set_attr "type" "call")
6870 (set_attr "length" "16")])
6872 ;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6873 ;; describing the extra call-preserved guarantees. This would work
6874 ;; for non-SVE too, but avoiding a call is probably better if we can.
6875 (define_insn "tlsdesc_small_sve_<mode>"
6876 [(set (reg:PTR R0_REGNUM)
6877 (call (mem:DI (unspec:PTR
6878 [(match_operand 0 "aarch64_valid_symref")]
6881 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
6882 (clobber (reg:DI LR_REGNUM))
6883 (clobber (match_scratch:DI 2 "=r"))]
6884 "TARGET_TLS_DESC && TARGET_SVE"
6885 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
6886 [(set_attr "type" "call")
6887 (set_attr "length" "16")])
6889 (define_insn "stack_tie"
6890 [(set (mem:BLK (scratch))
6891 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6892 (match_operand:DI 1 "register_operand" "rk")]
6896 [(set_attr "length" "0")]
6899 (define_insn "aarch64_fjcvtzs"
6900 [(set (match_operand:SI 0 "register_operand" "=r")
6901 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6904 "fjcvtzs\\t%w0, %d1"
6905 [(set_attr "type" "f_cvtf2i")]
6908 ;; Pointer authentication patterns are always provided. In architecture
6909 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6910 ;; This lets the user write portable software which authenticates pointers
6911 ;; when run on something which implements ARMv8.3-A, and which runs
6912 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6915 ;; Signing/Authenticating R30 using SP as the salt.
6917 (define_insn "<pauth_mnem_prefix>sp"
6918 [(set (reg:DI R30_REGNUM)
6919 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6921 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6924 ;; Signing/Authenticating X17 using X16 as the salt.
6926 (define_insn "<pauth_mnem_prefix>1716"
6927 [(set (reg:DI R17_REGNUM)
6928 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6930 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6933 ;; Stripping the signature in R30.
6935 (define_insn "xpaclri"
6936 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6938 "hint\t7 // xpaclri"
6941 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6942 ;; all of memory. This blocks insns from being moved across this point.
6944 (define_insn "blockage"
6945 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6948 [(set_attr "length" "0")
6949 (set_attr "type" "block")]
6952 (define_insn "probe_stack_range"
6953 [(set (match_operand:DI 0 "register_operand" "=rk")
6954 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6955 (match_operand:DI 2 "register_operand" "r")]
6956 UNSPECV_PROBE_STACK_RANGE))]
6959 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6961 [(set_attr "length" "32")]
6964 ;; This instruction is used to generate the stack clash stack adjustment and
6965 ;; probing loop. We can't change the control flow during prologue and epilogue
6966 ;; code generation. So we must emit a volatile unspec and expand it later on.
6968 (define_insn "@probe_sve_stack_clash_<mode>"
6969 [(set (match_operand:P 0 "register_operand" "=rk")
6970 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6971 (match_operand:P 2 "register_operand" "r")
6972 (match_operand:P 3 "const_int_operand" "n")
6973 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6974 UNSPECV_PROBE_STACK_RANGE))]
6977 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6978 operands[3], operands[4]);
6980 [(set_attr "length" "28")]
6983 ;; Named pattern for expanding thread pointer reference.
6984 (define_expand "get_thread_pointerdi"
6985 [(match_operand:DI 0 "register_operand")]
6988 rtx tmp = aarch64_load_tp (operands[0]);
6989 if (tmp != operands[0])
6990 emit_move_insn (operands[0], tmp);
6994 ;; Named patterns for stack smashing protection.
6995 (define_expand "stack_protect_set"
6996 [(match_operand 0 "memory_operand")
6997 (match_operand 1 "memory_operand")]
7000 machine_mode mode = GET_MODE (operands[0]);
7001 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7003 /* Generate access through the system register. */
7004 rtx tmp_reg = gen_reg_rtx (mode);
7007 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7008 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7009 GEN_INT (aarch64_stack_protector_guard_offset)));
7013 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7014 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7015 GEN_INT (aarch64_stack_protector_guard_offset)));
7018 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7021 emit_insn ((mode == DImode
7022 ? gen_stack_protect_set_di
7023 : gen_stack_protect_set_si) (operands[0], operands[1]));
7027 (define_insn "reg_stack_protect_address_<mode>"
7028 [(set (match_operand:PTR 0 "register_operand" "=r")
7029 (unspec:PTR [(const_int 0)]
7030 UNSPEC_SSP_SYSREG))]
7031 "aarch64_stack_protector_guard != SSP_GLOBAL"
7034 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7035 aarch64_stack_protector_guard_reg_str);
7036 output_asm_insn (buf, operands);
7039 [(set_attr "type" "mrs")])
7041 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7042 ;; canary value does not live beyond the life of this sequence.
7043 (define_insn "stack_protect_set_<mode>"
7044 [(set (match_operand:PTR 0 "memory_operand" "=m")
7045 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7047 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7049 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7050 [(set_attr "length" "12")
7051 (set_attr "type" "multiple")])
7053 (define_expand "stack_protect_test"
7054 [(match_operand 0 "memory_operand")
7055 (match_operand 1 "memory_operand")
7060 machine_mode mode = GET_MODE (operands[0]);
7062 result = gen_reg_rtx(mode);
7063 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7065 /* Generate access through the system register. The
7066 sequence we want here is the access
7067 of the stack offset to come with
7068 mrs scratch_reg, <system_register>
7069 add scratch_reg, scratch_reg, :lo12:offset. */
7070 rtx tmp_reg = gen_reg_rtx (mode);
7073 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7074 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7075 GEN_INT (aarch64_stack_protector_guard_offset)));
7079 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7080 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7081 GEN_INT (aarch64_stack_protector_guard_offset)));
7084 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7086 emit_insn ((mode == DImode
7087 ? gen_stack_protect_test_di
7088 : gen_stack_protect_test_si) (result,
7093 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7094 result, const0_rtx, operands[2]));
7096 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7097 result, const0_rtx, operands[2]));
7101 (define_insn "stack_protect_test_<mode>"
7102 [(set (match_operand:PTR 0 "register_operand" "=r")
7103 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7104 (match_operand:PTR 2 "memory_operand" "m")]
7106 (clobber (match_scratch:PTR 3 "=&r"))]
7108 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7109 [(set_attr "length" "12")
7110 (set_attr "type" "multiple")])
7112 ;; Write Floating-point Control Register.
7113 (define_insn "set_fpcr"
7114 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7117 [(set_attr "type" "mrs")])
7119 ;; Read Floating-point Control Register.
7120 (define_insn "get_fpcr"
7121 [(set (match_operand:SI 0 "register_operand" "=r")
7122 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7125 [(set_attr "type" "mrs")])
7127 ;; Write Floating-point Status Register.
7128 (define_insn "set_fpsr"
7129 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7132 [(set_attr "type" "mrs")])
7134 ;; Read Floating-point Status Register.
7135 (define_insn "get_fpsr"
7136 [(set (match_operand:SI 0 "register_operand" "=r")
7137 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7140 [(set_attr "type" "mrs")])
7143 ;; Define the subtract-one-and-jump insns so loop.c
7144 ;; knows what to generate.
7145 (define_expand "doloop_end"
7146 [(use (match_operand 0 "" "")) ; loop pseudo
7147 (use (match_operand 1 "" ""))] ; label
7148 "optimize > 0 && flag_modulo_sched"
7157 /* Currently SMS relies on the do-loop pattern to recognize loops
7158 where (1) the control part consists of all insns defining and/or
7159 using a certain 'count' register and (2) the loop count can be
7160 adjusted by modifying this register prior to the loop.
7161 ??? The possible introduction of a new block to initialize the
7162 new IV can potentially affect branch optimizations. */
7164 if (GET_MODE (operands[0]) != DImode)
7168 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7170 cmp = XVECEXP (PATTERN (insn), 0, 0);
7171 cc_reg = SET_DEST (cmp);
7172 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7173 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7174 emit_jump_insn (gen_rtx_SET (pc_rtx,
7175 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7180 ;; Track speculation through conditional branches. We assume that
7181 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7182 (define_insn "speculation_tracker"
7183 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7184 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7185 UNSPEC_SPECULATION_TRACKER))]
7188 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7189 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7192 [(set_attr "type" "csel")]
7195 ;; BTI <target> instructions
7196 (define_insn "bti_noarg"
7197 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7200 [(set_attr "type" "no_insn")]
7203 (define_insn "bti_c"
7204 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7207 [(set_attr "type" "no_insn")]
7210 (define_insn "bti_j"
7211 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7214 [(set_attr "type" "no_insn")]
7217 (define_insn "bti_jc"
7218 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7220 "hint\t38 // bti jc"
7221 [(set_attr "type" "no_insn")]
7224 ;; Hard speculation barrier.
7225 (define_insn "speculation_barrier"
7226 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7229 [(set_attr "length" "8")
7230 (set_attr "type" "block")
7231 (set_attr "speculation_barrier" "true")]
7234 ;; Support for __builtin_speculation_safe_value when we have speculation
7235 ;; tracking enabled. Use the speculation tracker to decide whether to
7236 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7237 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7238 [(set (match_operand:ALLI_TI 0 "register_operand")
7239 (unspec_volatile:ALLI_TI
7240 [(match_operand:ALLI_TI 1 "register_operand")
7241 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7242 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7243 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7247 if (operands[2] == const0_rtx)
7250 if (<MODE>mode == TImode)
7251 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7253 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7255 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7263 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7264 ;; encoding for CSDB, but will work in older versions of the assembler.
7265 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7266 [(set (match_operand:ALLI 0 "register_operand" "=r")
7267 (unspec_volatile:ALLI
7268 [(match_operand:ALLI 1 "register_operand" "r")
7269 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7270 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7271 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7274 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7275 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7279 [(set_attr "length" "12")
7280 (set_attr "type" "block")
7281 (set_attr "speculation_barrier" "true")]
7284 ;; Pattern to match despeculate_copyti
7285 (define_insn "*despeculate_copyti_insn"
7286 [(set (match_operand:TI 0 "register_operand" "=r")
7288 [(match_operand:TI 1 "register_operand" "r")
7289 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7290 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7291 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7294 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7296 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7300 [(set_attr "length" "16")
7301 (set_attr "type" "block")
7302 (set_attr "speculation_barrier" "true")]
7305 (define_insn "despeculate_simple<ALLI:mode>"
7306 [(set (match_operand:ALLI 0 "register_operand" "=r")
7307 (unspec_volatile:ALLI
7308 [(match_operand:ALLI 1 "register_operand" "r")
7309 (use (match_operand:ALLI 2 "register_operand" ""))]
7310 UNSPECV_SPECULATION_BARRIER))]
7312 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7313 [(set_attr "type" "block")
7314 (set_attr "length" "8")
7315 (set_attr "speculation_barrier" "true")]
7318 (define_insn "despeculate_simpleti"
7319 [(set (match_operand:TI 0 "register_operand" "=r")
7321 [(match_operand:TI 1 "register_operand" "r")
7322 (use (match_operand:DI 2 "register_operand" ""))]
7323 UNSPECV_SPECULATION_BARRIER))]
7325 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7326 [(set_attr "type" "block")
7327 (set_attr "length" "12")
7328 (set_attr "speculation_barrier" "true")]
7331 (define_insn "aarch64_<frintnzs_op><mode>"
7332 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7333 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7335 "TARGET_FRINT && TARGET_FLOAT
7336 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7337 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7338 [(set_attr "type" "f_rint<stype>")]
7341 ;; Transactional Memory Extension (TME) instructions.
7343 (define_insn "tstart"
7344 [(set (match_operand:DI 0 "register_operand" "=r")
7345 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7346 (clobber (mem:BLK (scratch)))]
7349 [(set_attr "type" "tme")]
7352 (define_insn "ttest"
7353 [(set (match_operand:DI 0 "register_operand" "=r")
7354 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7355 (clobber (mem:BLK (scratch)))]
7358 [(set_attr "type" "tme")]
7361 (define_insn "tcommit"
7362 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7363 (clobber (mem:BLK (scratch)))]
7366 [(set_attr "type" "tme")]
7369 (define_insn "tcancel"
7370 [(unspec_volatile:BLK
7371 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7372 (clobber (mem:BLK (scratch)))]
7373 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7375 [(set_attr "type" "tme")]
7378 (define_insn "aarch64_rndr"
7379 [(set (match_operand:DI 0 "register_operand" "=r")
7380 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7381 (set (reg:CC_Z CC_REGNUM)
7382 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7385 [(set_attr "type" "mrs")]
7388 (define_insn "aarch64_rndrrs"
7389 [(set (match_operand:DI 0 "register_operand" "=r")
7390 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7391 (set (reg:CC_Z CC_REGNUM)
7392 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7395 [(set_attr "type" "mrs")]
7398 ;; Memory Tagging Extension (MTE) instructions.
7401 [(set (match_operand:DI 0 "register_operand" "=rk")
7403 (and:DI (match_operand:DI 1 "register_operand" "rk")
7404 (const_int -1080863910568919041)) ;; 0xf0ff...
7405 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")]
7410 [(set_attr "type" "memtag")]
7414 [(set (match_operand:DI 0 "register_operand" "=r")
7417 (and:QI (lshiftrt:DI
7418 (match_operand:DI 1 "register_operand" "rk")
7419 (const_int 56)) (const_int 15)))
7420 (match_operand:DI 2 "register_operand" "r")))]
7423 [(set_attr "type" "memtag")]
7427 [(set (match_operand:DI 0 "register_operand" "=rk")
7429 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7430 (match_operand:DI 2 "aarch64_granule16_uimm6" "i"))
7431 (const_int -1080863910568919041)) ;; 0xf0ff...
7434 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15))
7435 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")]
7439 "addg\\t%0, %1, #%2, #%3"
7440 [(set_attr "type" "memtag")]
7444 [(set (match_operand:DI 0 "register_operand" "=r")
7446 (and:DI (match_operand:DI 1 "register_operand" "rk")
7447 (const_int 72057594037927935)) ;; 0x00ff...
7448 (and:DI (match_operand:DI 2 "register_operand" "rk")
7449 (const_int 72057594037927935))))] ;; 0x00ff...
7452 [(set_attr "type" "memtag")]
7455 ;; LDG will use the 16-byte aligned value of the address.
7457 [(set (match_operand:DI 0 "register_operand" "+r")
7459 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff...
7462 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7463 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))
7464 (const_int -16))] UNSPEC_TAG_SPACE))
7467 "ldg\\t%0, [%1, #%2]"
7468 [(set_attr "type" "memtag")]
7471 ;; STG doesn't align the address but aborts with alignment fault
7472 ;; when the address is not 16-byte aligned.
7474 [(set (mem:QI (unspec:DI
7475 [(plus:DI (match_operand:DI 1 "register_operand" "rk")
7476 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))]
7478 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk")
7479 (const_int 56)) (const_int 15)))]
7481 "stg\\t%0, [%1, #%2]"
7482 [(set_attr "type" "memtag")]
7486 (include "aarch64-simd.md")
7488 ;; Atomic Operations
7489 (include "atomics.md")
7491 ;; ldp/stp peephole patterns
7492 (include "aarch64-ldpstp.md")
7495 (include "aarch64-sve.md")
7498 (include "aarch64-sve2.md")