2 Copyright (C) 2011-2018 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
67 /* Fnspec of each internal function, indexed by function number. */
68 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
73 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
74 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
75 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
76 #include "internal-fn.def"
77 internal_fn_fnspec_array
[IFN_LAST
] = 0;
80 /* Create static initializers for the information returned by
81 direct_internal_fn. */
82 #define not_direct { -2, -2, false }
83 #define mask_load_direct { -1, 2, false }
84 #define load_lanes_direct { -1, -1, false }
85 #define mask_load_lanes_direct { -1, -1, false }
86 #define mask_store_direct { 3, 2, false }
87 #define store_lanes_direct { 0, 0, false }
88 #define mask_store_lanes_direct { 0, 0, false }
89 #define unary_direct { 0, 0, true }
90 #define binary_direct { 0, 0, true }
91 #define cond_unary_direct { 1, 1, true }
92 #define cond_binary_direct { 1, 1, true }
93 #define while_direct { 0, 2, false }
95 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
96 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
97 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
98 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
99 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
100 #include "internal-fn.def"
104 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
105 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
107 static enum insn_code
108 get_multi_vector_move (tree array_type
, convert_optab optab
)
113 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
114 imode
= TYPE_MODE (array_type
);
115 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
117 return convert_optab_handler (optab
, imode
, vmode
);
120 /* Expand LOAD_LANES call STMT using optab OPTAB. */
123 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
125 struct expand_operand ops
[2];
129 lhs
= gimple_call_lhs (stmt
);
130 rhs
= gimple_call_arg (stmt
, 0);
131 type
= TREE_TYPE (lhs
);
133 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
134 mem
= expand_normal (rhs
);
136 gcc_assert (MEM_P (mem
));
137 PUT_MODE (mem
, TYPE_MODE (type
));
139 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
140 create_fixed_operand (&ops
[1], mem
);
141 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
144 /* Expand STORE_LANES call STMT using optab OPTAB. */
147 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
149 struct expand_operand ops
[2];
153 lhs
= gimple_call_lhs (stmt
);
154 rhs
= gimple_call_arg (stmt
, 0);
155 type
= TREE_TYPE (rhs
);
157 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
158 reg
= expand_normal (rhs
);
160 gcc_assert (MEM_P (target
));
161 PUT_MODE (target
, TYPE_MODE (type
));
163 create_fixed_operand (&ops
[0], target
);
164 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
165 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
169 expand_ANNOTATE (internal_fn
, gcall
*)
174 /* This should get expanded in omp_device_lower pass. */
177 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
182 /* This should get expanded in omp_device_lower pass. */
185 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
190 /* Allocate per-lane storage and begin non-uniform execution region. */
193 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
196 tree lhs
= gimple_call_lhs (stmt
);
198 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
200 target
= gen_reg_rtx (Pmode
);
201 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
202 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
203 struct expand_operand ops
[3];
204 create_output_operand (&ops
[0], target
, Pmode
);
205 create_input_operand (&ops
[1], size
, Pmode
);
206 create_input_operand (&ops
[2], align
, Pmode
);
207 gcc_assert (targetm
.have_omp_simt_enter ());
208 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
211 /* Deallocate per-lane storage and leave non-uniform execution region. */
214 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
216 gcc_checking_assert (!gimple_call_lhs (stmt
));
217 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
218 struct expand_operand ops
[1];
219 create_input_operand (&ops
[0], arg
, Pmode
);
220 gcc_assert (targetm
.have_omp_simt_exit ());
221 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
224 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
225 without SIMT execution this should be expanded in omp_device_lower pass. */
228 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
230 tree lhs
= gimple_call_lhs (stmt
);
234 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
235 gcc_assert (targetm
.have_omp_simt_lane ());
236 emit_insn (targetm
.gen_omp_simt_lane (target
));
239 /* This should get expanded in omp_device_lower pass. */
242 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
247 /* Lane index of the first SIMT lane that supplies a non-zero argument.
248 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
249 lane that executed the last iteration for handling OpenMP lastprivate. */
252 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
254 tree lhs
= gimple_call_lhs (stmt
);
258 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
259 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
260 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
261 struct expand_operand ops
[2];
262 create_output_operand (&ops
[0], target
, mode
);
263 create_input_operand (&ops
[1], cond
, mode
);
264 gcc_assert (targetm
.have_omp_simt_last_lane ());
265 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
268 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
271 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
273 tree lhs
= gimple_call_lhs (stmt
);
277 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
278 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
279 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
280 struct expand_operand ops
[2];
281 create_output_operand (&ops
[0], target
, mode
);
282 create_input_operand (&ops
[1], ctr
, mode
);
283 gcc_assert (targetm
.have_omp_simt_ordered ());
284 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
287 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
288 any lane supplies a non-zero argument. */
291 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
293 tree lhs
= gimple_call_lhs (stmt
);
297 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
298 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
299 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
300 struct expand_operand ops
[2];
301 create_output_operand (&ops
[0], target
, mode
);
302 create_input_operand (&ops
[1], cond
, mode
);
303 gcc_assert (targetm
.have_omp_simt_vote_any ());
304 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
307 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
308 is destination lane index XOR given offset. */
311 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
313 tree lhs
= gimple_call_lhs (stmt
);
317 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
318 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
319 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
320 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
321 struct expand_operand ops
[3];
322 create_output_operand (&ops
[0], target
, mode
);
323 create_input_operand (&ops
[1], src
, mode
);
324 create_input_operand (&ops
[2], idx
, SImode
);
325 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
326 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
329 /* Exchange between SIMT lanes according to given source lane index. */
332 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
334 tree lhs
= gimple_call_lhs (stmt
);
338 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
339 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
340 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
341 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
342 struct expand_operand ops
[3];
343 create_output_operand (&ops
[0], target
, mode
);
344 create_input_operand (&ops
[1], src
, mode
);
345 create_input_operand (&ops
[2], idx
, SImode
);
346 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
347 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
350 /* This should get expanded in adjust_simduid_builtins. */
353 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
358 /* This should get expanded in adjust_simduid_builtins. */
361 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
366 /* This should get expanded in adjust_simduid_builtins. */
369 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
374 /* This should get expanded in adjust_simduid_builtins. */
377 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
382 /* This should get expanded in adjust_simduid_builtins. */
385 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
390 /* This should get expanded in the sanopt pass. */
393 expand_UBSAN_NULL (internal_fn
, gcall
*)
398 /* This should get expanded in the sanopt pass. */
401 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
406 /* This should get expanded in the sanopt pass. */
409 expand_UBSAN_VPTR (internal_fn
, gcall
*)
414 /* This should get expanded in the sanopt pass. */
417 expand_UBSAN_PTR (internal_fn
, gcall
*)
422 /* This should get expanded in the sanopt pass. */
425 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
430 /* This should get expanded in the sanopt pass. */
433 expand_ASAN_CHECK (internal_fn
, gcall
*)
438 /* This should get expanded in the sanopt pass. */
441 expand_ASAN_MARK (internal_fn
, gcall
*)
446 /* This should get expanded in the sanopt pass. */
449 expand_ASAN_POISON (internal_fn
, gcall
*)
454 /* This should get expanded in the sanopt pass. */
457 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
462 /* This should get expanded in the tsan pass. */
465 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
470 /* This should get expanded in the lower pass. */
473 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
475 error_at (gimple_location (call
),
476 "invalid use of attribute %<fallthrough%>");
479 /* Return minimum precision needed to represent all values
480 of ARG in SIGNed integral type. */
483 get_min_precision (tree arg
, signop sign
)
485 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
487 signop orig_sign
= sign
;
488 if (TREE_CODE (arg
) == INTEGER_CST
)
491 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
493 widest_int w
= wi::to_widest (arg
);
494 w
= wi::ext (w
, prec
, sign
);
495 p
= wi::min_precision (w
, sign
);
498 p
= wi::min_precision (wi::to_wide (arg
), sign
);
499 return MIN (p
, prec
);
501 while (CONVERT_EXPR_P (arg
)
502 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
503 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
505 arg
= TREE_OPERAND (arg
, 0);
506 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
508 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
510 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
511 return prec
+ (orig_sign
!= sign
);
512 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
515 return prec
+ (orig_sign
!= sign
);
517 if (TREE_CODE (arg
) != SSA_NAME
)
518 return prec
+ (orig_sign
!= sign
);
519 wide_int arg_min
, arg_max
;
520 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
522 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
523 if (is_gimple_assign (g
)
524 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
526 tree t
= gimple_assign_rhs1 (g
);
527 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
528 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
531 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
533 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
535 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
536 return prec
+ (orig_sign
!= sign
);
537 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
540 return prec
+ (orig_sign
!= sign
);
544 return prec
+ (orig_sign
!= sign
);
546 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
548 int p1
= wi::min_precision (arg_min
, sign
);
549 int p2
= wi::min_precision (arg_max
, sign
);
551 prec
= MIN (prec
, p1
);
553 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
555 int p
= wi::min_precision (arg_max
, UNSIGNED
);
556 prec
= MIN (prec
, p
);
558 return prec
+ (orig_sign
!= sign
);
561 /* Helper for expand_*_overflow. Set the __imag__ part to true
562 (1 except for signed:1 type, in which case store -1). */
565 expand_arith_set_overflow (tree lhs
, rtx target
)
567 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
568 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
569 write_complex_part (target
, constm1_rtx
, true);
571 write_complex_part (target
, const1_rtx
, true);
574 /* Helper for expand_*_overflow. Store RES into the __real__ part
575 of TARGET. If RES has larger MODE than __real__ part of TARGET,
576 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
577 if LHS has smaller precision than its mode. */
580 expand_arith_overflow_result_store (tree lhs
, rtx target
,
581 scalar_int_mode mode
, rtx res
)
583 scalar_int_mode tgtmode
584 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
588 rtx_code_label
*done_label
= gen_label_rtx ();
589 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
590 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
591 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
592 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
593 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
594 profile_probability::very_likely ());
595 expand_arith_set_overflow (lhs
, target
);
596 emit_label (done_label
);
598 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
599 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
602 rtx_code_label
*done_label
= gen_label_rtx ();
603 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
608 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
610 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
611 true, OPTAB_LIB_WIDEN
);
615 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
617 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
620 do_compare_rtx_and_jump (res
, lres
,
621 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
622 profile_probability::very_likely ());
623 expand_arith_set_overflow (lhs
, target
);
624 emit_label (done_label
);
626 write_complex_part (target
, lres
, false);
629 /* Helper for expand_*_overflow. Store RES into TARGET. */
632 expand_ubsan_result_store (rtx target
, rtx res
)
634 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
635 /* If this is a scalar in a register that is stored in a wider mode
636 than the declared mode, compute the result into its declared mode
637 and then convert to the wider mode. Our value is the computed
639 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
641 emit_move_insn (target
, res
);
644 /* Add sub/add overflow checking to the statement STMT.
645 CODE says whether the operation is +, or -. */
648 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
649 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
650 bool uns1_p
, bool is_ubsan
, tree
*datap
)
652 rtx res
, target
= NULL_RTX
;
654 rtx_code_label
*done_label
= gen_label_rtx ();
655 rtx_code_label
*do_error
= gen_label_rtx ();
656 do_pending_stack_adjust ();
657 rtx op0
= expand_normal (arg0
);
658 rtx op1
= expand_normal (arg1
);
659 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
660 int prec
= GET_MODE_PRECISION (mode
);
661 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
665 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
669 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
671 write_complex_part (target
, const0_rtx
, true);
674 /* We assume both operands and result have the same precision
675 here (GET_MODE_BITSIZE (mode)), S stands for signed type
676 with that precision, U for unsigned type with that precision,
677 sgn for unsigned most significant bit in that precision.
678 s1 is signed first operand, u1 is unsigned first operand,
679 s2 is signed second operand, u2 is unsigned second operand,
680 sr is signed result, ur is unsigned result and the following
681 rules say how to compute result (which is always result of
682 the operands as if both were unsigned, cast to the right
683 signedness) and how to compute whether operation overflowed.
686 res = (S) ((U) s1 + (U) s2)
687 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
689 res = (S) ((U) s1 - (U) s2)
690 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
693 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
696 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
698 res = (S) ((U) s1 + u2)
699 ovf = ((U) res ^ sgn) < u2
704 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
706 res = (S) ((U) s1 - u2)
707 ovf = u2 > ((U) s1 ^ sgn)
710 ovf = s1 < 0 || u2 > (U) s1
713 ovf = u1 >= ((U) s2 ^ sgn)
718 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
720 res = (U) s1 + (U) s2
721 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
724 ovf = (U) res < u2 || res < 0
727 ovf = u1 >= u2 ? res < 0 : res >= 0
729 res = (U) s1 - (U) s2
730 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
732 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
734 /* PLUS_EXPR is commutative, if operand signedness differs,
735 canonicalize to the first operand being signed and second
736 unsigned to simplify following code. */
737 std::swap (op0
, op1
);
738 std::swap (arg0
, arg1
);
744 if (uns0_p
&& uns1_p
&& unsr_p
)
746 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
747 : usubv4_optab
, mode
);
748 if (icode
!= CODE_FOR_nothing
)
750 struct expand_operand ops
[4];
751 rtx_insn
*last
= get_last_insn ();
753 res
= gen_reg_rtx (mode
);
754 create_output_operand (&ops
[0], res
, mode
);
755 create_input_operand (&ops
[1], op0
, mode
);
756 create_input_operand (&ops
[2], op1
, mode
);
757 create_fixed_operand (&ops
[3], do_error
);
758 if (maybe_expand_insn (icode
, 4, ops
))
760 last
= get_last_insn ();
761 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
763 && any_condjump_p (last
)
764 && !find_reg_note (last
, REG_BR_PROB
, 0))
765 add_reg_br_prob_note (last
,
766 profile_probability::very_unlikely ());
767 emit_jump (done_label
);
771 delete_insns_since (last
);
774 /* Compute the operation. On RTL level, the addition is always
776 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
777 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
779 /* For PLUS_EXPR, the operation is commutative, so we can pick
780 operand to compare against. For prec <= BITS_PER_WORD, I think
781 preferring REG operand is better over CONST_INT, because
782 the CONST_INT might enlarge the instruction or CSE would need
783 to figure out we'd already loaded it into a register before.
784 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
785 as then the multi-word comparison can be perhaps simplified. */
786 if (code
== PLUS_EXPR
787 && (prec
<= BITS_PER_WORD
788 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
789 : CONST_SCALAR_INT_P (op1
)))
791 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
792 true, mode
, NULL_RTX
, NULL
, done_label
,
793 profile_probability::very_likely ());
798 if (!uns0_p
&& uns1_p
&& !unsr_p
)
800 /* Compute the operation. On RTL level, the addition is always
802 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
803 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
804 rtx tem
= expand_binop (mode
, add_optab
,
805 code
== PLUS_EXPR
? res
: op0
, sgn
,
806 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
807 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
808 done_label
, profile_probability::very_likely ());
813 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
815 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
817 /* As we've changed op1, we have to avoid using the value range
818 for the original argument. */
819 arg1
= error_mark_node
;
825 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
827 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
829 /* As we've changed op0, we have to avoid using the value range
830 for the original argument. */
831 arg0
= error_mark_node
;
837 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
839 /* Compute the operation. On RTL level, the addition is always
841 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
843 int pos_neg
= get_range_pos_neg (arg0
);
845 /* If ARG0 is known to be always negative, this is always overflow. */
846 emit_jump (do_error
);
847 else if (pos_neg
== 3)
848 /* If ARG0 is not known to be always positive, check at runtime. */
849 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
850 NULL
, do_error
, profile_probability::very_unlikely ());
851 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
852 done_label
, profile_probability::very_likely ());
857 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
859 /* Compute the operation. On RTL level, the addition is always
861 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
863 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
865 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
866 done_label
, profile_probability::very_likely ());
871 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
873 /* Compute the operation. On RTL level, the addition is always
875 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
877 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
878 NULL
, do_error
, profile_probability::very_unlikely ());
880 /* The operation is commutative, so we can pick operand to compare
881 against. For prec <= BITS_PER_WORD, I think preferring REG operand
882 is better over CONST_INT, because the CONST_INT might enlarge the
883 instruction or CSE would need to figure out we'd already loaded it
884 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
885 might be more beneficial, as then the multi-word comparison can be
886 perhaps simplified. */
887 if (prec
<= BITS_PER_WORD
888 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
889 : CONST_SCALAR_INT_P (op0
))
891 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
892 done_label
, profile_probability::very_likely ());
897 if (!uns0_p
&& !uns1_p
&& unsr_p
)
899 /* Compute the operation. On RTL level, the addition is always
901 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
902 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
903 int pos_neg
= get_range_pos_neg (arg1
);
904 if (code
== PLUS_EXPR
)
906 int pos_neg0
= get_range_pos_neg (arg0
);
907 if (pos_neg0
!= 3 && pos_neg
== 3)
909 std::swap (op0
, op1
);
916 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
917 ? and_optab
: ior_optab
,
918 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
919 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
920 NULL
, done_label
, profile_probability::very_likely ());
924 rtx_code_label
*do_ior_label
= gen_label_rtx ();
925 do_compare_rtx_and_jump (op1
, const0_rtx
,
926 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
927 NULL_RTX
, NULL
, do_ior_label
,
928 profile_probability::even ());
929 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
931 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
932 NULL
, done_label
, profile_probability::very_likely ());
933 emit_jump (do_error
);
934 emit_label (do_ior_label
);
935 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
937 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
938 NULL
, done_label
, profile_probability::very_likely ());
944 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
946 /* Compute the operation. On RTL level, the addition is always
948 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
950 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
951 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
952 op0_geu_op1
, profile_probability::even ());
953 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
954 NULL
, done_label
, profile_probability::very_likely ());
955 emit_jump (do_error
);
956 emit_label (op0_geu_op1
);
957 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
958 NULL
, done_label
, profile_probability::very_likely ());
962 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
967 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
968 : subv4_optab
, mode
);
969 if (icode
!= CODE_FOR_nothing
)
971 struct expand_operand ops
[4];
972 rtx_insn
*last
= get_last_insn ();
974 res
= gen_reg_rtx (mode
);
975 create_output_operand (&ops
[0], res
, mode
);
976 create_input_operand (&ops
[1], op0
, mode
);
977 create_input_operand (&ops
[2], op1
, mode
);
978 create_fixed_operand (&ops
[3], do_error
);
979 if (maybe_expand_insn (icode
, 4, ops
))
981 last
= get_last_insn ();
982 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
984 && any_condjump_p (last
)
985 && !find_reg_note (last
, REG_BR_PROB
, 0))
986 add_reg_br_prob_note (last
,
987 profile_probability::very_unlikely ());
988 emit_jump (done_label
);
992 delete_insns_since (last
);
995 /* Compute the operation. On RTL level, the addition is always
997 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
998 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1000 /* If we can prove that one of the arguments (for MINUS_EXPR only
1001 the second operand, as subtraction is not commutative) is always
1002 non-negative or always negative, we can do just one comparison
1003 and conditional jump. */
1004 int pos_neg
= get_range_pos_neg (arg1
);
1005 if (code
== PLUS_EXPR
)
1007 int pos_neg0
= get_range_pos_neg (arg0
);
1008 if (pos_neg0
!= 3 && pos_neg
== 3)
1010 std::swap (op0
, op1
);
1015 /* Addition overflows if and only if the two operands have the same sign,
1016 and the result has the opposite sign. Subtraction overflows if and
1017 only if the two operands have opposite sign, and the subtrahend has
1018 the same sign as the result. Here 0 is counted as positive. */
1021 /* Compute op0 ^ op1 (operands have opposite sign). */
1022 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1025 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1026 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1030 if (code
== PLUS_EXPR
)
1032 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1033 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1034 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1039 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1040 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1041 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1045 /* No overflow if the result has bit sign cleared. */
1046 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1047 NULL
, done_label
, profile_probability::very_likely ());
1050 /* Compare the result of the operation with the first operand.
1051 No overflow for addition if second operand is positive and result
1052 is larger or second operand is negative and result is smaller.
1053 Likewise for subtraction with sign of second operand flipped. */
1055 do_compare_rtx_and_jump (res
, op0
,
1056 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1057 false, mode
, NULL_RTX
, NULL
, done_label
,
1058 profile_probability::very_likely ());
1062 emit_label (do_error
);
1065 /* Expand the ubsan builtin call. */
1067 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1071 do_pending_stack_adjust ();
1074 expand_arith_set_overflow (lhs
, target
);
1077 emit_label (done_label
);
1082 expand_ubsan_result_store (target
, res
);
1086 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1089 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1094 /* Add negate overflow checking to the statement STMT. */
1097 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1102 rtx_code_label
*done_label
, *do_error
;
1103 rtx target
= NULL_RTX
;
1105 done_label
= gen_label_rtx ();
1106 do_error
= gen_label_rtx ();
1108 do_pending_stack_adjust ();
1109 op1
= expand_normal (arg1
);
1111 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1114 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1116 write_complex_part (target
, const0_rtx
, true);
1119 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1120 if (icode
!= CODE_FOR_nothing
)
1122 struct expand_operand ops
[3];
1123 rtx_insn
*last
= get_last_insn ();
1125 res
= gen_reg_rtx (mode
);
1126 create_output_operand (&ops
[0], res
, mode
);
1127 create_input_operand (&ops
[1], op1
, mode
);
1128 create_fixed_operand (&ops
[2], do_error
);
1129 if (maybe_expand_insn (icode
, 3, ops
))
1131 last
= get_last_insn ();
1132 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1134 && any_condjump_p (last
)
1135 && !find_reg_note (last
, REG_BR_PROB
, 0))
1136 add_reg_br_prob_note (last
,
1137 profile_probability::very_unlikely ());
1138 emit_jump (done_label
);
1142 delete_insns_since (last
);
1143 icode
= CODE_FOR_nothing
;
1147 if (icode
== CODE_FOR_nothing
)
1149 /* Compute the operation. On RTL level, the addition is always
1151 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1153 /* Compare the operand with the most negative value. */
1154 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1155 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1156 done_label
, profile_probability::very_likely ());
1159 emit_label (do_error
);
1162 /* Expand the ubsan builtin call. */
1164 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1165 arg1
, NULL_TREE
, datap
);
1168 do_pending_stack_adjust ();
1171 expand_arith_set_overflow (lhs
, target
);
1174 emit_label (done_label
);
1179 expand_ubsan_result_store (target
, res
);
1181 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1185 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1186 mode MODE can be expanded without using a libcall. */
1189 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1190 rtx op0
, rtx op1
, bool uns
)
1192 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1193 != CODE_FOR_nothing
)
1196 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1197 != CODE_FOR_nothing
)
1200 rtx_insn
*last
= get_last_insn ();
1201 if (CONSTANT_P (op0
))
1202 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1204 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1205 if (CONSTANT_P (op1
))
1206 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1208 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1209 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1210 delete_insns_since (last
);
1211 return ret
!= NULL_RTX
;
1214 /* Add mul overflow checking to the statement STMT. */
1217 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1218 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1223 rtx_code_label
*done_label
, *do_error
;
1224 rtx target
= NULL_RTX
;
1226 enum insn_code icode
;
1228 done_label
= gen_label_rtx ();
1229 do_error
= gen_label_rtx ();
1231 do_pending_stack_adjust ();
1232 op0
= expand_normal (arg0
);
1233 op1
= expand_normal (arg1
);
1235 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1239 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1241 write_complex_part (target
, const0_rtx
, true);
1245 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1247 /* We assume both operands and result have the same precision
1248 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1249 with that precision, U for unsigned type with that precision,
1250 sgn for unsigned most significant bit in that precision.
1251 s1 is signed first operand, u1 is unsigned first operand,
1252 s2 is signed second operand, u2 is unsigned second operand,
1253 sr is signed result, ur is unsigned result and the following
1254 rules say how to compute result (which is always result of
1255 the operands as if both were unsigned, cast to the right
1256 signedness) and how to compute whether operation overflowed.
1257 main_ovf (false) stands for jump on signed multiplication
1258 overflow or the main algorithm with uns == false.
1259 main_ovf (true) stands for jump on unsigned multiplication
1260 overflow or the main algorithm with uns == true.
1263 res = (S) ((U) s1 * (U) s2)
1264 ovf = main_ovf (false)
1267 ovf = main_ovf (true)
1270 ovf = (s1 < 0 && u2) || main_ovf (true)
1273 ovf = res < 0 || main_ovf (true)
1275 res = (S) ((U) s1 * u2)
1276 ovf = (S) u2 >= 0 ? main_ovf (false)
1277 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1279 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1280 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1282 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1284 if (uns0_p
&& !uns1_p
)
1286 /* Multiplication is commutative, if operand signedness differs,
1287 canonicalize to the first operand being signed and second
1288 unsigned to simplify following code. */
1289 std::swap (op0
, op1
);
1290 std::swap (arg0
, arg1
);
1295 int pos_neg0
= get_range_pos_neg (arg0
);
1296 int pos_neg1
= get_range_pos_neg (arg1
);
1299 if (!uns0_p
&& uns1_p
&& unsr_p
)
1304 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1307 /* If s1 is negative, avoid the main code, just multiply and
1308 signal overflow if op1 is not 0. */
1309 struct separate_ops ops
;
1310 ops
.code
= MULT_EXPR
;
1311 ops
.type
= TREE_TYPE (arg1
);
1312 ops
.op0
= make_tree (ops
.type
, op0
);
1313 ops
.op1
= make_tree (ops
.type
, op1
);
1314 ops
.op2
= NULL_TREE
;
1316 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1317 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1318 NULL
, done_label
, profile_probability::very_likely ());
1319 goto do_error_label
;
1321 rtx_code_label
*do_main_label
;
1322 do_main_label
= gen_label_rtx ();
1323 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1324 NULL
, do_main_label
, profile_probability::very_likely ());
1325 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1326 NULL
, do_main_label
, profile_probability::very_likely ());
1327 expand_arith_set_overflow (lhs
, target
);
1328 emit_label (do_main_label
);
1336 if (uns0_p
&& uns1_p
&& !unsr_p
)
1339 /* Rest of handling of this case after res is computed. */
1344 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1351 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1352 avoid the main code, just multiply and signal overflow
1353 unless 0 * u2 or -1 * ((U) Smin). */
1354 struct separate_ops ops
;
1355 ops
.code
= MULT_EXPR
;
1356 ops
.type
= TREE_TYPE (arg1
);
1357 ops
.op0
= make_tree (ops
.type
, op0
);
1358 ops
.op1
= make_tree (ops
.type
, op1
);
1359 ops
.op2
= NULL_TREE
;
1361 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1362 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1363 NULL
, done_label
, profile_probability::very_likely ());
1364 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1365 NULL
, do_error
, profile_probability::very_unlikely ());
1367 prec
= GET_MODE_PRECISION (mode
);
1369 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1370 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1371 NULL
, done_label
, profile_probability::very_likely ());
1372 goto do_error_label
;
1374 /* Rest of handling of this case after res is computed. */
1382 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1385 switch (pos_neg0
| pos_neg1
)
1387 case 1: /* Both operands known to be non-negative. */
1389 case 2: /* Both operands known to be negative. */
1390 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1391 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1392 /* Avoid looking at arg0/arg1 ranges, as we've changed
1394 arg0
= error_mark_node
;
1395 arg1
= error_mark_node
;
1398 if ((pos_neg0
^ pos_neg1
) == 3)
1400 /* If one operand is known to be negative and the other
1401 non-negative, this overflows always, unless the non-negative
1402 one is 0. Just do normal multiply and set overflow
1403 unless one of the operands is 0. */
1404 struct separate_ops ops
;
1405 ops
.code
= MULT_EXPR
;
1407 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1409 ops
.op0
= make_tree (ops
.type
, op0
);
1410 ops
.op1
= make_tree (ops
.type
, op1
);
1411 ops
.op2
= NULL_TREE
;
1413 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1414 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1416 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1417 NULL_RTX
, NULL
, done_label
,
1418 profile_probability::very_likely ());
1419 goto do_error_label
;
1421 /* The general case, do all the needed comparisons at runtime. */
1422 rtx_code_label
*do_main_label
, *after_negate_label
;
1424 rop0
= gen_reg_rtx (mode
);
1425 rop1
= gen_reg_rtx (mode
);
1426 emit_move_insn (rop0
, op0
);
1427 emit_move_insn (rop1
, op1
);
1430 do_main_label
= gen_label_rtx ();
1431 after_negate_label
= gen_label_rtx ();
1432 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1434 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1435 NULL
, after_negate_label
, profile_probability::very_likely ());
1436 /* Both arguments negative here, negate them and continue with
1437 normal unsigned overflow checking multiplication. */
1438 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1440 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1442 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1444 arg0
= error_mark_node
;
1445 arg1
= error_mark_node
;
1446 emit_jump (do_main_label
);
1447 emit_label (after_negate_label
);
1448 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1450 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1451 NULL
, do_main_label
, profile_probability::very_likely ());
1452 /* One argument is negative here, the other positive. This
1453 overflows always, unless one of the arguments is 0. But
1454 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1455 is, thus we can keep do_main code oring in overflow as is. */
1456 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1457 NULL
, do_main_label
, profile_probability::very_likely ());
1458 expand_arith_set_overflow (lhs
, target
);
1459 emit_label (do_main_label
);
1467 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1468 sign
= uns
? UNSIGNED
: SIGNED
;
1469 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1471 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1472 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1474 /* Optimize unsigned multiplication by power of 2 constant
1475 using 2 shifts, one for result, one to extract the shifted
1476 out bits to see if they are all zero.
1477 Don't do this if optimizing for size and we have umulv4_optab,
1478 in that case assume multiplication will be shorter.
1479 This is heuristics based on the single target that provides
1480 umulv4 right now (i?86/x86_64), if further targets add it, this
1481 might need to be revisited.
1482 Cases where both operands are constant should be folded already
1483 during GIMPLE, and cases where one operand is constant but not
1484 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1485 below can be done without multiplication, just by shifts and adds,
1486 or we'd need to divide the result (and hope it actually doesn't
1487 really divide nor multiply) and compare the result of the division
1488 with the original operand. */
1493 if (integer_pow2p (arg0
))
1495 std::swap (opn0
, opn1
);
1496 std::swap (argn0
, argn1
);
1498 int cnt
= tree_log2 (argn1
);
1499 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1501 rtx upper
= const0_rtx
;
1502 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1504 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1505 GET_MODE_PRECISION (mode
) - cnt
,
1507 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1508 NULL_RTX
, NULL
, done_label
,
1509 profile_probability::very_likely ());
1510 goto do_error_label
;
1513 if (icode
!= CODE_FOR_nothing
)
1515 struct expand_operand ops
[4];
1516 rtx_insn
*last
= get_last_insn ();
1518 res
= gen_reg_rtx (mode
);
1519 create_output_operand (&ops
[0], res
, mode
);
1520 create_input_operand (&ops
[1], op0
, mode
);
1521 create_input_operand (&ops
[2], op1
, mode
);
1522 create_fixed_operand (&ops
[3], do_error
);
1523 if (maybe_expand_insn (icode
, 4, ops
))
1525 last
= get_last_insn ();
1526 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1528 && any_condjump_p (last
)
1529 && !find_reg_note (last
, REG_BR_PROB
, 0))
1530 add_reg_br_prob_note (last
,
1531 profile_probability::very_unlikely ());
1532 emit_jump (done_label
);
1536 delete_insns_since (last
);
1537 icode
= CODE_FOR_nothing
;
1541 if (icode
== CODE_FOR_nothing
)
1543 struct separate_ops ops
;
1544 int prec
= GET_MODE_PRECISION (mode
);
1545 scalar_int_mode hmode
, wmode
;
1546 ops
.op0
= make_tree (type
, op0
);
1547 ops
.op1
= make_tree (type
, op1
);
1548 ops
.op2
= NULL_TREE
;
1551 /* Optimize unsigned overflow check where we don't use the
1552 multiplication result, just whether overflow happened.
1553 If we can do MULT_HIGHPART_EXPR, that followed by
1554 comparison of the result against zero is cheapest.
1555 We'll still compute res, but it should be DCEd later. */
1561 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1562 && can_mult_highpart_p (mode
, uns
) == 1
1563 && single_imm_use (lhs
, &use
, &use_stmt
)
1564 && is_gimple_assign (use_stmt
)
1565 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1568 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1569 && targetm
.scalar_mode_supported_p (wmode
)
1570 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1573 ops
.code
= WIDEN_MULT_EXPR
;
1575 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1577 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1578 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1580 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1581 res
= convert_modes (mode
, wmode
, res
, uns
);
1583 /* For the unsigned multiplication, there was overflow if
1584 HIPART is non-zero. */
1585 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1586 NULL_RTX
, NULL
, done_label
,
1587 profile_probability::very_likely ());
1590 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1592 /* RES is low half of the double width result, HIPART
1593 the high half. There was overflow if
1594 HIPART is different from RES < 0 ? -1 : 0. */
1595 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1596 NULL_RTX
, NULL
, done_label
,
1597 profile_probability::very_likely ());
1600 else if (can_mult_highpart_p (mode
, uns
) == 1)
1603 ops
.code
= MULT_HIGHPART_EXPR
;
1606 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1608 ops
.code
= MULT_EXPR
;
1609 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1611 /* For the unsigned multiplication, there was overflow if
1612 HIPART is non-zero. */
1613 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1614 NULL_RTX
, NULL
, done_label
,
1615 profile_probability::very_likely ());
1618 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1620 /* RES is low half of the double width result, HIPART
1621 the high half. There was overflow if
1622 HIPART is different from RES < 0 ? -1 : 0. */
1623 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1624 NULL_RTX
, NULL
, done_label
,
1625 profile_probability::very_likely ());
1629 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1630 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1632 rtx_code_label
*large_op0
= gen_label_rtx ();
1633 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1634 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1635 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1636 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1637 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1638 rtx_code_label
*do_overflow
= gen_label_rtx ();
1639 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1641 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1642 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1644 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1645 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1646 rtx signbit0
= const0_rtx
;
1648 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1650 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1652 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1653 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1654 rtx signbit1
= const0_rtx
;
1656 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1659 res
= gen_reg_rtx (mode
);
1661 /* True if op0 resp. op1 are known to be in the range of
1663 bool op0_small_p
= false;
1664 bool op1_small_p
= false;
1665 /* True if op0 resp. op1 are known to have all zeros or all ones
1666 in the upper half of bits, but are not known to be
1668 bool op0_medium_p
= false;
1669 bool op1_medium_p
= false;
1670 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1671 nonnegative, 1 if unknown. */
1677 else if (pos_neg0
== 2)
1681 else if (pos_neg1
== 2)
1684 unsigned int mprec0
= prec
;
1685 if (arg0
!= error_mark_node
)
1686 mprec0
= get_min_precision (arg0
, sign
);
1687 if (mprec0
<= hprec
)
1689 else if (!uns
&& mprec0
<= hprec
+ 1)
1690 op0_medium_p
= true;
1691 unsigned int mprec1
= prec
;
1692 if (arg1
!= error_mark_node
)
1693 mprec1
= get_min_precision (arg1
, sign
);
1694 if (mprec1
<= hprec
)
1696 else if (!uns
&& mprec1
<= hprec
+ 1)
1697 op1_medium_p
= true;
1699 int smaller_sign
= 1;
1700 int larger_sign
= 1;
1703 smaller_sign
= op0_sign
;
1704 larger_sign
= op1_sign
;
1706 else if (op1_small_p
)
1708 smaller_sign
= op1_sign
;
1709 larger_sign
= op0_sign
;
1711 else if (op0_sign
== op1_sign
)
1713 smaller_sign
= op0_sign
;
1714 larger_sign
= op0_sign
;
1718 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1719 NULL_RTX
, NULL
, large_op0
,
1720 profile_probability::unlikely ());
1723 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1724 NULL_RTX
, NULL
, small_op0_large_op1
,
1725 profile_probability::unlikely ());
1727 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1728 hmode to mode, the multiplication will never overflow. We can
1729 do just one hmode x hmode => mode widening multiplication. */
1730 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1731 if (GET_CODE (lopart0
) == SUBREG
)
1733 lopart0s
= shallow_copy_rtx (lopart0
);
1734 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1735 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1737 if (GET_CODE (lopart1
) == SUBREG
)
1739 lopart1s
= shallow_copy_rtx (lopart1
);
1740 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1741 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1743 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1744 ops
.op0
= make_tree (halfstype
, lopart0s
);
1745 ops
.op1
= make_tree (halfstype
, lopart1s
);
1746 ops
.code
= WIDEN_MULT_EXPR
;
1749 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1750 emit_move_insn (res
, thisres
);
1751 emit_jump (done_label
);
1753 emit_label (small_op0_large_op1
);
1755 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1756 but op1 is not, just swap the arguments and handle it as op1
1757 sign/zero extended, op0 not. */
1758 rtx larger
= gen_reg_rtx (mode
);
1759 rtx hipart
= gen_reg_rtx (hmode
);
1760 rtx lopart
= gen_reg_rtx (hmode
);
1761 emit_move_insn (larger
, op1
);
1762 emit_move_insn (hipart
, hipart1
);
1763 emit_move_insn (lopart
, lopart0
);
1764 emit_jump (one_small_one_large
);
1766 emit_label (large_op0
);
1769 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1770 NULL_RTX
, NULL
, both_ops_large
,
1771 profile_probability::unlikely ());
1773 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1774 but op0 is not, prepare larger, hipart and lopart pseudos and
1775 handle it together with small_op0_large_op1. */
1776 emit_move_insn (larger
, op0
);
1777 emit_move_insn (hipart
, hipart0
);
1778 emit_move_insn (lopart
, lopart1
);
1780 emit_label (one_small_one_large
);
1782 /* lopart is the low part of the operand that is sign extended
1783 to mode, larger is the other operand, hipart is the
1784 high part of larger and lopart0 and lopart1 are the low parts
1786 We perform lopart0 * lopart1 and lopart * hipart widening
1788 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1789 ops
.op0
= make_tree (halfutype
, lopart0
);
1790 ops
.op1
= make_tree (halfutype
, lopart1
);
1792 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1794 ops
.op0
= make_tree (halfutype
, lopart
);
1795 ops
.op1
= make_tree (halfutype
, hipart
);
1796 rtx loxhi
= gen_reg_rtx (mode
);
1797 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1798 emit_move_insn (loxhi
, tem
);
1802 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1803 if (larger_sign
== 0)
1804 emit_jump (after_hipart_neg
);
1805 else if (larger_sign
!= -1)
1806 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1807 NULL_RTX
, NULL
, after_hipart_neg
,
1808 profile_probability::even ());
1810 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1811 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1812 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1814 emit_move_insn (loxhi
, tem
);
1816 emit_label (after_hipart_neg
);
1818 /* if (lopart < 0) loxhi -= larger; */
1819 if (smaller_sign
== 0)
1820 emit_jump (after_lopart_neg
);
1821 else if (smaller_sign
!= -1)
1822 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1823 NULL_RTX
, NULL
, after_lopart_neg
,
1824 profile_probability::even ());
1826 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1828 emit_move_insn (loxhi
, tem
);
1830 emit_label (after_lopart_neg
);
1833 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1834 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1835 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1837 emit_move_insn (loxhi
, tem
);
1839 /* if (loxhi >> (bitsize / 2)
1840 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1841 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1842 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1844 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1845 rtx signbitloxhi
= const0_rtx
;
1847 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1848 convert_modes (hmode
, mode
,
1850 hprec
- 1, NULL_RTX
, 0);
1852 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1853 NULL_RTX
, NULL
, do_overflow
,
1854 profile_probability::very_unlikely ());
1856 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1857 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1859 tem
= convert_modes (mode
, hmode
,
1860 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1862 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1865 emit_move_insn (res
, tem
);
1866 emit_jump (done_label
);
1868 emit_label (both_ops_large
);
1870 /* If both operands are large (not sign (!uns) or zero (uns)
1871 extended from hmode), then perform the full multiplication
1872 which will be the result of the operation.
1873 The only cases which don't overflow are for signed multiplication
1874 some cases where both hipart0 and highpart1 are 0 or -1.
1875 For unsigned multiplication when high parts are both non-zero
1876 this overflows always. */
1877 ops
.code
= MULT_EXPR
;
1878 ops
.op0
= make_tree (type
, op0
);
1879 ops
.op1
= make_tree (type
, op1
);
1880 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1881 emit_move_insn (res
, tem
);
1887 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1888 NULL_RTX
, 1, OPTAB_WIDEN
);
1889 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1890 NULL_RTX
, NULL
, do_error
,
1891 profile_probability::very_unlikely ());
1896 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1897 NULL_RTX
, 1, OPTAB_WIDEN
);
1898 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1899 NULL_RTX
, NULL
, do_error
,
1900 profile_probability::very_unlikely ());
1903 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1904 the same, overflow happened if res is non-positive, if they
1905 are different, overflow happened if res is positive. */
1906 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1907 emit_jump (hipart_different
);
1908 else if (op0_sign
== 1 || op1_sign
== 1)
1909 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1910 NULL_RTX
, NULL
, hipart_different
,
1911 profile_probability::even ());
1913 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1914 NULL_RTX
, NULL
, do_error
,
1915 profile_probability::very_unlikely ());
1916 emit_jump (done_label
);
1918 emit_label (hipart_different
);
1920 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1921 NULL_RTX
, NULL
, do_error
,
1922 profile_probability::very_unlikely ());
1923 emit_jump (done_label
);
1926 emit_label (do_overflow
);
1928 /* Overflow, do full multiplication and fallthru into do_error. */
1929 ops
.op0
= make_tree (type
, op0
);
1930 ops
.op1
= make_tree (type
, op1
);
1931 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1932 emit_move_insn (res
, tem
);
1934 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1935 && targetm
.scalar_mode_supported_p (wmode
))
1936 /* Even emitting a libcall is better than not detecting overflow
1941 gcc_assert (!is_ubsan
);
1942 ops
.code
= MULT_EXPR
;
1944 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1945 emit_jump (done_label
);
1950 emit_label (do_error
);
1953 /* Expand the ubsan builtin call. */
1955 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1959 do_pending_stack_adjust ();
1962 expand_arith_set_overflow (lhs
, target
);
1965 emit_label (done_label
);
1968 if (uns0_p
&& uns1_p
&& !unsr_p
)
1970 rtx_code_label
*all_done_label
= gen_label_rtx ();
1971 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1972 NULL
, all_done_label
, profile_probability::very_likely ());
1973 expand_arith_set_overflow (lhs
, target
);
1974 emit_label (all_done_label
);
1978 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1980 rtx_code_label
*all_done_label
= gen_label_rtx ();
1981 rtx_code_label
*set_noovf
= gen_label_rtx ();
1982 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1983 NULL
, all_done_label
, profile_probability::very_likely ());
1984 expand_arith_set_overflow (lhs
, target
);
1985 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1986 NULL
, set_noovf
, profile_probability::very_likely ());
1987 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1988 NULL
, all_done_label
, profile_probability::very_unlikely ());
1989 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1990 all_done_label
, profile_probability::very_unlikely ());
1991 emit_label (set_noovf
);
1992 write_complex_part (target
, const0_rtx
, true);
1993 emit_label (all_done_label
);
1999 expand_ubsan_result_store (target
, res
);
2001 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2005 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2008 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2009 tree arg0
, tree arg1
)
2011 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2012 rtx_code_label
*loop_lab
= NULL
;
2013 rtx cntvar
= NULL_RTX
;
2014 tree cntv
= NULL_TREE
;
2015 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2016 tree sz
= TYPE_SIZE (eltype
);
2017 tree data
= NULL_TREE
;
2018 tree resv
= NULL_TREE
;
2019 rtx lhsr
= NULL_RTX
;
2020 rtx resvr
= NULL_RTX
;
2021 unsigned HOST_WIDE_INT const_cnt
= 0;
2022 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2027 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2028 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2029 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2030 optab_default
)) == unknown_optab
2031 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2032 == CODE_FOR_nothing
))
2035 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2038 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2039 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2045 do_pending_stack_adjust ();
2046 loop_lab
= gen_label_rtx ();
2047 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2048 cntv
= make_tree (sizetype
, cntvar
);
2049 emit_move_insn (cntvar
, const0_rtx
);
2050 emit_label (loop_lab
);
2052 if (TREE_CODE (arg0
) != VECTOR_CST
)
2054 rtx arg0r
= expand_normal (arg0
);
2055 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2057 if (TREE_CODE (arg1
) != VECTOR_CST
)
2059 rtx arg1r
= expand_normal (arg1
);
2060 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2062 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2064 tree op0
, op1
, res
= NULL_TREE
;
2067 tree atype
= build_array_type_nelts (eltype
, cnt
);
2068 op0
= uniform_vector_p (arg0
);
2069 if (op0
== NULL_TREE
)
2071 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2072 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2073 NULL_TREE
, NULL_TREE
);
2075 op1
= uniform_vector_p (arg1
);
2076 if (op1
== NULL_TREE
)
2078 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2079 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2080 NULL_TREE
, NULL_TREE
);
2084 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2085 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2086 NULL_TREE
, NULL_TREE
);
2091 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2092 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2093 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2095 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2101 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2102 false, false, false, true, &data
);
2105 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2106 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2108 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2109 false, false, false, true, &data
);
2112 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2121 struct separate_ops ops
;
2122 ops
.code
= PLUS_EXPR
;
2123 ops
.type
= TREE_TYPE (cntv
);
2125 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2126 ops
.op2
= NULL_TREE
;
2128 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2131 emit_move_insn (cntvar
, ret
);
2132 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2133 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2134 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2135 profile_probability::very_likely ());
2137 if (lhs
&& resv
== NULL_TREE
)
2139 struct separate_ops ops
;
2141 ops
.type
= TREE_TYPE (arg0
);
2144 ops
.op2
= NULL_TREE
;
2146 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2149 emit_move_insn (lhsr
, ret
);
2152 emit_move_insn (lhsr
, resvr
);
2155 /* Expand UBSAN_CHECK_ADD call STMT. */
2158 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2160 location_t loc
= gimple_location (stmt
);
2161 tree lhs
= gimple_call_lhs (stmt
);
2162 tree arg0
= gimple_call_arg (stmt
, 0);
2163 tree arg1
= gimple_call_arg (stmt
, 1);
2164 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2165 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2167 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2168 false, false, false, true, NULL
);
2171 /* Expand UBSAN_CHECK_SUB call STMT. */
2174 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2176 location_t loc
= gimple_location (stmt
);
2177 tree lhs
= gimple_call_lhs (stmt
);
2178 tree arg0
= gimple_call_arg (stmt
, 0);
2179 tree arg1
= gimple_call_arg (stmt
, 1);
2180 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2181 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2182 else if (integer_zerop (arg0
))
2183 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2185 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2186 false, false, false, true, NULL
);
2189 /* Expand UBSAN_CHECK_MUL call STMT. */
2192 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2194 location_t loc
= gimple_location (stmt
);
2195 tree lhs
= gimple_call_lhs (stmt
);
2196 tree arg0
= gimple_call_arg (stmt
, 0);
2197 tree arg1
= gimple_call_arg (stmt
, 1);
2198 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2199 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2201 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2205 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2208 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2210 tree lhs
= gimple_call_lhs (stmt
);
2211 if (lhs
== NULL_TREE
)
2213 tree arg0
= gimple_call_arg (stmt
, 0);
2214 tree arg1
= gimple_call_arg (stmt
, 1);
2215 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2216 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2217 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2218 int unsr_p
= TYPE_UNSIGNED (type
);
2219 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2220 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2221 int precres
= TYPE_PRECISION (type
);
2222 location_t loc
= gimple_location (stmt
);
2223 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2225 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2227 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2228 prec0
= MIN (prec0
, pr
);
2229 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2230 prec1
= MIN (prec1
, pr
);
2232 /* If uns0_p && uns1_p, precop is minimum needed precision
2233 of unsigned type to hold the exact result, otherwise
2234 precop is minimum needed precision of signed type to
2235 hold the exact result. */
2237 if (code
== MULT_EXPR
)
2238 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2241 if (uns0_p
== uns1_p
)
2242 precop
= MAX (prec0
, prec1
) + 1;
2244 precop
= MAX (prec0
+ 1, prec1
) + 1;
2246 precop
= MAX (prec0
, prec1
+ 1) + 1;
2248 int orig_precres
= precres
;
2252 if ((uns0_p
&& uns1_p
)
2253 ? ((precop
+ !unsr_p
) <= precres
2254 /* u1 - u2 -> ur can overflow, no matter what precision
2256 && (code
!= MINUS_EXPR
|| !unsr_p
))
2257 : (!unsr_p
&& precop
<= precres
))
2259 /* The infinity precision result will always fit into result. */
2260 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2261 write_complex_part (target
, const0_rtx
, true);
2262 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2263 struct separate_ops ops
;
2266 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2267 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2268 ops
.op2
= NULL_TREE
;
2270 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2271 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2275 /* For operations with low precision, if target doesn't have them, start
2276 with precres widening right away, otherwise do it only if the most
2277 simple cases can't be used. */
2278 const int min_precision
= targetm
.min_arithmetic_precision ();
2279 if (orig_precres
== precres
&& precres
< min_precision
)
2281 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2282 && prec1
<= precres
)
2283 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2284 && prec0
+ uns0_p
<= precres
2285 && prec1
+ uns1_p
<= precres
))
2287 arg0
= fold_convert_loc (loc
, type
, arg0
);
2288 arg1
= fold_convert_loc (loc
, type
, arg1
);
2292 if (integer_zerop (arg0
) && !unsr_p
)
2294 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2299 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2300 unsr_p
, unsr_p
, false, NULL
);
2303 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2304 unsr_p
, unsr_p
, false, NULL
);
2311 /* For sub-word operations, retry with a wider type first. */
2312 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2314 int p
= MAX (min_precision
, precop
);
2315 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2316 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2319 p
= TYPE_PRECISION (optype
);
2323 unsr_p
= TYPE_UNSIGNED (optype
);
2329 if (prec0
<= precres
&& prec1
<= precres
)
2334 types
[0] = build_nonstandard_integer_type (precres
, 0);
2340 types
[1] = build_nonstandard_integer_type (precres
, 1);
2342 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2343 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2344 if (code
!= MULT_EXPR
)
2345 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2346 uns0_p
, uns1_p
, false, NULL
);
2348 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2349 uns0_p
, uns1_p
, false, NULL
);
2353 /* Retry with a wider type. */
2354 if (orig_precres
== precres
)
2356 int p
= MAX (prec0
, prec1
);
2357 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2358 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2361 p
= TYPE_PRECISION (optype
);
2365 unsr_p
= TYPE_UNSIGNED (optype
);
2376 /* Expand ADD_OVERFLOW STMT. */
2379 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2381 expand_arith_overflow (PLUS_EXPR
, stmt
);
2384 /* Expand SUB_OVERFLOW STMT. */
2387 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2389 expand_arith_overflow (MINUS_EXPR
, stmt
);
2392 /* Expand MUL_OVERFLOW STMT. */
2395 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2397 expand_arith_overflow (MULT_EXPR
, stmt
);
2400 /* This should get folded in tree-vectorizer.c. */
2403 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2408 /* This should get folded in tree-vectorizer.c. */
2411 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2416 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2417 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2420 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2422 tree addr
= gimple_call_arg (stmt
, index
);
2423 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2424 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2425 if (TYPE_ALIGN (type
) != align
)
2426 type
= build_aligned_type (type
, align
);
2429 if (TREE_CODE (tmp
) == SSA_NAME
)
2431 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2432 if (gimple_assign_single_p (def
))
2433 tmp
= gimple_assign_rhs1 (def
);
2436 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2438 tree mem
= TREE_OPERAND (tmp
, 0);
2439 if (TREE_CODE (mem
) == TARGET_MEM_REF
2440 && types_compatible_p (TREE_TYPE (mem
), type
))
2442 tree offset
= TMR_OFFSET (mem
);
2443 if (alias_ptr_type
!= TREE_TYPE (offset
) || !integer_zerop (offset
))
2445 mem
= copy_node (mem
);
2446 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2447 wi::to_poly_wide (offset
));
2453 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2456 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2459 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2461 struct expand_operand ops
[3];
2462 tree type
, lhs
, rhs
, maskt
;
2463 rtx mem
, target
, mask
;
2466 maskt
= gimple_call_arg (stmt
, 2);
2467 lhs
= gimple_call_lhs (stmt
);
2468 if (lhs
== NULL_TREE
)
2470 type
= TREE_TYPE (lhs
);
2471 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2473 if (optab
== vec_mask_load_lanes_optab
)
2474 icode
= get_multi_vector_move (type
, optab
);
2476 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2477 TYPE_MODE (TREE_TYPE (maskt
)));
2479 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2480 gcc_assert (MEM_P (mem
));
2481 mask
= expand_normal (maskt
);
2482 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2483 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2484 create_fixed_operand (&ops
[1], mem
);
2485 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2486 expand_insn (icode
, 3, ops
);
2489 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2491 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2494 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2496 struct expand_operand ops
[3];
2497 tree type
, lhs
, rhs
, maskt
;
2501 maskt
= gimple_call_arg (stmt
, 2);
2502 rhs
= gimple_call_arg (stmt
, 3);
2503 type
= TREE_TYPE (rhs
);
2504 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2506 if (optab
== vec_mask_store_lanes_optab
)
2507 icode
= get_multi_vector_move (type
, optab
);
2509 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2510 TYPE_MODE (TREE_TYPE (maskt
)));
2512 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2513 gcc_assert (MEM_P (mem
));
2514 mask
= expand_normal (maskt
);
2515 reg
= expand_normal (rhs
);
2516 create_fixed_operand (&ops
[0], mem
);
2517 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2518 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2519 expand_insn (icode
, 3, ops
);
2522 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2525 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2530 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2532 /* When guessing was done, the hints should be already stripped away. */
2533 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2536 tree lhs
= gimple_call_lhs (stmt
);
2538 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2540 target
= const0_rtx
;
2541 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2542 if (lhs
&& val
!= target
)
2543 emit_move_insn (target
, val
);
2546 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2547 should never be called. */
2550 expand_VA_ARG (internal_fn
, gcall
*)
2555 /* Expand the IFN_UNIQUE function according to its first argument. */
2558 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2560 rtx pattern
= NULL_RTX
;
2561 enum ifn_unique_kind kind
2562 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2569 case IFN_UNIQUE_UNSPEC
:
2570 if (targetm
.have_unique ())
2571 pattern
= targetm
.gen_unique ();
2574 case IFN_UNIQUE_OACC_FORK
:
2575 case IFN_UNIQUE_OACC_JOIN
:
2576 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2578 tree lhs
= gimple_call_lhs (stmt
);
2579 rtx target
= const0_rtx
;
2582 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2584 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2585 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2587 if (kind
== IFN_UNIQUE_OACC_FORK
)
2588 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2590 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2598 emit_insn (pattern
);
2601 /* The size of an OpenACC compute dimension. */
2604 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2606 tree lhs
= gimple_call_lhs (stmt
);
2611 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2612 if (targetm
.have_oacc_dim_size ())
2614 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2615 VOIDmode
, EXPAND_NORMAL
);
2616 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2619 emit_move_insn (target
, GEN_INT (1));
2622 /* The position of an OpenACC execution engine along one compute axis. */
2625 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2627 tree lhs
= gimple_call_lhs (stmt
);
2632 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2633 if (targetm
.have_oacc_dim_pos ())
2635 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2636 VOIDmode
, EXPAND_NORMAL
);
2637 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2640 emit_move_insn (target
, const0_rtx
);
2643 /* This is expanded by oacc_device_lower pass. */
2646 expand_GOACC_LOOP (internal_fn
, gcall
*)
2651 /* This is expanded by oacc_device_lower pass. */
2654 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2659 /* This is expanded by oacc_device_lower pass. */
2662 expand_GOACC_TILE (internal_fn
, gcall
*)
2667 /* Set errno to EDOM. */
2670 expand_SET_EDOM (internal_fn
, gcall
*)
2673 #ifdef GEN_ERRNO_RTX
2674 rtx errno_rtx
= GEN_ERRNO_RTX
;
2676 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2678 emit_move_insn (errno_rtx
,
2679 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2685 /* Expand atomic bit test and set. */
2688 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2690 expand_ifn_atomic_bit_test_and (call
);
2693 /* Expand atomic bit test and complement. */
2696 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2698 expand_ifn_atomic_bit_test_and (call
);
2701 /* Expand atomic bit test and reset. */
2704 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2706 expand_ifn_atomic_bit_test_and (call
);
2709 /* Expand atomic bit test and set. */
2712 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2714 expand_ifn_atomic_compare_exchange (call
);
2717 /* Expand LAUNDER to assignment, lhs = arg0. */
2720 expand_LAUNDER (internal_fn
, gcall
*call
)
2722 tree lhs
= gimple_call_lhs (call
);
2727 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2730 /* Expand DIVMOD() using:
2731 a) optab handler for udivmod/sdivmod if it is available.
2732 b) If optab_handler doesn't exist, generate call to
2733 target-specific divmod libfunc. */
2736 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2738 tree lhs
= gimple_call_lhs (call_stmt
);
2739 tree arg0
= gimple_call_arg (call_stmt
, 0);
2740 tree arg1
= gimple_call_arg (call_stmt
, 1);
2742 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2743 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2744 machine_mode mode
= TYPE_MODE (type
);
2745 bool unsignedp
= TYPE_UNSIGNED (type
);
2746 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2748 rtx op0
= expand_normal (arg0
);
2749 rtx op1
= expand_normal (arg1
);
2750 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2752 rtx quotient
, remainder
, libfunc
;
2754 /* Check if optab_handler exists for divmod_optab for given mode. */
2755 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2757 quotient
= gen_reg_rtx (mode
);
2758 remainder
= gen_reg_rtx (mode
);
2759 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2762 /* Generate call to divmod libfunc if it exists. */
2763 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2764 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2765 "ient
, &remainder
);
2770 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2771 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2772 make_tree (TREE_TYPE (arg0
), quotient
),
2773 make_tree (TREE_TYPE (arg1
), remainder
)),
2774 target
, VOIDmode
, EXPAND_NORMAL
);
2780 expand_NOP (internal_fn
, gcall
*)
2782 /* Nothing. But it shouldn't really prevail. */
2785 /* Expand a call to FN using the operands in STMT. FN has a single
2786 output operand and NARGS input operands. */
2789 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2792 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2794 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2795 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2797 tree lhs
= gimple_call_lhs (stmt
);
2798 tree lhs_type
= TREE_TYPE (lhs
);
2799 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2801 /* Do not assign directly to a promoted subreg, since there is no
2802 guarantee that the instruction will leave the upper bits of the
2803 register in the state required by SUBREG_PROMOTED_SIGN. */
2805 if (GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
2808 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
2810 for (unsigned int i
= 0; i
< nargs
; ++i
)
2812 tree rhs
= gimple_call_arg (stmt
, i
);
2813 tree rhs_type
= TREE_TYPE (rhs
);
2814 rtx rhs_rtx
= expand_normal (rhs
);
2815 if (INTEGRAL_TYPE_P (rhs_type
))
2816 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2817 TYPE_MODE (rhs_type
),
2818 TYPE_UNSIGNED (rhs_type
));
2820 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2823 expand_insn (icode
, nargs
+ 1, ops
);
2824 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2826 /* If the return value has an integral type, convert the instruction
2827 result to that type. This is useful for things that return an
2828 int regardless of the size of the input. If the instruction result
2829 is smaller than required, assume that it is signed.
2831 If the return value has a nonintegral type, its mode must match
2832 the instruction result. */
2833 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2835 /* If this is a scalar in a register that is stored in a wider
2836 mode than the declared mode, compute the result into its
2837 declared mode and then convert to the wider mode. */
2838 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2839 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2840 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2841 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2843 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2844 emit_move_insn (lhs_rtx
, ops
[0].value
);
2847 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2848 convert_move (lhs_rtx
, ops
[0].value
, 0);
2853 /* Expand WHILE_ULT call STMT using optab OPTAB. */
2856 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2858 expand_operand ops
[3];
2861 tree lhs
= gimple_call_lhs (stmt
);
2862 tree lhs_type
= TREE_TYPE (lhs
);
2863 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2864 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
2866 for (unsigned int i
= 0; i
< 2; ++i
)
2868 tree rhs
= gimple_call_arg (stmt
, i
);
2869 rhs_type
[i
] = TREE_TYPE (rhs
);
2870 rtx rhs_rtx
= expand_normal (rhs
);
2871 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
2874 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
2875 TYPE_MODE (lhs_type
));
2877 expand_insn (icode
, 3, ops
);
2878 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2879 emit_move_insn (lhs_rtx
, ops
[0].value
);
2882 /* Expanders for optabs that can use expand_direct_optab_fn. */
2884 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2885 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2887 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2888 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2890 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
2891 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2893 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
2894 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2896 /* RETURN_TYPE and ARGS are a return type and argument list that are
2897 in principle compatible with FN (which satisfies direct_internal_fn_p).
2898 Return the types that should be used to determine whether the
2899 target supports FN. */
2902 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2904 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2905 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2906 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2907 return tree_pair (type0
, type1
);
2910 /* CALL is a call whose return type and arguments are in principle
2911 compatible with FN (which satisfies direct_internal_fn_p). Return the
2912 types that should be used to determine whether the target supports FN. */
2915 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2917 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2918 tree op0
= (info
.type0
< 0
2919 ? gimple_call_lhs (call
)
2920 : gimple_call_arg (call
, info
.type0
));
2921 tree op1
= (info
.type1
< 0
2922 ? gimple_call_lhs (call
)
2923 : gimple_call_arg (call
, info
.type1
));
2924 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2927 /* Return true if OPTAB is supported for TYPES (whose modes should be
2928 the same) when the optimization type is OPT_TYPE. Used for simple
2932 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2933 optimization_type opt_type
)
2935 machine_mode mode
= TYPE_MODE (types
.first
);
2936 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2937 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2940 /* Return true if OPTAB is supported for TYPES, where the first type
2941 is the destination and the second type is the source. Used for
2945 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
2946 optimization_type opt_type
)
2948 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
2949 TYPE_MODE (types
.second
), opt_type
)
2950 != CODE_FOR_nothing
);
2953 /* Return true if load/store lanes optab OPTAB is supported for
2954 array type TYPES.first when the optimization type is OPT_TYPE. */
2957 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2958 optimization_type opt_type
)
2960 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2961 machine_mode imode
= TYPE_MODE (types
.first
);
2962 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2963 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2964 != CODE_FOR_nothing
);
2967 #define direct_unary_optab_supported_p direct_optab_supported_p
2968 #define direct_binary_optab_supported_p direct_optab_supported_p
2969 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
2970 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
2971 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2972 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2973 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
2974 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2975 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2976 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
2977 #define direct_while_optab_supported_p convert_optab_supported_p
2979 /* Return the optab used by internal function FN. */
2982 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
2986 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2987 case IFN_##CODE: break;
2988 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2989 case IFN_##CODE: return OPTAB##_optab;
2990 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
2991 UNSIGNED_OPTAB, TYPE) \
2992 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
2993 ? UNSIGNED_OPTAB ## _optab \
2994 : SIGNED_OPTAB ## _optab);
2995 #include "internal-fn.def"
3003 /* Return true if FN is supported for the types in TYPES when the
3004 optimization type is OPT_TYPE. The types are those associated with
3005 the "type0" and "type1" fields of FN's direct_internal_fn_info
3009 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3010 optimization_type opt_type
)
3014 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3015 case IFN_##CODE: break;
3016 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3018 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3020 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3021 UNSIGNED_OPTAB, TYPE) \
3024 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3025 ? UNSIGNED_OPTAB ## _optab \
3026 : SIGNED_OPTAB ## _optab); \
3027 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3030 #include "internal-fn.def"
3038 /* Return true if FN is supported for type TYPE when the optimization
3039 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3040 fields of FN's direct_internal_fn_info structure are the same. */
3043 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3044 optimization_type opt_type
)
3046 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3047 gcc_checking_assert (info
.type0
== info
.type1
);
3048 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3051 /* Return true if IFN_SET_EDOM is supported. */
3054 set_edom_supported_p (void)
3063 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3065 expand_##CODE (internal_fn fn, gcall *stmt) \
3067 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3069 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3070 UNSIGNED_OPTAB, TYPE) \
3072 expand_##CODE (internal_fn fn, gcall *stmt) \
3074 tree_pair types = direct_internal_fn_types (fn, stmt); \
3075 optab which_optab = direct_internal_fn_optab (fn, types); \
3076 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3078 #include "internal-fn.def"
3080 /* Routines to expand each internal function, indexed by function number.
3081 Each routine has the prototype:
3083 expand_<NAME> (gcall *stmt)
3085 where STMT is the statement that performs the call. */
3086 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3087 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3088 #include "internal-fn.def"
3092 /* Return a function that performs the conditional form of CODE, i.e.:
3094 LHS = RHS1 ? RHS2 CODE RHS3 : RHS2
3096 (operating elementwise if the operands are vectors). Return IFN_LAST
3097 if no such function exists. */
3100 get_conditional_internal_fn (tree_code code
)
3105 return IFN_COND_ADD
;
3107 return IFN_COND_SUB
;
3109 return IFN_COND_MIN
;
3111 return IFN_COND_MAX
;
3113 return IFN_COND_AND
;
3115 return IFN_COND_IOR
;
3117 return IFN_COND_XOR
;
3123 /* Expand STMT as though it were a call to internal function FN. */
3126 expand_internal_call (internal_fn fn
, gcall
*stmt
)
3128 internal_fn_expanders
[fn
] (fn
, stmt
);
3131 /* Expand STMT, which is a call to internal function FN. */
3134 expand_internal_call (gcall
*stmt
)
3136 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
3140 expand_PHI (internal_fn
, gcall
*)