2 Copyright (C) 2011-2016 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
35 #include "diagnostic-core.h"
36 #include "fold-const.h"
37 #include "internal-fn.h"
38 #include "stor-layout.h"
45 /* The names of each internal function, indexed by function number. */
46 const char *const internal_fn_name_array
[] = {
47 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
48 #include "internal-fn.def"
52 /* The ECF_* flags of each internal function, indexed by function number. */
53 const int internal_fn_flags_array
[] = {
54 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
55 #include "internal-fn.def"
59 /* Fnspec of each internal function, indexed by function number. */
60 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
65 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
66 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
67 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
68 #include "internal-fn.def"
69 internal_fn_fnspec_array
[IFN_LAST
] = 0;
72 /* Create static initializers for the information returned by
73 direct_internal_fn. */
74 #define not_direct { -2, -2, false }
75 #define mask_load_direct { -1, 2, false }
76 #define load_lanes_direct { -1, -1, false }
77 #define mask_store_direct { 3, 2, false }
78 #define store_lanes_direct { 0, 0, false }
79 #define unary_direct { 0, 0, true }
80 #define binary_direct { 0, 0, true }
82 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
83 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
84 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
85 #include "internal-fn.def"
89 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
90 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
93 get_multi_vector_move (tree array_type
, convert_optab optab
)
98 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
99 imode
= TYPE_MODE (array_type
);
100 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
102 return convert_optab_handler (optab
, imode
, vmode
);
105 /* Expand LOAD_LANES call STMT using optab OPTAB. */
108 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
110 struct expand_operand ops
[2];
114 lhs
= gimple_call_lhs (stmt
);
115 rhs
= gimple_call_arg (stmt
, 0);
116 type
= TREE_TYPE (lhs
);
118 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
119 mem
= expand_normal (rhs
);
121 gcc_assert (MEM_P (mem
));
122 PUT_MODE (mem
, TYPE_MODE (type
));
124 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
125 create_fixed_operand (&ops
[1], mem
);
126 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
129 /* Expand STORE_LANES call STMT using optab OPTAB. */
132 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
134 struct expand_operand ops
[2];
138 lhs
= gimple_call_lhs (stmt
);
139 rhs
= gimple_call_arg (stmt
, 0);
140 type
= TREE_TYPE (rhs
);
142 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
143 reg
= expand_normal (rhs
);
145 gcc_assert (MEM_P (target
));
146 PUT_MODE (target
, TYPE_MODE (type
));
148 create_fixed_operand (&ops
[0], target
);
149 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
150 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
154 expand_ANNOTATE (internal_fn
, gcall
*)
159 /* This should get expanded in adjust_simduid_builtins. */
162 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
167 /* This should get expanded in adjust_simduid_builtins. */
170 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
175 /* This should get expanded in adjust_simduid_builtins. */
178 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
183 /* This should get expanded in adjust_simduid_builtins. */
186 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
191 /* This should get expanded in adjust_simduid_builtins. */
194 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
199 /* This should get expanded in the sanopt pass. */
202 expand_UBSAN_NULL (internal_fn
, gcall
*)
207 /* This should get expanded in the sanopt pass. */
210 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
215 /* This should get expanded in the sanopt pass. */
218 expand_UBSAN_VPTR (internal_fn
, gcall
*)
223 /* This should get expanded in the sanopt pass. */
226 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
231 /* This should get expanded in the sanopt pass. */
234 expand_ASAN_CHECK (internal_fn
, gcall
*)
239 /* This should get expanded in the tsan pass. */
242 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
247 /* Helper function for expand_addsub_overflow. Return 1
248 if ARG interpreted as signed in its precision is known to be always
249 positive or 2 if ARG is known to be always negative, or 3 if ARG may
250 be positive or negative. */
253 get_range_pos_neg (tree arg
)
255 if (arg
== error_mark_node
)
258 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
260 if (TREE_CODE (arg
) == INTEGER_CST
)
262 wide_int w
= wi::sext (arg
, prec
);
268 while (CONVERT_EXPR_P (arg
)
269 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
270 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
272 arg
= TREE_OPERAND (arg
, 0);
273 /* Narrower value zero extended into wider type
274 will always result in positive values. */
275 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
276 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
278 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
283 if (TREE_CODE (arg
) != SSA_NAME
)
285 wide_int arg_min
, arg_max
;
286 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
288 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
289 if (is_gimple_assign (g
)
290 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
292 tree t
= gimple_assign_rhs1 (g
);
293 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
294 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
296 if (TYPE_UNSIGNED (TREE_TYPE (t
))
297 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
299 prec
= TYPE_PRECISION (TREE_TYPE (t
));
308 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
310 /* For unsigned values, the "positive" range comes
311 below the "negative" range. */
312 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
314 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
319 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
321 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
327 /* Return minimum precision needed to represent all values
328 of ARG in SIGNed integral type. */
331 get_min_precision (tree arg
, signop sign
)
333 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
335 signop orig_sign
= sign
;
336 if (TREE_CODE (arg
) == INTEGER_CST
)
339 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
341 widest_int w
= wi::to_widest (arg
);
342 w
= wi::ext (w
, prec
, sign
);
343 p
= wi::min_precision (w
, sign
);
346 p
= wi::min_precision (arg
, sign
);
347 return MIN (p
, prec
);
349 while (CONVERT_EXPR_P (arg
)
350 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
351 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
353 arg
= TREE_OPERAND (arg
, 0);
354 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
356 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
358 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
359 return prec
+ (orig_sign
!= sign
);
360 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
363 return prec
+ (orig_sign
!= sign
);
365 if (TREE_CODE (arg
) != SSA_NAME
)
366 return prec
+ (orig_sign
!= sign
);
367 wide_int arg_min
, arg_max
;
368 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
370 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
371 if (is_gimple_assign (g
)
372 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
374 tree t
= gimple_assign_rhs1 (g
);
375 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
376 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
379 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
381 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
383 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
384 return prec
+ (orig_sign
!= sign
);
385 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
388 return prec
+ (orig_sign
!= sign
);
392 return prec
+ (orig_sign
!= sign
);
394 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
396 int p1
= wi::min_precision (arg_min
, sign
);
397 int p2
= wi::min_precision (arg_max
, sign
);
399 prec
= MIN (prec
, p1
);
401 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
403 int p
= wi::min_precision (arg_max
, UNSIGNED
);
404 prec
= MIN (prec
, p
);
406 return prec
+ (orig_sign
!= sign
);
409 /* Helper for expand_*_overflow. Set the __imag__ part to true
410 (1 except for signed:1 type, in which case store -1). */
413 expand_arith_set_overflow (tree lhs
, rtx target
)
415 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
416 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
417 write_complex_part (target
, constm1_rtx
, true);
419 write_complex_part (target
, const1_rtx
, true);
422 /* Helper for expand_*_overflow. Store RES into the __real__ part
423 of TARGET. If RES has larger MODE than __real__ part of TARGET,
424 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
425 if LHS has smaller precision than its mode. */
428 expand_arith_overflow_result_store (tree lhs
, rtx target
,
429 machine_mode mode
, rtx res
)
431 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
435 rtx_code_label
*done_label
= gen_label_rtx ();
436 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
437 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
438 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
439 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
440 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
442 expand_arith_set_overflow (lhs
, target
);
443 emit_label (done_label
);
445 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
446 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
449 rtx_code_label
*done_label
= gen_label_rtx ();
450 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
455 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
457 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
458 true, OPTAB_LIB_WIDEN
);
462 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
464 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
467 do_compare_rtx_and_jump (res
, lres
,
468 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
470 expand_arith_set_overflow (lhs
, target
);
471 emit_label (done_label
);
473 write_complex_part (target
, lres
, false);
476 /* Helper for expand_*_overflow. Store RES into TARGET. */
479 expand_ubsan_result_store (rtx target
, rtx res
)
481 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
482 /* If this is a scalar in a register that is stored in a wider mode
483 than the declared mode, compute the result into its declared mode
484 and then convert to the wider mode. Our value is the computed
486 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
488 emit_move_insn (target
, res
);
491 /* Add sub/add overflow checking to the statement STMT.
492 CODE says whether the operation is +, or -. */
495 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
496 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
497 bool uns1_p
, bool is_ubsan
)
499 rtx res
, target
= NULL_RTX
;
501 rtx_code_label
*done_label
= gen_label_rtx ();
502 rtx_code_label
*do_error
= gen_label_rtx ();
503 do_pending_stack_adjust ();
504 rtx op0
= expand_normal (arg0
);
505 rtx op1
= expand_normal (arg1
);
506 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
507 int prec
= GET_MODE_PRECISION (mode
);
508 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
512 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
516 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
518 write_complex_part (target
, const0_rtx
, true);
521 /* We assume both operands and result have the same precision
522 here (GET_MODE_BITSIZE (mode)), S stands for signed type
523 with that precision, U for unsigned type with that precision,
524 sgn for unsigned most significant bit in that precision.
525 s1 is signed first operand, u1 is unsigned first operand,
526 s2 is signed second operand, u2 is unsigned second operand,
527 sr is signed result, ur is unsigned result and the following
528 rules say how to compute result (which is always result of
529 the operands as if both were unsigned, cast to the right
530 signedness) and how to compute whether operation overflowed.
533 res = (S) ((U) s1 + (U) s2)
534 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
536 res = (S) ((U) s1 - (U) s2)
537 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
540 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
543 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
545 res = (S) ((U) s1 + u2)
546 ovf = ((U) res ^ sgn) < u2
551 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
553 res = (S) ((U) s1 - u2)
554 ovf = u2 > ((U) s1 ^ sgn)
557 ovf = s1 < 0 || u2 > (U) s1
560 ovf = u1 >= ((U) s2 ^ sgn)
565 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
567 res = (U) s1 + (U) s2
568 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
571 ovf = (U) res < u2 || res < 0
574 ovf = u1 >= u2 ? res < 0 : res >= 0
576 res = (U) s1 - (U) s2
577 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
579 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
581 /* PLUS_EXPR is commutative, if operand signedness differs,
582 canonicalize to the first operand being signed and second
583 unsigned to simplify following code. */
584 std::swap (op0
, op1
);
585 std::swap (arg0
, arg1
);
591 if (uns0_p
&& uns1_p
&& unsr_p
)
593 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
594 : usubv4_optab
, mode
);
595 if (icode
!= CODE_FOR_nothing
)
597 struct expand_operand ops
[4];
598 rtx_insn
*last
= get_last_insn ();
600 res
= gen_reg_rtx (mode
);
601 create_output_operand (&ops
[0], res
, mode
);
602 create_input_operand (&ops
[1], op0
, mode
);
603 create_input_operand (&ops
[2], op1
, mode
);
604 create_fixed_operand (&ops
[3], do_error
);
605 if (maybe_expand_insn (icode
, 4, ops
))
607 last
= get_last_insn ();
608 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
610 && any_condjump_p (last
)
611 && !find_reg_note (last
, REG_BR_PROB
, 0))
612 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
613 emit_jump (done_label
);
617 delete_insns_since (last
);
620 /* Compute the operation. On RTL level, the addition is always
622 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
623 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
625 /* For PLUS_EXPR, the operation is commutative, so we can pick
626 operand to compare against. For prec <= BITS_PER_WORD, I think
627 preferring REG operand is better over CONST_INT, because
628 the CONST_INT might enlarge the instruction or CSE would need
629 to figure out we'd already loaded it into a register before.
630 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
631 as then the multi-word comparison can be perhaps simplified. */
632 if (code
== PLUS_EXPR
633 && (prec
<= BITS_PER_WORD
634 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
635 : CONST_SCALAR_INT_P (op1
)))
637 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
638 true, mode
, NULL_RTX
, NULL
, done_label
,
644 if (!uns0_p
&& uns1_p
&& !unsr_p
)
646 /* Compute the operation. On RTL level, the addition is always
648 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
649 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
650 rtx tem
= expand_binop (mode
, add_optab
,
651 code
== PLUS_EXPR
? res
: op0
, sgn
,
652 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
653 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
654 done_label
, PROB_VERY_LIKELY
);
659 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
661 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
663 /* As we've changed op1, we have to avoid using the value range
664 for the original argument. */
665 arg1
= error_mark_node
;
671 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
673 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
675 /* As we've changed op0, we have to avoid using the value range
676 for the original argument. */
677 arg0
= error_mark_node
;
683 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
685 /* Compute the operation. On RTL level, the addition is always
687 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
689 int pos_neg
= get_range_pos_neg (arg0
);
691 /* If ARG0 is known to be always negative, this is always overflow. */
692 emit_jump (do_error
);
693 else if (pos_neg
== 3)
694 /* If ARG0 is not known to be always positive, check at runtime. */
695 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
696 NULL
, do_error
, PROB_VERY_UNLIKELY
);
697 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
698 done_label
, PROB_VERY_LIKELY
);
703 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
705 /* Compute the operation. On RTL level, the addition is always
707 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
709 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
711 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
712 done_label
, PROB_VERY_LIKELY
);
717 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
719 /* Compute the operation. On RTL level, the addition is always
721 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
723 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
724 NULL
, do_error
, PROB_VERY_UNLIKELY
);
726 /* The operation is commutative, so we can pick operand to compare
727 against. For prec <= BITS_PER_WORD, I think preferring REG operand
728 is better over CONST_INT, because the CONST_INT might enlarge the
729 instruction or CSE would need to figure out we'd already loaded it
730 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
731 might be more beneficial, as then the multi-word comparison can be
732 perhaps simplified. */
733 if (prec
<= BITS_PER_WORD
734 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
735 : CONST_SCALAR_INT_P (op0
))
737 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
738 done_label
, PROB_VERY_LIKELY
);
743 if (!uns0_p
&& !uns1_p
&& unsr_p
)
745 /* Compute the operation. On RTL level, the addition is always
747 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
748 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
749 int pos_neg
= get_range_pos_neg (arg1
);
750 if (code
== PLUS_EXPR
)
752 int pos_neg0
= get_range_pos_neg (arg0
);
753 if (pos_neg0
!= 3 && pos_neg
== 3)
755 std::swap (op0
, op1
);
762 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
763 ? and_optab
: ior_optab
,
764 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
765 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
766 NULL
, done_label
, PROB_VERY_LIKELY
);
770 rtx_code_label
*do_ior_label
= gen_label_rtx ();
771 do_compare_rtx_and_jump (op1
, const0_rtx
,
772 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
773 NULL_RTX
, NULL
, do_ior_label
,
775 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
777 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
778 NULL
, done_label
, PROB_VERY_LIKELY
);
779 emit_jump (do_error
);
780 emit_label (do_ior_label
);
781 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
783 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
784 NULL
, done_label
, PROB_VERY_LIKELY
);
790 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
792 /* Compute the operation. On RTL level, the addition is always
794 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
796 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
797 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
798 op0_geu_op1
, PROB_EVEN
);
799 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
800 NULL
, done_label
, PROB_VERY_LIKELY
);
801 emit_jump (do_error
);
802 emit_label (op0_geu_op1
);
803 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
804 NULL
, done_label
, PROB_VERY_LIKELY
);
808 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
813 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
814 : subv4_optab
, mode
);
815 if (icode
!= CODE_FOR_nothing
)
817 struct expand_operand ops
[4];
818 rtx_insn
*last
= get_last_insn ();
820 res
= gen_reg_rtx (mode
);
821 create_output_operand (&ops
[0], res
, mode
);
822 create_input_operand (&ops
[1], op0
, mode
);
823 create_input_operand (&ops
[2], op1
, mode
);
824 create_fixed_operand (&ops
[3], do_error
);
825 if (maybe_expand_insn (icode
, 4, ops
))
827 last
= get_last_insn ();
828 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
830 && any_condjump_p (last
)
831 && !find_reg_note (last
, REG_BR_PROB
, 0))
832 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
833 emit_jump (done_label
);
837 delete_insns_since (last
);
840 rtx_code_label
*sub_check
= gen_label_rtx ();
843 /* Compute the operation. On RTL level, the addition is always
845 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
846 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
848 /* If we can prove one of the arguments (for MINUS_EXPR only
849 the second operand, as subtraction is not commutative) is always
850 non-negative or always negative, we can do just one comparison
851 and conditional jump instead of 2 at runtime, 3 present in the
852 emitted code. If one of the arguments is CONST_INT, all we
853 need is to make sure it is op1, then the first
854 do_compare_rtx_and_jump will be just folded. Otherwise try
855 to use range info if available. */
856 if (code
== PLUS_EXPR
&& CONST_INT_P (op0
))
857 std::swap (op0
, op1
);
858 else if (CONST_INT_P (op1
))
860 else if (code
== PLUS_EXPR
&& TREE_CODE (arg0
) == SSA_NAME
)
862 pos_neg
= get_range_pos_neg (arg0
);
864 std::swap (op0
, op1
);
866 if (pos_neg
== 3 && !CONST_INT_P (op1
) && TREE_CODE (arg1
) == SSA_NAME
)
867 pos_neg
= get_range_pos_neg (arg1
);
869 /* If the op1 is negative, we have to use a different check. */
871 do_compare_rtx_and_jump (op1
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
872 NULL
, sub_check
, PROB_EVEN
);
874 /* Compare the result of the operation with one of the operands. */
876 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? GE
: LE
,
877 false, mode
, NULL_RTX
, NULL
, done_label
,
880 /* If we get here, we have to print the error. */
883 emit_jump (do_error
);
884 emit_label (sub_check
);
887 /* We have k = a + b for b < 0 here. k <= a must hold. */
889 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? LE
: GE
,
890 false, mode
, NULL_RTX
, NULL
, done_label
,
895 emit_label (do_error
);
898 /* Expand the ubsan builtin call. */
900 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
904 do_pending_stack_adjust ();
907 expand_arith_set_overflow (lhs
, target
);
910 emit_label (done_label
);
915 expand_ubsan_result_store (target
, res
);
919 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
922 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
927 /* Add negate overflow checking to the statement STMT. */
930 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
)
934 rtx_code_label
*done_label
, *do_error
;
935 rtx target
= NULL_RTX
;
937 done_label
= gen_label_rtx ();
938 do_error
= gen_label_rtx ();
940 do_pending_stack_adjust ();
941 op1
= expand_normal (arg1
);
943 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
946 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
948 write_complex_part (target
, const0_rtx
, true);
951 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
952 if (icode
!= CODE_FOR_nothing
)
954 struct expand_operand ops
[3];
955 rtx_insn
*last
= get_last_insn ();
957 res
= gen_reg_rtx (mode
);
958 create_output_operand (&ops
[0], res
, mode
);
959 create_input_operand (&ops
[1], op1
, mode
);
960 create_fixed_operand (&ops
[2], do_error
);
961 if (maybe_expand_insn (icode
, 3, ops
))
963 last
= get_last_insn ();
964 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
966 && any_condjump_p (last
)
967 && !find_reg_note (last
, REG_BR_PROB
, 0))
968 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
969 emit_jump (done_label
);
973 delete_insns_since (last
);
974 icode
= CODE_FOR_nothing
;
978 if (icode
== CODE_FOR_nothing
)
980 /* Compute the operation. On RTL level, the addition is always
982 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
984 /* Compare the operand with the most negative value. */
985 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
986 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
987 done_label
, PROB_VERY_LIKELY
);
990 emit_label (do_error
);
993 /* Expand the ubsan builtin call. */
995 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
999 do_pending_stack_adjust ();
1002 expand_arith_set_overflow (lhs
, target
);
1005 emit_label (done_label
);
1010 expand_ubsan_result_store (target
, res
);
1012 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1016 /* Add mul overflow checking to the statement STMT. */
1019 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1020 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
)
1024 rtx_code_label
*done_label
, *do_error
;
1025 rtx target
= NULL_RTX
;
1027 enum insn_code icode
;
1029 done_label
= gen_label_rtx ();
1030 do_error
= gen_label_rtx ();
1032 do_pending_stack_adjust ();
1033 op0
= expand_normal (arg0
);
1034 op1
= expand_normal (arg1
);
1036 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
1040 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1042 write_complex_part (target
, const0_rtx
, true);
1046 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1048 /* We assume both operands and result have the same precision
1049 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1050 with that precision, U for unsigned type with that precision,
1051 sgn for unsigned most significant bit in that precision.
1052 s1 is signed first operand, u1 is unsigned first operand,
1053 s2 is signed second operand, u2 is unsigned second operand,
1054 sr is signed result, ur is unsigned result and the following
1055 rules say how to compute result (which is always result of
1056 the operands as if both were unsigned, cast to the right
1057 signedness) and how to compute whether operation overflowed.
1058 main_ovf (false) stands for jump on signed multiplication
1059 overflow or the main algorithm with uns == false.
1060 main_ovf (true) stands for jump on unsigned multiplication
1061 overflow or the main algorithm with uns == true.
1064 res = (S) ((U) s1 * (U) s2)
1065 ovf = main_ovf (false)
1068 ovf = main_ovf (true)
1071 ovf = (s1 < 0 && u2) || main_ovf (true)
1074 ovf = res < 0 || main_ovf (true)
1076 res = (S) ((U) s1 * u2)
1077 ovf = (S) u2 >= 0 ? main_ovf (false)
1078 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1080 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1081 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1083 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1085 if (uns0_p
&& !uns1_p
)
1087 /* Multiplication is commutative, if operand signedness differs,
1088 canonicalize to the first operand being signed and second
1089 unsigned to simplify following code. */
1090 std::swap (op0
, op1
);
1091 std::swap (arg0
, arg1
);
1096 int pos_neg0
= get_range_pos_neg (arg0
);
1097 int pos_neg1
= get_range_pos_neg (arg1
);
1100 if (!uns0_p
&& uns1_p
&& unsr_p
)
1105 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1108 /* If s1 is negative, avoid the main code, just multiply and
1109 signal overflow if op1 is not 0. */
1110 struct separate_ops ops
;
1111 ops
.code
= MULT_EXPR
;
1112 ops
.type
= TREE_TYPE (arg1
);
1113 ops
.op0
= make_tree (ops
.type
, op0
);
1114 ops
.op1
= make_tree (ops
.type
, op1
);
1115 ops
.op2
= NULL_TREE
;
1117 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1118 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1119 NULL
, done_label
, PROB_VERY_LIKELY
);
1120 goto do_error_label
;
1122 rtx_code_label
*do_main_label
;
1123 do_main_label
= gen_label_rtx ();
1124 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1125 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1126 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1127 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1128 expand_arith_set_overflow (lhs
, target
);
1129 emit_label (do_main_label
);
1137 if (uns0_p
&& uns1_p
&& !unsr_p
)
1140 /* Rest of handling of this case after res is computed. */
1145 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1152 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1153 avoid the main code, just multiply and signal overflow
1154 unless 0 * u2 or -1 * ((U) Smin). */
1155 struct separate_ops ops
;
1156 ops
.code
= MULT_EXPR
;
1157 ops
.type
= TREE_TYPE (arg1
);
1158 ops
.op0
= make_tree (ops
.type
, op0
);
1159 ops
.op1
= make_tree (ops
.type
, op1
);
1160 ops
.op2
= NULL_TREE
;
1162 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1163 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1164 NULL
, done_label
, PROB_VERY_LIKELY
);
1165 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1166 NULL
, do_error
, PROB_VERY_UNLIKELY
);
1168 prec
= GET_MODE_PRECISION (mode
);
1170 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1171 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1172 NULL
, done_label
, PROB_VERY_LIKELY
);
1173 goto do_error_label
;
1175 /* Rest of handling of this case after res is computed. */
1183 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1186 switch (pos_neg0
| pos_neg1
)
1188 case 1: /* Both operands known to be non-negative. */
1190 case 2: /* Both operands known to be negative. */
1191 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1192 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1193 /* Avoid looking at arg0/arg1 ranges, as we've changed
1195 arg0
= error_mark_node
;
1196 arg1
= error_mark_node
;
1199 if ((pos_neg0
^ pos_neg1
) == 3)
1201 /* If one operand is known to be negative and the other
1202 non-negative, this overflows always, unless the non-negative
1203 one is 0. Just do normal multiply and set overflow
1204 unless one of the operands is 0. */
1205 struct separate_ops ops
;
1206 ops
.code
= MULT_EXPR
;
1208 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1210 ops
.op0
= make_tree (ops
.type
, op0
);
1211 ops
.op1
= make_tree (ops
.type
, op1
);
1212 ops
.op2
= NULL_TREE
;
1214 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1215 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1217 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1218 NULL_RTX
, NULL
, done_label
,
1220 goto do_error_label
;
1222 /* The general case, do all the needed comparisons at runtime. */
1223 rtx_code_label
*do_main_label
, *after_negate_label
;
1225 rop0
= gen_reg_rtx (mode
);
1226 rop1
= gen_reg_rtx (mode
);
1227 emit_move_insn (rop0
, op0
);
1228 emit_move_insn (rop1
, op1
);
1231 do_main_label
= gen_label_rtx ();
1232 after_negate_label
= gen_label_rtx ();
1233 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1235 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1236 NULL
, after_negate_label
, PROB_VERY_LIKELY
);
1237 /* Both arguments negative here, negate them and continue with
1238 normal unsigned overflow checking multiplication. */
1239 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1241 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1243 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1245 arg0
= error_mark_node
;
1246 arg1
= error_mark_node
;
1247 emit_jump (do_main_label
);
1248 emit_label (after_negate_label
);
1249 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1251 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1252 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1253 /* One argument is negative here, the other positive. This
1254 overflows always, unless one of the arguments is 0. But
1255 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1256 is, thus we can keep do_main code oring in overflow as is. */
1257 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1258 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1259 expand_arith_set_overflow (lhs
, target
);
1260 emit_label (do_main_label
);
1268 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1269 sign
= uns
? UNSIGNED
: SIGNED
;
1270 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1271 if (icode
!= CODE_FOR_nothing
)
1273 struct expand_operand ops
[4];
1274 rtx_insn
*last
= get_last_insn ();
1276 res
= gen_reg_rtx (mode
);
1277 create_output_operand (&ops
[0], res
, mode
);
1278 create_input_operand (&ops
[1], op0
, mode
);
1279 create_input_operand (&ops
[2], op1
, mode
);
1280 create_fixed_operand (&ops
[3], do_error
);
1281 if (maybe_expand_insn (icode
, 4, ops
))
1283 last
= get_last_insn ();
1284 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1286 && any_condjump_p (last
)
1287 && !find_reg_note (last
, REG_BR_PROB
, 0))
1288 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1289 emit_jump (done_label
);
1293 delete_insns_since (last
);
1294 icode
= CODE_FOR_nothing
;
1298 if (icode
== CODE_FOR_nothing
)
1300 struct separate_ops ops
;
1301 int prec
= GET_MODE_PRECISION (mode
);
1302 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1303 ops
.op0
= make_tree (type
, op0
);
1304 ops
.op1
= make_tree (type
, op1
);
1305 ops
.op2
= NULL_TREE
;
1307 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1308 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1310 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1311 ops
.code
= WIDEN_MULT_EXPR
;
1313 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1315 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1316 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1318 hipart
= gen_lowpart (mode
, hipart
);
1319 res
= gen_lowpart (mode
, res
);
1321 /* For the unsigned multiplication, there was overflow if
1322 HIPART is non-zero. */
1323 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1324 NULL_RTX
, NULL
, done_label
,
1328 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1330 /* RES is low half of the double width result, HIPART
1331 the high half. There was overflow if
1332 HIPART is different from RES < 0 ? -1 : 0. */
1333 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1334 NULL_RTX
, NULL
, done_label
,
1338 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1340 rtx_code_label
*large_op0
= gen_label_rtx ();
1341 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1342 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1343 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1344 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1345 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1346 rtx_code_label
*do_overflow
= gen_label_rtx ();
1347 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1349 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1350 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1352 hipart0
= gen_lowpart (hmode
, hipart0
);
1353 rtx lopart0
= gen_lowpart (hmode
, op0
);
1354 rtx signbit0
= const0_rtx
;
1356 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1358 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1360 hipart1
= gen_lowpart (hmode
, hipart1
);
1361 rtx lopart1
= gen_lowpart (hmode
, op1
);
1362 rtx signbit1
= const0_rtx
;
1364 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1367 res
= gen_reg_rtx (mode
);
1369 /* True if op0 resp. op1 are known to be in the range of
1371 bool op0_small_p
= false;
1372 bool op1_small_p
= false;
1373 /* True if op0 resp. op1 are known to have all zeros or all ones
1374 in the upper half of bits, but are not known to be
1376 bool op0_medium_p
= false;
1377 bool op1_medium_p
= false;
1378 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1379 nonnegative, 1 if unknown. */
1385 else if (pos_neg0
== 2)
1389 else if (pos_neg1
== 2)
1392 unsigned int mprec0
= prec
;
1393 if (arg0
!= error_mark_node
)
1394 mprec0
= get_min_precision (arg0
, sign
);
1395 if (mprec0
<= hprec
)
1397 else if (!uns
&& mprec0
<= hprec
+ 1)
1398 op0_medium_p
= true;
1399 unsigned int mprec1
= prec
;
1400 if (arg1
!= error_mark_node
)
1401 mprec1
= get_min_precision (arg1
, sign
);
1402 if (mprec1
<= hprec
)
1404 else if (!uns
&& mprec1
<= hprec
+ 1)
1405 op1_medium_p
= true;
1407 int smaller_sign
= 1;
1408 int larger_sign
= 1;
1411 smaller_sign
= op0_sign
;
1412 larger_sign
= op1_sign
;
1414 else if (op1_small_p
)
1416 smaller_sign
= op1_sign
;
1417 larger_sign
= op0_sign
;
1419 else if (op0_sign
== op1_sign
)
1421 smaller_sign
= op0_sign
;
1422 larger_sign
= op0_sign
;
1426 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1427 NULL_RTX
, NULL
, large_op0
,
1431 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1432 NULL_RTX
, NULL
, small_op0_large_op1
,
1435 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1436 hmode to mode, the multiplication will never overflow. We can
1437 do just one hmode x hmode => mode widening multiplication. */
1438 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1439 if (GET_CODE (lopart0
) == SUBREG
)
1441 lopart0s
= shallow_copy_rtx (lopart0
);
1442 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1443 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1445 if (GET_CODE (lopart1
) == SUBREG
)
1447 lopart1s
= shallow_copy_rtx (lopart1
);
1448 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1449 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1451 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1452 ops
.op0
= make_tree (halfstype
, lopart0s
);
1453 ops
.op1
= make_tree (halfstype
, lopart1s
);
1454 ops
.code
= WIDEN_MULT_EXPR
;
1457 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1458 emit_move_insn (res
, thisres
);
1459 emit_jump (done_label
);
1461 emit_label (small_op0_large_op1
);
1463 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1464 but op1 is not, just swap the arguments and handle it as op1
1465 sign/zero extended, op0 not. */
1466 rtx larger
= gen_reg_rtx (mode
);
1467 rtx hipart
= gen_reg_rtx (hmode
);
1468 rtx lopart
= gen_reg_rtx (hmode
);
1469 emit_move_insn (larger
, op1
);
1470 emit_move_insn (hipart
, hipart1
);
1471 emit_move_insn (lopart
, lopart0
);
1472 emit_jump (one_small_one_large
);
1474 emit_label (large_op0
);
1477 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1478 NULL_RTX
, NULL
, both_ops_large
,
1481 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1482 but op0 is not, prepare larger, hipart and lopart pseudos and
1483 handle it together with small_op0_large_op1. */
1484 emit_move_insn (larger
, op0
);
1485 emit_move_insn (hipart
, hipart0
);
1486 emit_move_insn (lopart
, lopart1
);
1488 emit_label (one_small_one_large
);
1490 /* lopart is the low part of the operand that is sign extended
1491 to mode, larger is the other operand, hipart is the
1492 high part of larger and lopart0 and lopart1 are the low parts
1494 We perform lopart0 * lopart1 and lopart * hipart widening
1496 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1497 ops
.op0
= make_tree (halfutype
, lopart0
);
1498 ops
.op1
= make_tree (halfutype
, lopart1
);
1500 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1502 ops
.op0
= make_tree (halfutype
, lopart
);
1503 ops
.op1
= make_tree (halfutype
, hipart
);
1504 rtx loxhi
= gen_reg_rtx (mode
);
1505 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1506 emit_move_insn (loxhi
, tem
);
1510 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1511 if (larger_sign
== 0)
1512 emit_jump (after_hipart_neg
);
1513 else if (larger_sign
!= -1)
1514 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1515 NULL_RTX
, NULL
, after_hipart_neg
,
1518 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1519 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1520 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1522 emit_move_insn (loxhi
, tem
);
1524 emit_label (after_hipart_neg
);
1526 /* if (lopart < 0) loxhi -= larger; */
1527 if (smaller_sign
== 0)
1528 emit_jump (after_lopart_neg
);
1529 else if (smaller_sign
!= -1)
1530 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1531 NULL_RTX
, NULL
, after_lopart_neg
,
1534 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1536 emit_move_insn (loxhi
, tem
);
1538 emit_label (after_lopart_neg
);
1541 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1542 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1543 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1545 emit_move_insn (loxhi
, tem
);
1547 /* if (loxhi >> (bitsize / 2)
1548 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1549 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1550 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1552 hipartloxhi
= gen_lowpart (hmode
, hipartloxhi
);
1553 rtx signbitloxhi
= const0_rtx
;
1555 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1556 gen_lowpart (hmode
, loxhi
),
1557 hprec
- 1, NULL_RTX
, 0);
1559 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1560 NULL_RTX
, NULL
, do_overflow
,
1561 PROB_VERY_UNLIKELY
);
1563 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1564 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1566 tem
= convert_modes (mode
, hmode
, gen_lowpart (hmode
, lo0xlo1
), 1);
1568 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1571 emit_move_insn (res
, tem
);
1572 emit_jump (done_label
);
1574 emit_label (both_ops_large
);
1576 /* If both operands are large (not sign (!uns) or zero (uns)
1577 extended from hmode), then perform the full multiplication
1578 which will be the result of the operation.
1579 The only cases which don't overflow are for signed multiplication
1580 some cases where both hipart0 and highpart1 are 0 or -1.
1581 For unsigned multiplication when high parts are both non-zero
1582 this overflows always. */
1583 ops
.code
= MULT_EXPR
;
1584 ops
.op0
= make_tree (type
, op0
);
1585 ops
.op1
= make_tree (type
, op1
);
1586 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1587 emit_move_insn (res
, tem
);
1593 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1594 NULL_RTX
, 1, OPTAB_DIRECT
);
1595 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1596 NULL_RTX
, NULL
, do_error
,
1597 PROB_VERY_UNLIKELY
);
1602 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1603 NULL_RTX
, 1, OPTAB_DIRECT
);
1604 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1605 NULL_RTX
, NULL
, do_error
,
1606 PROB_VERY_UNLIKELY
);
1609 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1610 the same, overflow happened if res is negative, if they are
1611 different, overflow happened if res is positive. */
1612 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1613 emit_jump (hipart_different
);
1614 else if (op0_sign
== 1 || op1_sign
== 1)
1615 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1616 NULL_RTX
, NULL
, hipart_different
,
1619 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1620 NULL_RTX
, NULL
, do_error
,
1621 PROB_VERY_UNLIKELY
);
1622 emit_jump (done_label
);
1624 emit_label (hipart_different
);
1626 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1627 NULL_RTX
, NULL
, do_error
,
1628 PROB_VERY_UNLIKELY
);
1629 emit_jump (done_label
);
1632 emit_label (do_overflow
);
1634 /* Overflow, do full multiplication and fallthru into do_error. */
1635 ops
.op0
= make_tree (type
, op0
);
1636 ops
.op1
= make_tree (type
, op1
);
1637 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1638 emit_move_insn (res
, tem
);
1642 gcc_assert (!is_ubsan
);
1643 ops
.code
= MULT_EXPR
;
1645 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1646 emit_jump (done_label
);
1651 emit_label (do_error
);
1654 /* Expand the ubsan builtin call. */
1656 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1660 do_pending_stack_adjust ();
1663 expand_arith_set_overflow (lhs
, target
);
1666 emit_label (done_label
);
1669 if (uns0_p
&& uns1_p
&& !unsr_p
)
1671 rtx_code_label
*all_done_label
= gen_label_rtx ();
1672 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1673 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1674 expand_arith_set_overflow (lhs
, target
);
1675 emit_label (all_done_label
);
1679 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1681 rtx_code_label
*all_done_label
= gen_label_rtx ();
1682 rtx_code_label
*set_noovf
= gen_label_rtx ();
1683 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1684 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1685 expand_arith_set_overflow (lhs
, target
);
1686 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1687 NULL
, set_noovf
, PROB_VERY_LIKELY
);
1688 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1689 NULL
, all_done_label
, PROB_VERY_UNLIKELY
);
1690 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1691 all_done_label
, PROB_VERY_UNLIKELY
);
1692 emit_label (set_noovf
);
1693 write_complex_part (target
, const0_rtx
, true);
1694 emit_label (all_done_label
);
1700 expand_ubsan_result_store (target
, res
);
1702 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1706 /* Expand UBSAN_CHECK_ADD call STMT. */
1709 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
1711 location_t loc
= gimple_location (stmt
);
1712 tree lhs
= gimple_call_lhs (stmt
);
1713 tree arg0
= gimple_call_arg (stmt
, 0);
1714 tree arg1
= gimple_call_arg (stmt
, 1);
1715 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
1716 false, false, false, true);
1719 /* Expand UBSAN_CHECK_SUB call STMT. */
1722 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
1724 location_t loc
= gimple_location (stmt
);
1725 tree lhs
= gimple_call_lhs (stmt
);
1726 tree arg0
= gimple_call_arg (stmt
, 0);
1727 tree arg1
= gimple_call_arg (stmt
, 1);
1728 if (integer_zerop (arg0
))
1729 expand_neg_overflow (loc
, lhs
, arg1
, true);
1731 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
1732 false, false, false, true);
1735 /* Expand UBSAN_CHECK_MUL call STMT. */
1738 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
1740 location_t loc
= gimple_location (stmt
);
1741 tree lhs
= gimple_call_lhs (stmt
);
1742 tree arg0
= gimple_call_arg (stmt
, 0);
1743 tree arg1
= gimple_call_arg (stmt
, 1);
1744 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true);
1747 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1750 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
1752 tree lhs
= gimple_call_lhs (stmt
);
1753 if (lhs
== NULL_TREE
)
1755 tree arg0
= gimple_call_arg (stmt
, 0);
1756 tree arg1
= gimple_call_arg (stmt
, 1);
1757 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
1758 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
1759 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
1760 int unsr_p
= TYPE_UNSIGNED (type
);
1761 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
1762 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
1763 int precres
= TYPE_PRECISION (type
);
1764 location_t loc
= gimple_location (stmt
);
1765 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
1767 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
1769 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
1770 prec0
= MIN (prec0
, pr
);
1771 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
1772 prec1
= MIN (prec1
, pr
);
1774 /* If uns0_p && uns1_p, precop is minimum needed precision
1775 of unsigned type to hold the exact result, otherwise
1776 precop is minimum needed precision of signed type to
1777 hold the exact result. */
1779 if (code
== MULT_EXPR
)
1780 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
1783 if (uns0_p
== uns1_p
)
1784 precop
= MAX (prec0
, prec1
) + 1;
1786 precop
= MAX (prec0
+ 1, prec1
) + 1;
1788 precop
= MAX (prec0
, prec1
+ 1) + 1;
1790 int orig_precres
= precres
;
1794 if ((uns0_p
&& uns1_p
)
1795 ? ((precop
+ !unsr_p
) <= precres
1796 /* u1 - u2 -> ur can overflow, no matter what precision
1798 && (code
!= MINUS_EXPR
|| !unsr_p
))
1799 : (!unsr_p
&& precop
<= precres
))
1801 /* The infinity precision result will always fit into result. */
1802 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1803 write_complex_part (target
, const0_rtx
, true);
1804 enum machine_mode mode
= TYPE_MODE (type
);
1805 struct separate_ops ops
;
1808 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
1809 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
1810 ops
.op2
= NULL_TREE
;
1812 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1813 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
1817 /* For sub-word operations, if target doesn't have them, start
1818 with precres widening right away, otherwise do it only
1819 if the most simple cases can't be used. */
1820 if (WORD_REGISTER_OPERATIONS
1821 && orig_precres
== precres
1822 && precres
< BITS_PER_WORD
)
1824 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
1825 && prec1
<= precres
)
1826 || ((!uns0_p
|| !uns1_p
) && !unsr_p
1827 && prec0
+ uns0_p
<= precres
1828 && prec1
+ uns1_p
<= precres
))
1830 arg0
= fold_convert_loc (loc
, type
, arg0
);
1831 arg1
= fold_convert_loc (loc
, type
, arg1
);
1835 if (integer_zerop (arg0
) && !unsr_p
)
1837 expand_neg_overflow (loc
, lhs
, arg1
, false);
1842 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
,
1843 unsr_p
, unsr_p
, unsr_p
, false);
1846 expand_mul_overflow (loc
, lhs
, arg0
, arg1
,
1847 unsr_p
, unsr_p
, unsr_p
, false);
1854 /* For sub-word operations, retry with a wider type first. */
1855 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
1857 int p
= WORD_REGISTER_OPERATIONS
? BITS_PER_WORD
: precop
;
1858 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1859 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1862 p
= TYPE_PRECISION (optype
);
1866 unsr_p
= TYPE_UNSIGNED (optype
);
1872 if (prec0
<= precres
&& prec1
<= precres
)
1877 types
[0] = build_nonstandard_integer_type (precres
, 0);
1883 types
[1] = build_nonstandard_integer_type (precres
, 1);
1885 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
1886 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
1887 if (code
!= MULT_EXPR
)
1888 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
1889 uns0_p
, uns1_p
, false);
1891 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
1892 uns0_p
, uns1_p
, false);
1896 /* Retry with a wider type. */
1897 if (orig_precres
== precres
)
1899 int p
= MAX (prec0
, prec1
);
1900 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1901 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1904 p
= TYPE_PRECISION (optype
);
1908 unsr_p
= TYPE_UNSIGNED (optype
);
1919 /* Expand ADD_OVERFLOW STMT. */
1922 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
1924 expand_arith_overflow (PLUS_EXPR
, stmt
);
1927 /* Expand SUB_OVERFLOW STMT. */
1930 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
1932 expand_arith_overflow (MINUS_EXPR
, stmt
);
1935 /* Expand MUL_OVERFLOW STMT. */
1938 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
1940 expand_arith_overflow (MULT_EXPR
, stmt
);
1943 /* This should get folded in tree-vectorizer.c. */
1946 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
1951 /* Expand MASK_LOAD call STMT using optab OPTAB. */
1954 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
1956 struct expand_operand ops
[3];
1957 tree type
, lhs
, rhs
, maskt
, ptr
;
1958 rtx mem
, target
, mask
;
1961 maskt
= gimple_call_arg (stmt
, 2);
1962 lhs
= gimple_call_lhs (stmt
);
1963 if (lhs
== NULL_TREE
)
1965 type
= TREE_TYPE (lhs
);
1966 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
1967 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
1968 if (TYPE_ALIGN (type
) != align
)
1969 type
= build_aligned_type (type
, align
);
1970 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
1972 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1973 gcc_assert (MEM_P (mem
));
1974 mask
= expand_normal (maskt
);
1975 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1976 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
1977 create_fixed_operand (&ops
[1], mem
);
1978 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1979 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
1980 TYPE_MODE (TREE_TYPE (maskt
))),
1984 /* Expand MASK_STORE call STMT using optab OPTAB. */
1987 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
1989 struct expand_operand ops
[3];
1990 tree type
, lhs
, rhs
, maskt
, ptr
;
1994 maskt
= gimple_call_arg (stmt
, 2);
1995 rhs
= gimple_call_arg (stmt
, 3);
1996 type
= TREE_TYPE (rhs
);
1997 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
1998 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
1999 if (TYPE_ALIGN (type
) != align
)
2000 type
= build_aligned_type (type
, align
);
2001 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2003 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2004 gcc_assert (MEM_P (mem
));
2005 mask
= expand_normal (maskt
);
2006 reg
= expand_normal (rhs
);
2007 create_fixed_operand (&ops
[0], mem
);
2008 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2009 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2010 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2011 TYPE_MODE (TREE_TYPE (maskt
))),
2016 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2021 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2023 /* When guessing was done, the hints should be already stripped away. */
2024 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2027 tree lhs
= gimple_call_lhs (stmt
);
2029 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2031 target
= const0_rtx
;
2032 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2033 if (lhs
&& val
!= target
)
2034 emit_move_insn (target
, val
);
2037 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2038 should never be called. */
2041 expand_VA_ARG (internal_fn
, gcall
*)
2046 /* Expand the IFN_UNIQUE function according to its first argument. */
2049 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2051 rtx pattern
= NULL_RTX
;
2052 enum ifn_unique_kind kind
2053 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2060 case IFN_UNIQUE_UNSPEC
:
2061 if (targetm
.have_unique ())
2062 pattern
= targetm
.gen_unique ();
2065 case IFN_UNIQUE_OACC_FORK
:
2066 case IFN_UNIQUE_OACC_JOIN
:
2067 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2069 tree lhs
= gimple_call_lhs (stmt
);
2070 rtx target
= const0_rtx
;
2073 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2075 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2076 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2078 if (kind
== IFN_UNIQUE_OACC_FORK
)
2079 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2081 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2089 emit_insn (pattern
);
2092 /* The size of an OpenACC compute dimension. */
2095 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2097 tree lhs
= gimple_call_lhs (stmt
);
2102 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2103 if (targetm
.have_oacc_dim_size ())
2105 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2106 VOIDmode
, EXPAND_NORMAL
);
2107 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2110 emit_move_insn (target
, GEN_INT (1));
2113 /* The position of an OpenACC execution engine along one compute axis. */
2116 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2118 tree lhs
= gimple_call_lhs (stmt
);
2123 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2124 if (targetm
.have_oacc_dim_pos ())
2126 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2127 VOIDmode
, EXPAND_NORMAL
);
2128 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2131 emit_move_insn (target
, const0_rtx
);
2134 /* This is expanded by oacc_device_lower pass. */
2137 expand_GOACC_LOOP (internal_fn
, gcall
*)
2142 /* This is expanded by oacc_device_lower pass. */
2145 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2150 /* Set errno to EDOM. */
2153 expand_SET_EDOM (internal_fn
, gcall
*)
2156 #ifdef GEN_ERRNO_RTX
2157 rtx errno_rtx
= GEN_ERRNO_RTX
;
2159 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2161 emit_move_insn (errno_rtx
,
2162 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2168 /* Expand atomic bit test and set. */
2171 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2173 expand_ifn_atomic_bit_test_and (call
);
2176 /* Expand atomic bit test and complement. */
2179 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2181 expand_ifn_atomic_bit_test_and (call
);
2184 /* Expand atomic bit test and reset. */
2187 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2189 expand_ifn_atomic_bit_test_and (call
);
2192 /* Expand atomic bit test and set. */
2195 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2197 expand_ifn_atomic_compare_exchange (call
);
2200 /* Expand a call to FN using the operands in STMT. FN has a single
2201 output operand and NARGS input operands. */
2204 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2207 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2209 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2210 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2212 tree lhs
= gimple_call_lhs (stmt
);
2213 tree lhs_type
= TREE_TYPE (lhs
);
2214 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2215 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2217 for (unsigned int i
= 0; i
< nargs
; ++i
)
2219 tree rhs
= gimple_call_arg (stmt
, i
);
2220 tree rhs_type
= TREE_TYPE (rhs
);
2221 rtx rhs_rtx
= expand_normal (rhs
);
2222 if (INTEGRAL_TYPE_P (rhs_type
))
2223 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2224 TYPE_MODE (rhs_type
),
2225 TYPE_UNSIGNED (rhs_type
));
2227 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2230 expand_insn (icode
, nargs
+ 1, ops
);
2231 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2233 /* If the return value has an integral type, convert the instruction
2234 result to that type. This is useful for things that return an
2235 int regardless of the size of the input. If the instruction result
2236 is smaller than required, assume that it is signed.
2238 If the return value has a nonintegral type, its mode must match
2239 the instruction result. */
2240 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2242 /* If this is a scalar in a register that is stored in a wider
2243 mode than the declared mode, compute the result into its
2244 declared mode and then convert to the wider mode. */
2245 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2246 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2247 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2248 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2250 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2251 emit_move_insn (lhs_rtx
, ops
[0].value
);
2254 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2255 convert_move (lhs_rtx
, ops
[0].value
, 0);
2260 /* Expanders for optabs that can use expand_direct_optab_fn. */
2262 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2263 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2265 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2266 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2268 /* RETURN_TYPE and ARGS are a return type and argument list that are
2269 in principle compatible with FN (which satisfies direct_internal_fn_p).
2270 Return the types that should be used to determine whether the
2271 target supports FN. */
2274 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2276 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2277 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2278 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2279 return tree_pair (type0
, type1
);
2282 /* CALL is a call whose return type and arguments are in principle
2283 compatible with FN (which satisfies direct_internal_fn_p). Return the
2284 types that should be used to determine whether the target supports FN. */
2287 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2289 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2290 tree op0
= (info
.type0
< 0
2291 ? gimple_call_lhs (call
)
2292 : gimple_call_arg (call
, info
.type0
));
2293 tree op1
= (info
.type1
< 0
2294 ? gimple_call_lhs (call
)
2295 : gimple_call_arg (call
, info
.type1
));
2296 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2299 /* Return true if OPTAB is supported for TYPES (whose modes should be
2300 the same) when the optimization type is OPT_TYPE. Used for simple
2304 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2305 optimization_type opt_type
)
2307 machine_mode mode
= TYPE_MODE (types
.first
);
2308 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2309 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2312 /* Return true if load/store lanes optab OPTAB is supported for
2313 array type TYPES.first when the optimization type is OPT_TYPE. */
2316 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2317 optimization_type opt_type
)
2319 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2320 machine_mode imode
= TYPE_MODE (types
.first
);
2321 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2322 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2323 != CODE_FOR_nothing
);
2326 #define direct_unary_optab_supported_p direct_optab_supported_p
2327 #define direct_binary_optab_supported_p direct_optab_supported_p
2328 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2329 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2330 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2331 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2333 /* Return true if FN is supported for the types in TYPES when the
2334 optimization type is OPT_TYPE. The types are those associated with
2335 the "type0" and "type1" fields of FN's direct_internal_fn_info
2339 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2340 optimization_type opt_type
)
2344 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2345 case IFN_##CODE: break;
2346 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2348 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2350 #include "internal-fn.def"
2358 /* Return true if FN is supported for type TYPE when the optimization
2359 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2360 fields of FN's direct_internal_fn_info structure are the same. */
2363 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2364 optimization_type opt_type
)
2366 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2367 gcc_checking_assert (info
.type0
== info
.type1
);
2368 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2371 /* Return true if IFN_SET_EDOM is supported. */
2374 set_edom_supported_p (void)
2383 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2385 expand_##CODE (internal_fn fn, gcall *stmt) \
2387 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2389 #include "internal-fn.def"
2391 /* Routines to expand each internal function, indexed by function number.
2392 Each routine has the prototype:
2394 expand_<NAME> (gcall *stmt)
2396 where STMT is the statement that performs the call. */
2397 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2398 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2399 #include "internal-fn.def"
2403 /* Expand STMT as though it were a call to internal function FN. */
2406 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2408 internal_fn_expanders
[fn
] (fn
, stmt
);
2411 /* Expand STMT, which is a call to internal function FN. */
2414 expand_internal_call (gcall
*stmt
)
2416 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);