2 Copyright (C) 2011-2020 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 /* The names of each internal function, indexed by function number. */
55 const char *const internal_fn_name_array
[] = {
56 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
57 #include "internal-fn.def"
61 /* The ECF_* flags of each internal function, indexed by function number. */
62 const int internal_fn_flags_array
[] = {
63 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
64 #include "internal-fn.def"
68 /* Return the internal function called NAME, or IFN_LAST if there's
72 lookup_internal_fn (const char *name
)
74 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
75 static name_to_fn_map_type
*name_to_fn_map
;
79 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
80 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
81 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
84 internal_fn
*entry
= name_to_fn_map
->get (name
);
85 return entry
? *entry
: IFN_LAST
;
88 /* Fnspec of each internal function, indexed by function number. */
89 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
94 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
95 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
96 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
97 #include "internal-fn.def"
98 internal_fn_fnspec_array
[IFN_LAST
] = 0;
101 /* Create static initializers for the information returned by
102 direct_internal_fn. */
103 #define not_direct { -2, -2, false }
104 #define mask_load_direct { -1, 2, false }
105 #define load_lanes_direct { -1, -1, false }
106 #define mask_load_lanes_direct { -1, -1, false }
107 #define gather_load_direct { 3, 1, false }
108 #define mask_store_direct { 3, 2, false }
109 #define store_lanes_direct { 0, 0, false }
110 #define mask_store_lanes_direct { 0, 0, false }
111 #define vec_cond_mask_direct { 0, 0, false }
112 #define vec_cond_direct { 0, 0, false }
113 #define vec_condu_direct { 0, 0, false }
114 #define vec_condeq_direct { 0, 0, false }
115 #define scatter_store_direct { 3, 1, false }
116 #define unary_direct { 0, 0, true }
117 #define binary_direct { 0, 0, true }
118 #define ternary_direct { 0, 0, true }
119 #define cond_unary_direct { 1, 1, true }
120 #define cond_binary_direct { 1, 1, true }
121 #define cond_ternary_direct { 1, 1, true }
122 #define while_direct { 0, 2, false }
123 #define fold_extract_direct { 2, 2, false }
124 #define fold_left_direct { 1, 1, false }
125 #define mask_fold_left_direct { 1, 1, false }
126 #define check_ptrs_direct { 0, 0, false }
128 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
129 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
130 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
131 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
132 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
133 #include "internal-fn.def"
137 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
138 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
140 static enum insn_code
141 get_multi_vector_move (tree array_type
, convert_optab optab
)
146 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
147 imode
= TYPE_MODE (array_type
);
148 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
150 return convert_optab_handler (optab
, imode
, vmode
);
153 /* Expand LOAD_LANES call STMT using optab OPTAB. */
156 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
158 class expand_operand ops
[2];
162 lhs
= gimple_call_lhs (stmt
);
163 rhs
= gimple_call_arg (stmt
, 0);
164 type
= TREE_TYPE (lhs
);
166 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
167 mem
= expand_normal (rhs
);
169 gcc_assert (MEM_P (mem
));
170 PUT_MODE (mem
, TYPE_MODE (type
));
172 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
173 create_fixed_operand (&ops
[1], mem
);
174 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
175 if (!rtx_equal_p (target
, ops
[0].value
))
176 emit_move_insn (target
, ops
[0].value
);
179 /* Expand STORE_LANES call STMT using optab OPTAB. */
182 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
184 class expand_operand ops
[2];
188 lhs
= gimple_call_lhs (stmt
);
189 rhs
= gimple_call_arg (stmt
, 0);
190 type
= TREE_TYPE (rhs
);
192 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
193 reg
= expand_normal (rhs
);
195 gcc_assert (MEM_P (target
));
196 PUT_MODE (target
, TYPE_MODE (type
));
198 create_fixed_operand (&ops
[0], target
);
199 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
200 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
204 expand_ANNOTATE (internal_fn
, gcall
*)
209 /* This should get expanded in omp_device_lower pass. */
212 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
217 /* This should get expanded in omp_device_lower pass. */
220 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
225 /* Allocate per-lane storage and begin non-uniform execution region. */
228 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
231 tree lhs
= gimple_call_lhs (stmt
);
233 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
235 target
= gen_reg_rtx (Pmode
);
236 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
237 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
238 class expand_operand ops
[3];
239 create_output_operand (&ops
[0], target
, Pmode
);
240 create_input_operand (&ops
[1], size
, Pmode
);
241 create_input_operand (&ops
[2], align
, Pmode
);
242 gcc_assert (targetm
.have_omp_simt_enter ());
243 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
246 /* Deallocate per-lane storage and leave non-uniform execution region. */
249 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
251 gcc_checking_assert (!gimple_call_lhs (stmt
));
252 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
253 class expand_operand ops
[1];
254 create_input_operand (&ops
[0], arg
, Pmode
);
255 gcc_assert (targetm
.have_omp_simt_exit ());
256 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
259 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
260 without SIMT execution this should be expanded in omp_device_lower pass. */
263 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
265 tree lhs
= gimple_call_lhs (stmt
);
269 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
270 gcc_assert (targetm
.have_omp_simt_lane ());
271 emit_insn (targetm
.gen_omp_simt_lane (target
));
274 /* This should get expanded in omp_device_lower pass. */
277 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
282 /* Lane index of the first SIMT lane that supplies a non-zero argument.
283 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
284 lane that executed the last iteration for handling OpenMP lastprivate. */
287 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
289 tree lhs
= gimple_call_lhs (stmt
);
293 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
294 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
295 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
296 class expand_operand ops
[2];
297 create_output_operand (&ops
[0], target
, mode
);
298 create_input_operand (&ops
[1], cond
, mode
);
299 gcc_assert (targetm
.have_omp_simt_last_lane ());
300 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
303 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
306 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
308 tree lhs
= gimple_call_lhs (stmt
);
312 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
313 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
314 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
315 class expand_operand ops
[2];
316 create_output_operand (&ops
[0], target
, mode
);
317 create_input_operand (&ops
[1], ctr
, mode
);
318 gcc_assert (targetm
.have_omp_simt_ordered ());
319 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
322 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
323 any lane supplies a non-zero argument. */
326 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
328 tree lhs
= gimple_call_lhs (stmt
);
332 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
333 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
334 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
335 class expand_operand ops
[2];
336 create_output_operand (&ops
[0], target
, mode
);
337 create_input_operand (&ops
[1], cond
, mode
);
338 gcc_assert (targetm
.have_omp_simt_vote_any ());
339 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
342 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
343 is destination lane index XOR given offset. */
346 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
348 tree lhs
= gimple_call_lhs (stmt
);
352 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
353 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
354 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
355 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
356 class expand_operand ops
[3];
357 create_output_operand (&ops
[0], target
, mode
);
358 create_input_operand (&ops
[1], src
, mode
);
359 create_input_operand (&ops
[2], idx
, SImode
);
360 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
361 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
364 /* Exchange between SIMT lanes according to given source lane index. */
367 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
369 tree lhs
= gimple_call_lhs (stmt
);
373 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
374 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
375 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
376 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
377 class expand_operand ops
[3];
378 create_output_operand (&ops
[0], target
, mode
);
379 create_input_operand (&ops
[1], src
, mode
);
380 create_input_operand (&ops
[2], idx
, SImode
);
381 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
382 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
385 /* This should get expanded in adjust_simduid_builtins. */
388 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
393 /* This should get expanded in adjust_simduid_builtins. */
396 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
401 /* This should get expanded in adjust_simduid_builtins. */
404 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
409 /* This should get expanded in adjust_simduid_builtins. */
412 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
417 /* This should get expanded in adjust_simduid_builtins. */
420 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
425 /* This should get expanded in the sanopt pass. */
428 expand_UBSAN_NULL (internal_fn
, gcall
*)
433 /* This should get expanded in the sanopt pass. */
436 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
441 /* This should get expanded in the sanopt pass. */
444 expand_UBSAN_VPTR (internal_fn
, gcall
*)
449 /* This should get expanded in the sanopt pass. */
452 expand_UBSAN_PTR (internal_fn
, gcall
*)
457 /* This should get expanded in the sanopt pass. */
460 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
465 /* This should get expanded in the sanopt pass. */
468 expand_ASAN_CHECK (internal_fn
, gcall
*)
473 /* This should get expanded in the sanopt pass. */
476 expand_ASAN_MARK (internal_fn
, gcall
*)
481 /* This should get expanded in the sanopt pass. */
484 expand_ASAN_POISON (internal_fn
, gcall
*)
489 /* This should get expanded in the sanopt pass. */
492 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
497 /* This should get expanded in the tsan pass. */
500 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
505 /* This should get expanded in the lower pass. */
508 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
510 error_at (gimple_location (call
),
511 "invalid use of attribute %<fallthrough%>");
514 /* Return minimum precision needed to represent all values
515 of ARG in SIGNed integral type. */
518 get_min_precision (tree arg
, signop sign
)
520 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
522 signop orig_sign
= sign
;
523 if (TREE_CODE (arg
) == INTEGER_CST
)
526 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
528 widest_int w
= wi::to_widest (arg
);
529 w
= wi::ext (w
, prec
, sign
);
530 p
= wi::min_precision (w
, sign
);
533 p
= wi::min_precision (wi::to_wide (arg
), sign
);
534 return MIN (p
, prec
);
536 while (CONVERT_EXPR_P (arg
)
537 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
538 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
540 arg
= TREE_OPERAND (arg
, 0);
541 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
543 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
545 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
546 return prec
+ (orig_sign
!= sign
);
547 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
550 return prec
+ (orig_sign
!= sign
);
552 if (TREE_CODE (arg
) != SSA_NAME
)
553 return prec
+ (orig_sign
!= sign
);
554 wide_int arg_min
, arg_max
;
555 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
557 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
558 if (is_gimple_assign (g
)
559 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
561 tree t
= gimple_assign_rhs1 (g
);
562 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
563 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
566 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
568 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
570 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
571 return prec
+ (orig_sign
!= sign
);
572 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
575 return prec
+ (orig_sign
!= sign
);
579 return prec
+ (orig_sign
!= sign
);
581 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
583 int p1
= wi::min_precision (arg_min
, sign
);
584 int p2
= wi::min_precision (arg_max
, sign
);
586 prec
= MIN (prec
, p1
);
588 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
590 int p
= wi::min_precision (arg_max
, UNSIGNED
);
591 prec
= MIN (prec
, p
);
593 return prec
+ (orig_sign
!= sign
);
596 /* Helper for expand_*_overflow. Set the __imag__ part to true
597 (1 except for signed:1 type, in which case store -1). */
600 expand_arith_set_overflow (tree lhs
, rtx target
)
602 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
603 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
604 write_complex_part (target
, constm1_rtx
, true);
606 write_complex_part (target
, const1_rtx
, true);
609 /* Helper for expand_*_overflow. Store RES into the __real__ part
610 of TARGET. If RES has larger MODE than __real__ part of TARGET,
611 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
612 if LHS has smaller precision than its mode. */
615 expand_arith_overflow_result_store (tree lhs
, rtx target
,
616 scalar_int_mode mode
, rtx res
)
618 scalar_int_mode tgtmode
619 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
623 rtx_code_label
*done_label
= gen_label_rtx ();
624 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
625 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
626 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
627 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
628 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
629 profile_probability::very_likely ());
630 expand_arith_set_overflow (lhs
, target
);
631 emit_label (done_label
);
633 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
634 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
637 rtx_code_label
*done_label
= gen_label_rtx ();
638 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
643 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
645 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
646 true, OPTAB_LIB_WIDEN
);
650 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
652 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
655 do_compare_rtx_and_jump (res
, lres
,
656 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
657 profile_probability::very_likely ());
658 expand_arith_set_overflow (lhs
, target
);
659 emit_label (done_label
);
661 write_complex_part (target
, lres
, false);
664 /* Helper for expand_*_overflow. Store RES into TARGET. */
667 expand_ubsan_result_store (rtx target
, rtx res
)
669 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
670 /* If this is a scalar in a register that is stored in a wider mode
671 than the declared mode, compute the result into its declared mode
672 and then convert to the wider mode. Our value is the computed
674 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
676 emit_move_insn (target
, res
);
679 /* Add sub/add overflow checking to the statement STMT.
680 CODE says whether the operation is +, or -. */
683 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
684 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
685 bool uns1_p
, bool is_ubsan
, tree
*datap
)
687 rtx res
, target
= NULL_RTX
;
689 rtx_code_label
*done_label
= gen_label_rtx ();
690 rtx_code_label
*do_error
= gen_label_rtx ();
691 do_pending_stack_adjust ();
692 rtx op0
= expand_normal (arg0
);
693 rtx op1
= expand_normal (arg1
);
694 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
695 int prec
= GET_MODE_PRECISION (mode
);
696 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
700 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
704 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
706 write_complex_part (target
, const0_rtx
, true);
709 /* We assume both operands and result have the same precision
710 here (GET_MODE_BITSIZE (mode)), S stands for signed type
711 with that precision, U for unsigned type with that precision,
712 sgn for unsigned most significant bit in that precision.
713 s1 is signed first operand, u1 is unsigned first operand,
714 s2 is signed second operand, u2 is unsigned second operand,
715 sr is signed result, ur is unsigned result and the following
716 rules say how to compute result (which is always result of
717 the operands as if both were unsigned, cast to the right
718 signedness) and how to compute whether operation overflowed.
721 res = (S) ((U) s1 + (U) s2)
722 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
724 res = (S) ((U) s1 - (U) s2)
725 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
728 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
731 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
733 res = (S) ((U) s1 + u2)
734 ovf = ((U) res ^ sgn) < u2
739 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
741 res = (S) ((U) s1 - u2)
742 ovf = u2 > ((U) s1 ^ sgn)
745 ovf = s1 < 0 || u2 > (U) s1
748 ovf = u1 >= ((U) s2 ^ sgn)
753 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
755 res = (U) s1 + (U) s2
756 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
759 ovf = (U) res < u2 || res < 0
762 ovf = u1 >= u2 ? res < 0 : res >= 0
764 res = (U) s1 - (U) s2
765 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
767 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
769 /* PLUS_EXPR is commutative, if operand signedness differs,
770 canonicalize to the first operand being signed and second
771 unsigned to simplify following code. */
772 std::swap (op0
, op1
);
773 std::swap (arg0
, arg1
);
779 if (uns0_p
&& uns1_p
&& unsr_p
)
781 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
782 : usubv4_optab
, mode
);
783 if (icode
!= CODE_FOR_nothing
)
785 class expand_operand ops
[4];
786 rtx_insn
*last
= get_last_insn ();
788 res
= gen_reg_rtx (mode
);
789 create_output_operand (&ops
[0], res
, mode
);
790 create_input_operand (&ops
[1], op0
, mode
);
791 create_input_operand (&ops
[2], op1
, mode
);
792 create_fixed_operand (&ops
[3], do_error
);
793 if (maybe_expand_insn (icode
, 4, ops
))
795 last
= get_last_insn ();
796 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
798 && any_condjump_p (last
)
799 && !find_reg_note (last
, REG_BR_PROB
, 0))
800 add_reg_br_prob_note (last
,
801 profile_probability::very_unlikely ());
802 emit_jump (done_label
);
806 delete_insns_since (last
);
809 /* Compute the operation. On RTL level, the addition is always
811 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
812 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
814 /* For PLUS_EXPR, the operation is commutative, so we can pick
815 operand to compare against. For prec <= BITS_PER_WORD, I think
816 preferring REG operand is better over CONST_INT, because
817 the CONST_INT might enlarge the instruction or CSE would need
818 to figure out we'd already loaded it into a register before.
819 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
820 as then the multi-word comparison can be perhaps simplified. */
821 if (code
== PLUS_EXPR
822 && (prec
<= BITS_PER_WORD
823 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
824 : CONST_SCALAR_INT_P (op1
)))
826 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
827 true, mode
, NULL_RTX
, NULL
, done_label
,
828 profile_probability::very_likely ());
833 if (!uns0_p
&& uns1_p
&& !unsr_p
)
835 /* Compute the operation. On RTL level, the addition is always
837 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
838 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
839 rtx tem
= expand_binop (mode
, add_optab
,
840 code
== PLUS_EXPR
? res
: op0
, sgn
,
841 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
842 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
843 done_label
, profile_probability::very_likely ());
848 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
850 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
852 /* As we've changed op1, we have to avoid using the value range
853 for the original argument. */
854 arg1
= error_mark_node
;
860 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
862 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
864 /* As we've changed op0, we have to avoid using the value range
865 for the original argument. */
866 arg0
= error_mark_node
;
872 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
874 /* Compute the operation. On RTL level, the addition is always
876 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
878 int pos_neg
= get_range_pos_neg (arg0
);
880 /* If ARG0 is known to be always negative, this is always overflow. */
881 emit_jump (do_error
);
882 else if (pos_neg
== 3)
883 /* If ARG0 is not known to be always positive, check at runtime. */
884 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
885 NULL
, do_error
, profile_probability::very_unlikely ());
886 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
887 done_label
, profile_probability::very_likely ());
892 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
894 /* Compute the operation. On RTL level, the addition is always
896 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
898 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
900 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
901 done_label
, profile_probability::very_likely ());
906 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
908 /* Compute the operation. On RTL level, the addition is always
910 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
912 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
913 NULL
, do_error
, profile_probability::very_unlikely ());
915 /* The operation is commutative, so we can pick operand to compare
916 against. For prec <= BITS_PER_WORD, I think preferring REG operand
917 is better over CONST_INT, because the CONST_INT might enlarge the
918 instruction or CSE would need to figure out we'd already loaded it
919 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
920 might be more beneficial, as then the multi-word comparison can be
921 perhaps simplified. */
922 if (prec
<= BITS_PER_WORD
923 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
924 : CONST_SCALAR_INT_P (op0
))
926 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
927 done_label
, profile_probability::very_likely ());
932 if (!uns0_p
&& !uns1_p
&& unsr_p
)
934 /* Compute the operation. On RTL level, the addition is always
936 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
937 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
938 int pos_neg
= get_range_pos_neg (arg1
);
939 if (code
== PLUS_EXPR
)
941 int pos_neg0
= get_range_pos_neg (arg0
);
942 if (pos_neg0
!= 3 && pos_neg
== 3)
944 std::swap (op0
, op1
);
951 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
952 ? and_optab
: ior_optab
,
953 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
954 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
955 NULL
, done_label
, profile_probability::very_likely ());
959 rtx_code_label
*do_ior_label
= gen_label_rtx ();
960 do_compare_rtx_and_jump (op1
, const0_rtx
,
961 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
962 NULL_RTX
, NULL
, do_ior_label
,
963 profile_probability::even ());
964 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
966 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
967 NULL
, done_label
, profile_probability::very_likely ());
968 emit_jump (do_error
);
969 emit_label (do_ior_label
);
970 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
972 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
973 NULL
, done_label
, profile_probability::very_likely ());
979 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
981 /* Compute the operation. On RTL level, the addition is always
983 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
985 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
986 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
987 op0_geu_op1
, profile_probability::even ());
988 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
989 NULL
, done_label
, profile_probability::very_likely ());
990 emit_jump (do_error
);
991 emit_label (op0_geu_op1
);
992 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
993 NULL
, done_label
, profile_probability::very_likely ());
997 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1002 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1003 : subv4_optab
, mode
);
1004 if (icode
!= CODE_FOR_nothing
)
1006 class expand_operand ops
[4];
1007 rtx_insn
*last
= get_last_insn ();
1009 res
= gen_reg_rtx (mode
);
1010 create_output_operand (&ops
[0], res
, mode
);
1011 create_input_operand (&ops
[1], op0
, mode
);
1012 create_input_operand (&ops
[2], op1
, mode
);
1013 create_fixed_operand (&ops
[3], do_error
);
1014 if (maybe_expand_insn (icode
, 4, ops
))
1016 last
= get_last_insn ();
1017 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1019 && any_condjump_p (last
)
1020 && !find_reg_note (last
, REG_BR_PROB
, 0))
1021 add_reg_br_prob_note (last
,
1022 profile_probability::very_unlikely ());
1023 emit_jump (done_label
);
1024 goto do_error_label
;
1027 delete_insns_since (last
);
1030 /* Compute the operation. On RTL level, the addition is always
1032 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1033 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1035 /* If we can prove that one of the arguments (for MINUS_EXPR only
1036 the second operand, as subtraction is not commutative) is always
1037 non-negative or always negative, we can do just one comparison
1038 and conditional jump. */
1039 int pos_neg
= get_range_pos_neg (arg1
);
1040 if (code
== PLUS_EXPR
)
1042 int pos_neg0
= get_range_pos_neg (arg0
);
1043 if (pos_neg0
!= 3 && pos_neg
== 3)
1045 std::swap (op0
, op1
);
1050 /* Addition overflows if and only if the two operands have the same sign,
1051 and the result has the opposite sign. Subtraction overflows if and
1052 only if the two operands have opposite sign, and the subtrahend has
1053 the same sign as the result. Here 0 is counted as positive. */
1056 /* Compute op0 ^ op1 (operands have opposite sign). */
1057 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1060 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1061 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1065 if (code
== PLUS_EXPR
)
1067 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1068 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1069 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1074 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1075 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1076 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1080 /* No overflow if the result has bit sign cleared. */
1081 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1082 NULL
, done_label
, profile_probability::very_likely ());
1085 /* Compare the result of the operation with the first operand.
1086 No overflow for addition if second operand is positive and result
1087 is larger or second operand is negative and result is smaller.
1088 Likewise for subtraction with sign of second operand flipped. */
1090 do_compare_rtx_and_jump (res
, op0
,
1091 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1092 false, mode
, NULL_RTX
, NULL
, done_label
,
1093 profile_probability::very_likely ());
1097 emit_label (do_error
);
1100 /* Expand the ubsan builtin call. */
1102 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1106 do_pending_stack_adjust ();
1109 expand_arith_set_overflow (lhs
, target
);
1112 emit_label (done_label
);
1117 expand_ubsan_result_store (target
, res
);
1121 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1124 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1129 /* Add negate overflow checking to the statement STMT. */
1132 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1137 rtx_code_label
*done_label
, *do_error
;
1138 rtx target
= NULL_RTX
;
1140 done_label
= gen_label_rtx ();
1141 do_error
= gen_label_rtx ();
1143 do_pending_stack_adjust ();
1144 op1
= expand_normal (arg1
);
1146 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1149 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1151 write_complex_part (target
, const0_rtx
, true);
1154 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1155 if (icode
!= CODE_FOR_nothing
)
1157 class expand_operand ops
[3];
1158 rtx_insn
*last
= get_last_insn ();
1160 res
= gen_reg_rtx (mode
);
1161 create_output_operand (&ops
[0], res
, mode
);
1162 create_input_operand (&ops
[1], op1
, mode
);
1163 create_fixed_operand (&ops
[2], do_error
);
1164 if (maybe_expand_insn (icode
, 3, ops
))
1166 last
= get_last_insn ();
1167 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1169 && any_condjump_p (last
)
1170 && !find_reg_note (last
, REG_BR_PROB
, 0))
1171 add_reg_br_prob_note (last
,
1172 profile_probability::very_unlikely ());
1173 emit_jump (done_label
);
1177 delete_insns_since (last
);
1178 icode
= CODE_FOR_nothing
;
1182 if (icode
== CODE_FOR_nothing
)
1184 /* Compute the operation. On RTL level, the addition is always
1186 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1188 /* Compare the operand with the most negative value. */
1189 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1190 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1191 done_label
, profile_probability::very_likely ());
1194 emit_label (do_error
);
1197 /* Expand the ubsan builtin call. */
1199 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1200 arg1
, NULL_TREE
, datap
);
1203 do_pending_stack_adjust ();
1206 expand_arith_set_overflow (lhs
, target
);
1209 emit_label (done_label
);
1214 expand_ubsan_result_store (target
, res
);
1216 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1220 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1221 mode MODE can be expanded without using a libcall. */
1224 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1225 rtx op0
, rtx op1
, bool uns
)
1227 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1228 != CODE_FOR_nothing
)
1231 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1232 != CODE_FOR_nothing
)
1235 rtx_insn
*last
= get_last_insn ();
1236 if (CONSTANT_P (op0
))
1237 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1239 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1240 if (CONSTANT_P (op1
))
1241 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1243 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1244 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1245 delete_insns_since (last
);
1246 return ret
!= NULL_RTX
;
1249 /* Add mul overflow checking to the statement STMT. */
1252 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1253 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1258 rtx_code_label
*done_label
, *do_error
;
1259 rtx target
= NULL_RTX
;
1261 enum insn_code icode
;
1263 done_label
= gen_label_rtx ();
1264 do_error
= gen_label_rtx ();
1266 do_pending_stack_adjust ();
1267 op0
= expand_normal (arg0
);
1268 op1
= expand_normal (arg1
);
1270 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1274 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1276 write_complex_part (target
, const0_rtx
, true);
1280 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1282 /* We assume both operands and result have the same precision
1283 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1284 with that precision, U for unsigned type with that precision,
1285 sgn for unsigned most significant bit in that precision.
1286 s1 is signed first operand, u1 is unsigned first operand,
1287 s2 is signed second operand, u2 is unsigned second operand,
1288 sr is signed result, ur is unsigned result and the following
1289 rules say how to compute result (which is always result of
1290 the operands as if both were unsigned, cast to the right
1291 signedness) and how to compute whether operation overflowed.
1292 main_ovf (false) stands for jump on signed multiplication
1293 overflow or the main algorithm with uns == false.
1294 main_ovf (true) stands for jump on unsigned multiplication
1295 overflow or the main algorithm with uns == true.
1298 res = (S) ((U) s1 * (U) s2)
1299 ovf = main_ovf (false)
1302 ovf = main_ovf (true)
1305 ovf = (s1 < 0 && u2) || main_ovf (true)
1308 ovf = res < 0 || main_ovf (true)
1310 res = (S) ((U) s1 * u2)
1311 ovf = (S) u2 >= 0 ? main_ovf (false)
1312 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1314 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1315 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1317 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1319 if (uns0_p
&& !uns1_p
)
1321 /* Multiplication is commutative, if operand signedness differs,
1322 canonicalize to the first operand being signed and second
1323 unsigned to simplify following code. */
1324 std::swap (op0
, op1
);
1325 std::swap (arg0
, arg1
);
1330 int pos_neg0
= get_range_pos_neg (arg0
);
1331 int pos_neg1
= get_range_pos_neg (arg1
);
1334 if (!uns0_p
&& uns1_p
&& unsr_p
)
1339 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1342 /* If s1 is negative, avoid the main code, just multiply and
1343 signal overflow if op1 is not 0. */
1344 struct separate_ops ops
;
1345 ops
.code
= MULT_EXPR
;
1346 ops
.type
= TREE_TYPE (arg1
);
1347 ops
.op0
= make_tree (ops
.type
, op0
);
1348 ops
.op1
= make_tree (ops
.type
, op1
);
1349 ops
.op2
= NULL_TREE
;
1351 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1352 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1353 NULL
, done_label
, profile_probability::very_likely ());
1354 goto do_error_label
;
1356 rtx_code_label
*do_main_label
;
1357 do_main_label
= gen_label_rtx ();
1358 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1359 NULL
, do_main_label
, profile_probability::very_likely ());
1360 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1361 NULL
, do_main_label
, profile_probability::very_likely ());
1362 expand_arith_set_overflow (lhs
, target
);
1363 emit_label (do_main_label
);
1371 if (uns0_p
&& uns1_p
&& !unsr_p
)
1374 /* Rest of handling of this case after res is computed. */
1379 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1386 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1387 avoid the main code, just multiply and signal overflow
1388 unless 0 * u2 or -1 * ((U) Smin). */
1389 struct separate_ops ops
;
1390 ops
.code
= MULT_EXPR
;
1391 ops
.type
= TREE_TYPE (arg1
);
1392 ops
.op0
= make_tree (ops
.type
, op0
);
1393 ops
.op1
= make_tree (ops
.type
, op1
);
1394 ops
.op2
= NULL_TREE
;
1396 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1397 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1398 NULL
, done_label
, profile_probability::very_likely ());
1399 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1400 NULL
, do_error
, profile_probability::very_unlikely ());
1402 prec
= GET_MODE_PRECISION (mode
);
1404 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1405 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1406 NULL
, done_label
, profile_probability::very_likely ());
1407 goto do_error_label
;
1409 /* Rest of handling of this case after res is computed. */
1417 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1420 switch (pos_neg0
| pos_neg1
)
1422 case 1: /* Both operands known to be non-negative. */
1424 case 2: /* Both operands known to be negative. */
1425 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1426 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1427 /* Avoid looking at arg0/arg1 ranges, as we've changed
1429 arg0
= error_mark_node
;
1430 arg1
= error_mark_node
;
1433 if ((pos_neg0
^ pos_neg1
) == 3)
1435 /* If one operand is known to be negative and the other
1436 non-negative, this overflows always, unless the non-negative
1437 one is 0. Just do normal multiply and set overflow
1438 unless one of the operands is 0. */
1439 struct separate_ops ops
;
1440 ops
.code
= MULT_EXPR
;
1442 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1444 ops
.op0
= make_tree (ops
.type
, op0
);
1445 ops
.op1
= make_tree (ops
.type
, op1
);
1446 ops
.op2
= NULL_TREE
;
1448 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1449 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1450 true, mode
, NULL_RTX
, NULL
, done_label
,
1451 profile_probability::very_likely ());
1452 goto do_error_label
;
1454 /* The general case, do all the needed comparisons at runtime. */
1455 rtx_code_label
*do_main_label
, *after_negate_label
;
1457 rop0
= gen_reg_rtx (mode
);
1458 rop1
= gen_reg_rtx (mode
);
1459 emit_move_insn (rop0
, op0
);
1460 emit_move_insn (rop1
, op1
);
1463 do_main_label
= gen_label_rtx ();
1464 after_negate_label
= gen_label_rtx ();
1465 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1467 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1468 NULL
, after_negate_label
, profile_probability::very_likely ());
1469 /* Both arguments negative here, negate them and continue with
1470 normal unsigned overflow checking multiplication. */
1471 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1473 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1475 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1477 arg0
= error_mark_node
;
1478 arg1
= error_mark_node
;
1479 emit_jump (do_main_label
);
1480 emit_label (after_negate_label
);
1481 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1483 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1484 NULL
, do_main_label
,
1485 profile_probability::very_likely ());
1486 /* One argument is negative here, the other positive. This
1487 overflows always, unless one of the arguments is 0. But
1488 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1489 is, thus we can keep do_main code oring in overflow as is. */
1491 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1492 NULL
, do_main_label
,
1493 profile_probability::very_unlikely ());
1495 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1496 NULL
, do_main_label
,
1497 profile_probability::very_unlikely ());
1498 expand_arith_set_overflow (lhs
, target
);
1499 emit_label (do_main_label
);
1507 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1508 sign
= uns
? UNSIGNED
: SIGNED
;
1509 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1511 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1512 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1514 /* Optimize unsigned multiplication by power of 2 constant
1515 using 2 shifts, one for result, one to extract the shifted
1516 out bits to see if they are all zero.
1517 Don't do this if optimizing for size and we have umulv4_optab,
1518 in that case assume multiplication will be shorter.
1519 This is heuristics based on the single target that provides
1520 umulv4 right now (i?86/x86_64), if further targets add it, this
1521 might need to be revisited.
1522 Cases where both operands are constant should be folded already
1523 during GIMPLE, and cases where one operand is constant but not
1524 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1525 below can be done without multiplication, just by shifts and adds,
1526 or we'd need to divide the result (and hope it actually doesn't
1527 really divide nor multiply) and compare the result of the division
1528 with the original operand. */
1533 if (integer_pow2p (arg0
))
1535 std::swap (opn0
, opn1
);
1536 std::swap (argn0
, argn1
);
1538 int cnt
= tree_log2 (argn1
);
1539 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1541 rtx upper
= const0_rtx
;
1542 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1544 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1545 GET_MODE_PRECISION (mode
) - cnt
,
1547 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1548 NULL_RTX
, NULL
, done_label
,
1549 profile_probability::very_likely ());
1550 goto do_error_label
;
1553 if (icode
!= CODE_FOR_nothing
)
1555 class expand_operand ops
[4];
1556 rtx_insn
*last
= get_last_insn ();
1558 res
= gen_reg_rtx (mode
);
1559 create_output_operand (&ops
[0], res
, mode
);
1560 create_input_operand (&ops
[1], op0
, mode
);
1561 create_input_operand (&ops
[2], op1
, mode
);
1562 create_fixed_operand (&ops
[3], do_error
);
1563 if (maybe_expand_insn (icode
, 4, ops
))
1565 last
= get_last_insn ();
1566 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1568 && any_condjump_p (last
)
1569 && !find_reg_note (last
, REG_BR_PROB
, 0))
1570 add_reg_br_prob_note (last
,
1571 profile_probability::very_unlikely ());
1572 emit_jump (done_label
);
1576 delete_insns_since (last
);
1577 icode
= CODE_FOR_nothing
;
1581 if (icode
== CODE_FOR_nothing
)
1583 struct separate_ops ops
;
1584 int prec
= GET_MODE_PRECISION (mode
);
1585 scalar_int_mode hmode
, wmode
;
1586 ops
.op0
= make_tree (type
, op0
);
1587 ops
.op1
= make_tree (type
, op1
);
1588 ops
.op2
= NULL_TREE
;
1591 /* Optimize unsigned overflow check where we don't use the
1592 multiplication result, just whether overflow happened.
1593 If we can do MULT_HIGHPART_EXPR, that followed by
1594 comparison of the result against zero is cheapest.
1595 We'll still compute res, but it should be DCEd later. */
1601 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1602 && can_mult_highpart_p (mode
, uns
) == 1
1603 && single_imm_use (lhs
, &use
, &use_stmt
)
1604 && is_gimple_assign (use_stmt
)
1605 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1608 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1609 && targetm
.scalar_mode_supported_p (wmode
)
1610 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1613 ops
.code
= WIDEN_MULT_EXPR
;
1615 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1617 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1618 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1620 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1621 res
= convert_modes (mode
, wmode
, res
, uns
);
1623 /* For the unsigned multiplication, there was overflow if
1624 HIPART is non-zero. */
1625 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1626 NULL_RTX
, NULL
, done_label
,
1627 profile_probability::very_likely ());
1630 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1632 /* RES is low half of the double width result, HIPART
1633 the high half. There was overflow if
1634 HIPART is different from RES < 0 ? -1 : 0. */
1635 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1636 NULL_RTX
, NULL
, done_label
,
1637 profile_probability::very_likely ());
1640 else if (can_mult_highpart_p (mode
, uns
) == 1)
1643 ops
.code
= MULT_HIGHPART_EXPR
;
1646 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1648 ops
.code
= MULT_EXPR
;
1649 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1651 /* For the unsigned multiplication, there was overflow if
1652 HIPART is non-zero. */
1653 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1654 NULL_RTX
, NULL
, done_label
,
1655 profile_probability::very_likely ());
1658 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1660 /* RES is low half of the double width result, HIPART
1661 the high half. There was overflow if
1662 HIPART is different from RES < 0 ? -1 : 0. */
1663 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1664 NULL_RTX
, NULL
, done_label
,
1665 profile_probability::very_likely ());
1669 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1670 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1672 rtx_code_label
*large_op0
= gen_label_rtx ();
1673 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1674 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1675 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1676 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1677 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1678 rtx_code_label
*do_overflow
= gen_label_rtx ();
1679 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1681 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1682 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1684 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1685 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1686 rtx signbit0
= const0_rtx
;
1688 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1690 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1692 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1693 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1694 rtx signbit1
= const0_rtx
;
1696 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1699 res
= gen_reg_rtx (mode
);
1701 /* True if op0 resp. op1 are known to be in the range of
1703 bool op0_small_p
= false;
1704 bool op1_small_p
= false;
1705 /* True if op0 resp. op1 are known to have all zeros or all ones
1706 in the upper half of bits, but are not known to be
1708 bool op0_medium_p
= false;
1709 bool op1_medium_p
= false;
1710 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1711 nonnegative, 1 if unknown. */
1717 else if (pos_neg0
== 2)
1721 else if (pos_neg1
== 2)
1724 unsigned int mprec0
= prec
;
1725 if (arg0
!= error_mark_node
)
1726 mprec0
= get_min_precision (arg0
, sign
);
1727 if (mprec0
<= hprec
)
1729 else if (!uns
&& mprec0
<= hprec
+ 1)
1730 op0_medium_p
= true;
1731 unsigned int mprec1
= prec
;
1732 if (arg1
!= error_mark_node
)
1733 mprec1
= get_min_precision (arg1
, sign
);
1734 if (mprec1
<= hprec
)
1736 else if (!uns
&& mprec1
<= hprec
+ 1)
1737 op1_medium_p
= true;
1739 int smaller_sign
= 1;
1740 int larger_sign
= 1;
1743 smaller_sign
= op0_sign
;
1744 larger_sign
= op1_sign
;
1746 else if (op1_small_p
)
1748 smaller_sign
= op1_sign
;
1749 larger_sign
= op0_sign
;
1751 else if (op0_sign
== op1_sign
)
1753 smaller_sign
= op0_sign
;
1754 larger_sign
= op0_sign
;
1758 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1759 NULL_RTX
, NULL
, large_op0
,
1760 profile_probability::unlikely ());
1763 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1764 NULL_RTX
, NULL
, small_op0_large_op1
,
1765 profile_probability::unlikely ());
1767 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1768 hmode to mode, the multiplication will never overflow. We can
1769 do just one hmode x hmode => mode widening multiplication. */
1770 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1771 ops
.op0
= make_tree (halfstype
, lopart0
);
1772 ops
.op1
= make_tree (halfstype
, lopart1
);
1773 ops
.code
= WIDEN_MULT_EXPR
;
1776 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1777 emit_move_insn (res
, thisres
);
1778 emit_jump (done_label
);
1780 emit_label (small_op0_large_op1
);
1782 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1783 but op1 is not, just swap the arguments and handle it as op1
1784 sign/zero extended, op0 not. */
1785 rtx larger
= gen_reg_rtx (mode
);
1786 rtx hipart
= gen_reg_rtx (hmode
);
1787 rtx lopart
= gen_reg_rtx (hmode
);
1788 emit_move_insn (larger
, op1
);
1789 emit_move_insn (hipart
, hipart1
);
1790 emit_move_insn (lopart
, lopart0
);
1791 emit_jump (one_small_one_large
);
1793 emit_label (large_op0
);
1796 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1797 NULL_RTX
, NULL
, both_ops_large
,
1798 profile_probability::unlikely ());
1800 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1801 but op0 is not, prepare larger, hipart and lopart pseudos and
1802 handle it together with small_op0_large_op1. */
1803 emit_move_insn (larger
, op0
);
1804 emit_move_insn (hipart
, hipart0
);
1805 emit_move_insn (lopart
, lopart1
);
1807 emit_label (one_small_one_large
);
1809 /* lopart is the low part of the operand that is sign extended
1810 to mode, larger is the other operand, hipart is the
1811 high part of larger and lopart0 and lopart1 are the low parts
1813 We perform lopart0 * lopart1 and lopart * hipart widening
1815 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1816 ops
.op0
= make_tree (halfutype
, lopart0
);
1817 ops
.op1
= make_tree (halfutype
, lopart1
);
1819 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1821 ops
.op0
= make_tree (halfutype
, lopart
);
1822 ops
.op1
= make_tree (halfutype
, hipart
);
1823 rtx loxhi
= gen_reg_rtx (mode
);
1824 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1825 emit_move_insn (loxhi
, tem
);
1829 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1830 if (larger_sign
== 0)
1831 emit_jump (after_hipart_neg
);
1832 else if (larger_sign
!= -1)
1833 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1834 NULL_RTX
, NULL
, after_hipart_neg
,
1835 profile_probability::even ());
1837 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1838 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1839 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1841 emit_move_insn (loxhi
, tem
);
1843 emit_label (after_hipart_neg
);
1845 /* if (lopart < 0) loxhi -= larger; */
1846 if (smaller_sign
== 0)
1847 emit_jump (after_lopart_neg
);
1848 else if (smaller_sign
!= -1)
1849 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1850 NULL_RTX
, NULL
, after_lopart_neg
,
1851 profile_probability::even ());
1853 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1855 emit_move_insn (loxhi
, tem
);
1857 emit_label (after_lopart_neg
);
1860 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1861 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1862 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1864 emit_move_insn (loxhi
, tem
);
1866 /* if (loxhi >> (bitsize / 2)
1867 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1868 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1869 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1871 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1872 rtx signbitloxhi
= const0_rtx
;
1874 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1875 convert_modes (hmode
, mode
,
1877 hprec
- 1, NULL_RTX
, 0);
1879 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1880 NULL_RTX
, NULL
, do_overflow
,
1881 profile_probability::very_unlikely ());
1883 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1884 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1886 tem
= convert_modes (mode
, hmode
,
1887 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1889 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1892 emit_move_insn (res
, tem
);
1893 emit_jump (done_label
);
1895 emit_label (both_ops_large
);
1897 /* If both operands are large (not sign (!uns) or zero (uns)
1898 extended from hmode), then perform the full multiplication
1899 which will be the result of the operation.
1900 The only cases which don't overflow are for signed multiplication
1901 some cases where both hipart0 and highpart1 are 0 or -1.
1902 For unsigned multiplication when high parts are both non-zero
1903 this overflows always. */
1904 ops
.code
= MULT_EXPR
;
1905 ops
.op0
= make_tree (type
, op0
);
1906 ops
.op1
= make_tree (type
, op1
);
1907 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1908 emit_move_insn (res
, tem
);
1914 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1915 NULL_RTX
, 1, OPTAB_WIDEN
);
1916 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1917 NULL_RTX
, NULL
, do_error
,
1918 profile_probability::very_unlikely ());
1923 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1924 NULL_RTX
, 1, OPTAB_WIDEN
);
1925 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1926 NULL_RTX
, NULL
, do_error
,
1927 profile_probability::very_unlikely ());
1930 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1931 the same, overflow happened if res is non-positive, if they
1932 are different, overflow happened if res is positive. */
1933 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1934 emit_jump (hipart_different
);
1935 else if (op0_sign
== 1 || op1_sign
== 1)
1936 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1937 NULL_RTX
, NULL
, hipart_different
,
1938 profile_probability::even ());
1940 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1941 NULL_RTX
, NULL
, do_error
,
1942 profile_probability::very_unlikely ());
1943 emit_jump (done_label
);
1945 emit_label (hipart_different
);
1947 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1948 NULL_RTX
, NULL
, do_error
,
1949 profile_probability::very_unlikely ());
1950 emit_jump (done_label
);
1953 emit_label (do_overflow
);
1955 /* Overflow, do full multiplication and fallthru into do_error. */
1956 ops
.op0
= make_tree (type
, op0
);
1957 ops
.op1
= make_tree (type
, op1
);
1958 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1959 emit_move_insn (res
, tem
);
1961 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1962 && targetm
.scalar_mode_supported_p (wmode
))
1963 /* Even emitting a libcall is better than not detecting overflow
1968 gcc_assert (!is_ubsan
);
1969 ops
.code
= MULT_EXPR
;
1971 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1972 emit_jump (done_label
);
1977 emit_label (do_error
);
1980 /* Expand the ubsan builtin call. */
1982 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1986 do_pending_stack_adjust ();
1989 expand_arith_set_overflow (lhs
, target
);
1992 emit_label (done_label
);
1995 if (uns0_p
&& uns1_p
&& !unsr_p
)
1997 rtx_code_label
*all_done_label
= gen_label_rtx ();
1998 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1999 NULL
, all_done_label
, profile_probability::very_likely ());
2000 expand_arith_set_overflow (lhs
, target
);
2001 emit_label (all_done_label
);
2005 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2007 rtx_code_label
*all_done_label
= gen_label_rtx ();
2008 rtx_code_label
*set_noovf
= gen_label_rtx ();
2009 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2010 NULL
, all_done_label
, profile_probability::very_likely ());
2011 expand_arith_set_overflow (lhs
, target
);
2012 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2013 NULL
, set_noovf
, profile_probability::very_likely ());
2014 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2015 NULL
, all_done_label
, profile_probability::very_unlikely ());
2016 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2017 all_done_label
, profile_probability::very_unlikely ());
2018 emit_label (set_noovf
);
2019 write_complex_part (target
, const0_rtx
, true);
2020 emit_label (all_done_label
);
2026 expand_ubsan_result_store (target
, res
);
2028 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2032 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2035 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2036 tree arg0
, tree arg1
)
2038 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2039 rtx_code_label
*loop_lab
= NULL
;
2040 rtx cntvar
= NULL_RTX
;
2041 tree cntv
= NULL_TREE
;
2042 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2043 tree sz
= TYPE_SIZE (eltype
);
2044 tree data
= NULL_TREE
;
2045 tree resv
= NULL_TREE
;
2046 rtx lhsr
= NULL_RTX
;
2047 rtx resvr
= NULL_RTX
;
2048 unsigned HOST_WIDE_INT const_cnt
= 0;
2049 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2054 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2055 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2056 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2057 optab_default
)) == unknown_optab
2058 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2059 == CODE_FOR_nothing
))
2062 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2065 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2066 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2072 do_pending_stack_adjust ();
2073 loop_lab
= gen_label_rtx ();
2074 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2075 cntv
= make_tree (sizetype
, cntvar
);
2076 emit_move_insn (cntvar
, const0_rtx
);
2077 emit_label (loop_lab
);
2079 if (TREE_CODE (arg0
) != VECTOR_CST
)
2081 rtx arg0r
= expand_normal (arg0
);
2082 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2084 if (TREE_CODE (arg1
) != VECTOR_CST
)
2086 rtx arg1r
= expand_normal (arg1
);
2087 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2089 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2091 tree op0
, op1
, res
= NULL_TREE
;
2094 tree atype
= build_array_type_nelts (eltype
, cnt
);
2095 op0
= uniform_vector_p (arg0
);
2096 if (op0
== NULL_TREE
)
2098 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2099 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2100 NULL_TREE
, NULL_TREE
);
2102 op1
= uniform_vector_p (arg1
);
2103 if (op1
== NULL_TREE
)
2105 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2106 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2107 NULL_TREE
, NULL_TREE
);
2111 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2112 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2113 NULL_TREE
, NULL_TREE
);
2118 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2119 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2120 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2122 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2128 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2129 false, false, false, true, &data
);
2132 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2133 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2135 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2136 false, false, false, true, &data
);
2139 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2148 struct separate_ops ops
;
2149 ops
.code
= PLUS_EXPR
;
2150 ops
.type
= TREE_TYPE (cntv
);
2152 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2153 ops
.op2
= NULL_TREE
;
2155 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2158 emit_move_insn (cntvar
, ret
);
2159 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2160 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2161 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2162 profile_probability::very_likely ());
2164 if (lhs
&& resv
== NULL_TREE
)
2166 struct separate_ops ops
;
2168 ops
.type
= TREE_TYPE (arg0
);
2171 ops
.op2
= NULL_TREE
;
2173 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2176 emit_move_insn (lhsr
, ret
);
2179 emit_move_insn (lhsr
, resvr
);
2182 /* Expand UBSAN_CHECK_ADD call STMT. */
2185 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2187 location_t loc
= gimple_location (stmt
);
2188 tree lhs
= gimple_call_lhs (stmt
);
2189 tree arg0
= gimple_call_arg (stmt
, 0);
2190 tree arg1
= gimple_call_arg (stmt
, 1);
2191 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2192 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2194 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2195 false, false, false, true, NULL
);
2198 /* Expand UBSAN_CHECK_SUB call STMT. */
2201 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2203 location_t loc
= gimple_location (stmt
);
2204 tree lhs
= gimple_call_lhs (stmt
);
2205 tree arg0
= gimple_call_arg (stmt
, 0);
2206 tree arg1
= gimple_call_arg (stmt
, 1);
2207 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2208 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2209 else if (integer_zerop (arg0
))
2210 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2212 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2213 false, false, false, true, NULL
);
2216 /* Expand UBSAN_CHECK_MUL call STMT. */
2219 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2221 location_t loc
= gimple_location (stmt
);
2222 tree lhs
= gimple_call_lhs (stmt
);
2223 tree arg0
= gimple_call_arg (stmt
, 0);
2224 tree arg1
= gimple_call_arg (stmt
, 1);
2225 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2226 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2228 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2232 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2235 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2237 tree lhs
= gimple_call_lhs (stmt
);
2238 if (lhs
== NULL_TREE
)
2240 tree arg0
= gimple_call_arg (stmt
, 0);
2241 tree arg1
= gimple_call_arg (stmt
, 1);
2242 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2243 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2244 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2245 int unsr_p
= TYPE_UNSIGNED (type
);
2246 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2247 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2248 int precres
= TYPE_PRECISION (type
);
2249 location_t loc
= gimple_location (stmt
);
2250 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2252 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2254 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2255 prec0
= MIN (prec0
, pr
);
2256 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2257 prec1
= MIN (prec1
, pr
);
2259 /* If uns0_p && uns1_p, precop is minimum needed precision
2260 of unsigned type to hold the exact result, otherwise
2261 precop is minimum needed precision of signed type to
2262 hold the exact result. */
2264 if (code
== MULT_EXPR
)
2265 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2268 if (uns0_p
== uns1_p
)
2269 precop
= MAX (prec0
, prec1
) + 1;
2271 precop
= MAX (prec0
+ 1, prec1
) + 1;
2273 precop
= MAX (prec0
, prec1
+ 1) + 1;
2275 int orig_precres
= precres
;
2279 if ((uns0_p
&& uns1_p
)
2280 ? ((precop
+ !unsr_p
) <= precres
2281 /* u1 - u2 -> ur can overflow, no matter what precision
2283 && (code
!= MINUS_EXPR
|| !unsr_p
))
2284 : (!unsr_p
&& precop
<= precres
))
2286 /* The infinity precision result will always fit into result. */
2287 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2288 write_complex_part (target
, const0_rtx
, true);
2289 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2290 struct separate_ops ops
;
2293 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2294 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2295 ops
.op2
= NULL_TREE
;
2297 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2298 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2302 /* For operations with low precision, if target doesn't have them, start
2303 with precres widening right away, otherwise do it only if the most
2304 simple cases can't be used. */
2305 const int min_precision
= targetm
.min_arithmetic_precision ();
2306 if (orig_precres
== precres
&& precres
< min_precision
)
2308 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2309 && prec1
<= precres
)
2310 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2311 && prec0
+ uns0_p
<= precres
2312 && prec1
+ uns1_p
<= precres
))
2314 arg0
= fold_convert_loc (loc
, type
, arg0
);
2315 arg1
= fold_convert_loc (loc
, type
, arg1
);
2319 if (integer_zerop (arg0
) && !unsr_p
)
2321 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2326 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2327 unsr_p
, unsr_p
, false, NULL
);
2330 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2331 unsr_p
, unsr_p
, false, NULL
);
2338 /* For sub-word operations, retry with a wider type first. */
2339 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2341 int p
= MAX (min_precision
, precop
);
2342 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2343 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2346 p
= TYPE_PRECISION (optype
);
2350 unsr_p
= TYPE_UNSIGNED (optype
);
2356 if (prec0
<= precres
&& prec1
<= precres
)
2361 types
[0] = build_nonstandard_integer_type (precres
, 0);
2367 types
[1] = build_nonstandard_integer_type (precres
, 1);
2369 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2370 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2371 if (code
!= MULT_EXPR
)
2372 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2373 uns0_p
, uns1_p
, false, NULL
);
2375 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2376 uns0_p
, uns1_p
, false, NULL
);
2380 /* Retry with a wider type. */
2381 if (orig_precres
== precres
)
2383 int p
= MAX (prec0
, prec1
);
2384 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2385 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2388 p
= TYPE_PRECISION (optype
);
2392 unsr_p
= TYPE_UNSIGNED (optype
);
2403 /* Expand ADD_OVERFLOW STMT. */
2406 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2408 expand_arith_overflow (PLUS_EXPR
, stmt
);
2411 /* Expand SUB_OVERFLOW STMT. */
2414 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2416 expand_arith_overflow (MINUS_EXPR
, stmt
);
2419 /* Expand MUL_OVERFLOW STMT. */
2422 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2424 expand_arith_overflow (MULT_EXPR
, stmt
);
2427 /* This should get folded in tree-vectorizer.c. */
2430 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2435 /* This should get folded in tree-vectorizer.c. */
2438 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2443 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2444 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2447 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2449 tree addr
= gimple_call_arg (stmt
, index
);
2450 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2451 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2452 if (TYPE_ALIGN (type
) != align
)
2453 type
= build_aligned_type (type
, align
);
2456 if (TREE_CODE (tmp
) == SSA_NAME
)
2458 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2459 if (gimple_assign_single_p (def
))
2460 tmp
= gimple_assign_rhs1 (def
);
2463 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2465 tree mem
= TREE_OPERAND (tmp
, 0);
2466 if (TREE_CODE (mem
) == TARGET_MEM_REF
2467 && types_compatible_p (TREE_TYPE (mem
), type
))
2469 tree offset
= TMR_OFFSET (mem
);
2470 if (type
!= TREE_TYPE (mem
)
2471 || alias_ptr_type
!= TREE_TYPE (offset
)
2472 || !integer_zerop (offset
))
2474 mem
= copy_node (mem
);
2475 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2476 wi::to_poly_wide (offset
));
2477 TREE_TYPE (mem
) = type
;
2483 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2486 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2489 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2491 class expand_operand ops
[3];
2492 tree type
, lhs
, rhs
, maskt
;
2493 rtx mem
, target
, mask
;
2496 maskt
= gimple_call_arg (stmt
, 2);
2497 lhs
= gimple_call_lhs (stmt
);
2498 if (lhs
== NULL_TREE
)
2500 type
= TREE_TYPE (lhs
);
2501 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2503 if (optab
== vec_mask_load_lanes_optab
)
2504 icode
= get_multi_vector_move (type
, optab
);
2506 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2507 TYPE_MODE (TREE_TYPE (maskt
)));
2509 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2510 gcc_assert (MEM_P (mem
));
2511 mask
= expand_normal (maskt
);
2512 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2513 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2514 create_fixed_operand (&ops
[1], mem
);
2515 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2516 expand_insn (icode
, 3, ops
);
2517 if (!rtx_equal_p (target
, ops
[0].value
))
2518 emit_move_insn (target
, ops
[0].value
);
2521 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2523 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2526 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2528 class expand_operand ops
[3];
2529 tree type
, lhs
, rhs
, maskt
;
2533 maskt
= gimple_call_arg (stmt
, 2);
2534 rhs
= gimple_call_arg (stmt
, 3);
2535 type
= TREE_TYPE (rhs
);
2536 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2538 if (optab
== vec_mask_store_lanes_optab
)
2539 icode
= get_multi_vector_move (type
, optab
);
2541 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2542 TYPE_MODE (TREE_TYPE (maskt
)));
2544 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2545 gcc_assert (MEM_P (mem
));
2546 mask
= expand_normal (maskt
);
2547 reg
= expand_normal (rhs
);
2548 create_fixed_operand (&ops
[0], mem
);
2549 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2550 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2551 expand_insn (icode
, 3, ops
);
2554 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2556 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2557 The expansion of STMT happens based on OPTAB table associated. */
2560 expand_vect_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2562 class expand_operand ops
[6];
2564 tree lhs
= gimple_call_lhs (stmt
);
2565 tree op0a
= gimple_call_arg (stmt
, 0);
2566 tree op0b
= gimple_call_arg (stmt
, 1);
2567 tree op1
= gimple_call_arg (stmt
, 2);
2568 tree op2
= gimple_call_arg (stmt
, 3);
2569 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
2571 tree vec_cond_type
= TREE_TYPE (lhs
);
2572 tree op_mode
= TREE_TYPE (op0a
);
2573 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
2575 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2576 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
2578 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
2580 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
2581 rtx rtx_op1
= expand_normal (op1
);
2582 rtx rtx_op2
= expand_normal (op2
);
2584 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2585 create_output_operand (&ops
[0], target
, mode
);
2586 create_input_operand (&ops
[1], rtx_op1
, mode
);
2587 create_input_operand (&ops
[2], rtx_op2
, mode
);
2588 create_fixed_operand (&ops
[3], comparison
);
2589 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
2590 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
2591 expand_insn (icode
, 6, ops
);
2594 #define expand_vec_cond_optab_fn expand_vect_cond_optab_fn
2595 #define expand_vec_condu_optab_fn expand_vect_cond_optab_fn
2596 #define expand_vec_condeq_optab_fn expand_vect_cond_optab_fn
2598 /* Expand VCOND_MASK optab internal function.
2599 The expansion of STMT happens based on OPTAB table associated. */
2602 expand_vect_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2604 class expand_operand ops
[4];
2606 tree lhs
= gimple_call_lhs (stmt
);
2607 tree op0
= gimple_call_arg (stmt
, 0);
2608 tree op1
= gimple_call_arg (stmt
, 1);
2609 tree op2
= gimple_call_arg (stmt
, 2);
2610 tree vec_cond_type
= TREE_TYPE (lhs
);
2612 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2613 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
2614 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
2615 rtx mask
, rtx_op1
, rtx_op2
;
2617 gcc_assert (icode
!= CODE_FOR_nothing
);
2619 mask
= expand_normal (op0
);
2620 rtx_op1
= expand_normal (op1
);
2621 rtx_op2
= expand_normal (op2
);
2623 mask
= force_reg (mask_mode
, mask
);
2624 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
2626 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2627 create_output_operand (&ops
[0], target
, mode
);
2628 create_input_operand (&ops
[1], rtx_op1
, mode
);
2629 create_input_operand (&ops
[2], rtx_op2
, mode
);
2630 create_input_operand (&ops
[3], mask
, mask_mode
);
2631 expand_insn (icode
, 4, ops
);
2634 #define expand_vec_cond_mask_optab_fn expand_vect_cond_mask_optab_fn
2637 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2642 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2644 /* When guessing was done, the hints should be already stripped away. */
2645 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2648 tree lhs
= gimple_call_lhs (stmt
);
2650 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2652 target
= const0_rtx
;
2653 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2654 if (lhs
&& val
!= target
)
2655 emit_move_insn (target
, val
);
2658 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2659 should never be called. */
2662 expand_VA_ARG (internal_fn
, gcall
*)
2667 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2668 dummy function should never be called. */
2671 expand_VEC_CONVERT (internal_fn
, gcall
*)
2676 /* Expand the IFN_UNIQUE function according to its first argument. */
2679 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2681 rtx pattern
= NULL_RTX
;
2682 enum ifn_unique_kind kind
2683 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2690 case IFN_UNIQUE_UNSPEC
:
2691 if (targetm
.have_unique ())
2692 pattern
= targetm
.gen_unique ();
2695 case IFN_UNIQUE_OACC_FORK
:
2696 case IFN_UNIQUE_OACC_JOIN
:
2697 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2699 tree lhs
= gimple_call_lhs (stmt
);
2700 rtx target
= const0_rtx
;
2703 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2705 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2706 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2708 if (kind
== IFN_UNIQUE_OACC_FORK
)
2709 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2711 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2719 emit_insn (pattern
);
2722 /* The size of an OpenACC compute dimension. */
2725 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2727 tree lhs
= gimple_call_lhs (stmt
);
2732 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2733 if (targetm
.have_oacc_dim_size ())
2735 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2736 VOIDmode
, EXPAND_NORMAL
);
2737 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2740 emit_move_insn (target
, GEN_INT (1));
2743 /* The position of an OpenACC execution engine along one compute axis. */
2746 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2748 tree lhs
= gimple_call_lhs (stmt
);
2753 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2754 if (targetm
.have_oacc_dim_pos ())
2756 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2757 VOIDmode
, EXPAND_NORMAL
);
2758 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2761 emit_move_insn (target
, const0_rtx
);
2764 /* This is expanded by oacc_device_lower pass. */
2767 expand_GOACC_LOOP (internal_fn
, gcall
*)
2772 /* This is expanded by oacc_device_lower pass. */
2775 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2780 /* This is expanded by oacc_device_lower pass. */
2783 expand_GOACC_TILE (internal_fn
, gcall
*)
2788 /* Set errno to EDOM. */
2791 expand_SET_EDOM (internal_fn
, gcall
*)
2794 #ifdef GEN_ERRNO_RTX
2795 rtx errno_rtx
= GEN_ERRNO_RTX
;
2797 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2799 emit_move_insn (errno_rtx
,
2800 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2806 /* Expand atomic bit test and set. */
2809 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2811 expand_ifn_atomic_bit_test_and (call
);
2814 /* Expand atomic bit test and complement. */
2817 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2819 expand_ifn_atomic_bit_test_and (call
);
2822 /* Expand atomic bit test and reset. */
2825 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2827 expand_ifn_atomic_bit_test_and (call
);
2830 /* Expand atomic bit test and set. */
2833 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2835 expand_ifn_atomic_compare_exchange (call
);
2838 /* Expand LAUNDER to assignment, lhs = arg0. */
2841 expand_LAUNDER (internal_fn
, gcall
*call
)
2843 tree lhs
= gimple_call_lhs (call
);
2848 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2851 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
2854 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
2856 internal_fn ifn
= gimple_call_internal_fn (stmt
);
2857 int rhs_index
= internal_fn_stored_value_index (ifn
);
2858 int mask_index
= internal_fn_mask_index (ifn
);
2859 tree base
= gimple_call_arg (stmt
, 0);
2860 tree offset
= gimple_call_arg (stmt
, 1);
2861 tree scale
= gimple_call_arg (stmt
, 2);
2862 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
2864 rtx base_rtx
= expand_normal (base
);
2865 rtx offset_rtx
= expand_normal (offset
);
2866 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
2867 rtx rhs_rtx
= expand_normal (rhs
);
2869 class expand_operand ops
[6];
2871 create_address_operand (&ops
[i
++], base_rtx
);
2872 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
2873 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
2874 create_integer_operand (&ops
[i
++], scale_int
);
2875 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
2876 if (mask_index
>= 0)
2878 tree mask
= gimple_call_arg (stmt
, mask_index
);
2879 rtx mask_rtx
= expand_normal (mask
);
2880 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
2883 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
2884 TYPE_MODE (TREE_TYPE (offset
)));
2885 expand_insn (icode
, i
, ops
);
2888 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
2891 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
2893 tree lhs
= gimple_call_lhs (stmt
);
2894 tree base
= gimple_call_arg (stmt
, 0);
2895 tree offset
= gimple_call_arg (stmt
, 1);
2896 tree scale
= gimple_call_arg (stmt
, 2);
2898 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2899 rtx base_rtx
= expand_normal (base
);
2900 rtx offset_rtx
= expand_normal (offset
);
2901 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
2904 class expand_operand ops
[6];
2905 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
2906 create_address_operand (&ops
[i
++], base_rtx
);
2907 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
2908 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
2909 create_integer_operand (&ops
[i
++], scale_int
);
2910 if (optab
== mask_gather_load_optab
)
2912 tree mask
= gimple_call_arg (stmt
, 4);
2913 rtx mask_rtx
= expand_normal (mask
);
2914 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
2916 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
2917 TYPE_MODE (TREE_TYPE (offset
)));
2918 expand_insn (icode
, i
, ops
);
2919 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2920 emit_move_insn (lhs_rtx
, ops
[0].value
);
2923 /* Expand DIVMOD() using:
2924 a) optab handler for udivmod/sdivmod if it is available.
2925 b) If optab_handler doesn't exist, generate call to
2926 target-specific divmod libfunc. */
2929 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2931 tree lhs
= gimple_call_lhs (call_stmt
);
2932 tree arg0
= gimple_call_arg (call_stmt
, 0);
2933 tree arg1
= gimple_call_arg (call_stmt
, 1);
2935 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2936 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2937 machine_mode mode
= TYPE_MODE (type
);
2938 bool unsignedp
= TYPE_UNSIGNED (type
);
2939 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2941 rtx op0
= expand_normal (arg0
);
2942 rtx op1
= expand_normal (arg1
);
2943 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2945 rtx quotient
, remainder
, libfunc
;
2947 /* Check if optab_handler exists for divmod_optab for given mode. */
2948 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2950 quotient
= gen_reg_rtx (mode
);
2951 remainder
= gen_reg_rtx (mode
);
2952 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2955 /* Generate call to divmod libfunc if it exists. */
2956 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2957 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2958 "ient
, &remainder
);
2963 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2964 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2965 make_tree (TREE_TYPE (arg0
), quotient
),
2966 make_tree (TREE_TYPE (arg1
), remainder
)),
2967 target
, VOIDmode
, EXPAND_NORMAL
);
2973 expand_NOP (internal_fn
, gcall
*)
2975 /* Nothing. But it shouldn't really prevail. */
2978 /* Coroutines, all should have been processed at this stage. */
2981 expand_CO_FRAME (internal_fn
, gcall
*)
2987 expand_CO_YIELD (internal_fn
, gcall
*)
2993 expand_CO_SUSPN (internal_fn
, gcall
*)
2999 expand_CO_ACTOR (internal_fn
, gcall
*)
3004 /* Expand a call to FN using the operands in STMT. FN has a single
3005 output operand and NARGS input operands. */
3008 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3011 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
3013 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3014 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3015 gcc_assert (icode
!= CODE_FOR_nothing
);
3017 tree lhs
= gimple_call_lhs (stmt
);
3018 rtx lhs_rtx
= NULL_RTX
;
3020 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3022 /* Do not assign directly to a promoted subreg, since there is no
3023 guarantee that the instruction will leave the upper bits of the
3024 register in the state required by SUBREG_PROMOTED_SIGN. */
3026 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
3029 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
3031 for (unsigned int i
= 0; i
< nargs
; ++i
)
3033 tree rhs
= gimple_call_arg (stmt
, i
);
3034 tree rhs_type
= TREE_TYPE (rhs
);
3035 rtx rhs_rtx
= expand_normal (rhs
);
3036 if (INTEGRAL_TYPE_P (rhs_type
))
3037 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
3038 TYPE_MODE (rhs_type
),
3039 TYPE_UNSIGNED (rhs_type
));
3041 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
3044 expand_insn (icode
, nargs
+ 1, ops
);
3045 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
3047 /* If the return value has an integral type, convert the instruction
3048 result to that type. This is useful for things that return an
3049 int regardless of the size of the input. If the instruction result
3050 is smaller than required, assume that it is signed.
3052 If the return value has a nonintegral type, its mode must match
3053 the instruction result. */
3054 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
3056 /* If this is a scalar in a register that is stored in a wider
3057 mode than the declared mode, compute the result into its
3058 declared mode and then convert to the wider mode. */
3059 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3060 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
3061 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
3062 SUBREG_PROMOTED_SIGN (lhs_rtx
));
3064 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
3065 emit_move_insn (lhs_rtx
, ops
[0].value
);
3068 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3069 convert_move (lhs_rtx
, ops
[0].value
, 0);
3074 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3077 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3079 expand_operand ops
[3];
3082 tree lhs
= gimple_call_lhs (stmt
);
3083 tree lhs_type
= TREE_TYPE (lhs
);
3084 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3085 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3087 for (unsigned int i
= 0; i
< 2; ++i
)
3089 tree rhs
= gimple_call_arg (stmt
, i
);
3090 rhs_type
[i
] = TREE_TYPE (rhs
);
3091 rtx rhs_rtx
= expand_normal (rhs
);
3092 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3095 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3096 TYPE_MODE (lhs_type
));
3098 expand_insn (icode
, 3, ops
);
3099 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3100 emit_move_insn (lhs_rtx
, ops
[0].value
);
3103 /* Expanders for optabs that can use expand_direct_optab_fn. */
3105 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3106 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3108 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3109 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3111 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3112 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3114 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3115 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3117 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3118 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3120 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3121 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3123 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3124 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3126 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3127 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3129 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3130 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3132 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3133 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3135 /* RETURN_TYPE and ARGS are a return type and argument list that are
3136 in principle compatible with FN (which satisfies direct_internal_fn_p).
3137 Return the types that should be used to determine whether the
3138 target supports FN. */
3141 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3143 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3144 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3145 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3146 return tree_pair (type0
, type1
);
3149 /* CALL is a call whose return type and arguments are in principle
3150 compatible with FN (which satisfies direct_internal_fn_p). Return the
3151 types that should be used to determine whether the target supports FN. */
3154 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3156 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3157 tree op0
= (info
.type0
< 0
3158 ? gimple_call_lhs (call
)
3159 : gimple_call_arg (call
, info
.type0
));
3160 tree op1
= (info
.type1
< 0
3161 ? gimple_call_lhs (call
)
3162 : gimple_call_arg (call
, info
.type1
));
3163 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3166 /* Return true if OPTAB is supported for TYPES (whose modes should be
3167 the same) when the optimization type is OPT_TYPE. Used for simple
3171 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3172 optimization_type opt_type
)
3174 machine_mode mode
= TYPE_MODE (types
.first
);
3175 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3176 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3179 /* Return true if OPTAB is supported for TYPES, where the first type
3180 is the destination and the second type is the source. Used for
3184 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3185 optimization_type opt_type
)
3187 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3188 TYPE_MODE (types
.second
), opt_type
)
3189 != CODE_FOR_nothing
);
3192 /* Return true if load/store lanes optab OPTAB is supported for
3193 array type TYPES.first when the optimization type is OPT_TYPE. */
3196 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3197 optimization_type opt_type
)
3199 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3200 machine_mode imode
= TYPE_MODE (types
.first
);
3201 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3202 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3203 != CODE_FOR_nothing
);
3206 #define direct_unary_optab_supported_p direct_optab_supported_p
3207 #define direct_binary_optab_supported_p direct_optab_supported_p
3208 #define direct_ternary_optab_supported_p direct_optab_supported_p
3209 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3210 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3211 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3212 #define direct_mask_load_optab_supported_p direct_optab_supported_p
3213 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3214 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3215 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3216 #define direct_mask_store_optab_supported_p direct_optab_supported_p
3217 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3218 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3219 #define direct_vec_cond_mask_optab_supported_p multi_vector_optab_supported_p
3220 #define direct_vec_cond_optab_supported_p multi_vector_optab_supported_p
3221 #define direct_vec_condu_optab_supported_p multi_vector_optab_supported_p
3222 #define direct_vec_condeq_optab_supported_p multi_vector_optab_supported_p
3223 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3224 #define direct_while_optab_supported_p convert_optab_supported_p
3225 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3226 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3227 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3228 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3230 /* Return the optab used by internal function FN. */
3233 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3237 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3238 case IFN_##CODE: break;
3239 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3240 case IFN_##CODE: return OPTAB##_optab;
3241 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3242 UNSIGNED_OPTAB, TYPE) \
3243 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3244 ? UNSIGNED_OPTAB ## _optab \
3245 : SIGNED_OPTAB ## _optab);
3246 #include "internal-fn.def"
3254 /* Return the optab used by internal function FN. */
3257 direct_internal_fn_optab (internal_fn fn
)
3261 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3262 case IFN_##CODE: break;
3263 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3264 case IFN_##CODE: return OPTAB##_optab;
3265 #include "internal-fn.def"
3273 /* Return true if FN is supported for the types in TYPES when the
3274 optimization type is OPT_TYPE. The types are those associated with
3275 the "type0" and "type1" fields of FN's direct_internal_fn_info
3279 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3280 optimization_type opt_type
)
3284 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3285 case IFN_##CODE: break;
3286 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3288 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3290 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3291 UNSIGNED_OPTAB, TYPE) \
3294 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3295 ? UNSIGNED_OPTAB ## _optab \
3296 : SIGNED_OPTAB ## _optab); \
3297 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3300 #include "internal-fn.def"
3308 /* Return true if FN is supported for type TYPE when the optimization
3309 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3310 fields of FN's direct_internal_fn_info structure are the same. */
3313 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3314 optimization_type opt_type
)
3316 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3317 gcc_checking_assert (info
.type0
== info
.type1
);
3318 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3321 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3322 given that STMT is a call to a direct internal function. */
3325 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
3327 internal_fn fn
= gimple_call_internal_fn (stmt
);
3328 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3329 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
3332 /* If FN is commutative in two consecutive arguments, return the
3333 index of the first, otherwise return -1. */
3336 first_commutative_argument (internal_fn fn
)
3370 /* Return true if IFN_SET_EDOM is supported. */
3373 set_edom_supported_p (void)
3382 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3384 expand_##CODE (internal_fn fn, gcall *stmt) \
3386 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3388 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3389 UNSIGNED_OPTAB, TYPE) \
3391 expand_##CODE (internal_fn fn, gcall *stmt) \
3393 tree_pair types = direct_internal_fn_types (fn, stmt); \
3394 optab which_optab = direct_internal_fn_optab (fn, types); \
3395 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3397 #include "internal-fn.def"
3399 /* Routines to expand each internal function, indexed by function number.
3400 Each routine has the prototype:
3402 expand_<NAME> (gcall *stmt)
3404 where STMT is the statement that performs the call. */
3405 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3406 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3407 #include "internal-fn.def"
3411 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
3413 #define FOR_EACH_CODE_MAPPING(T) \
3414 T (PLUS_EXPR, IFN_COND_ADD) \
3415 T (MINUS_EXPR, IFN_COND_SUB) \
3416 T (MULT_EXPR, IFN_COND_MUL) \
3417 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
3418 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
3419 T (RDIV_EXPR, IFN_COND_RDIV) \
3420 T (MIN_EXPR, IFN_COND_MIN) \
3421 T (MAX_EXPR, IFN_COND_MAX) \
3422 T (BIT_AND_EXPR, IFN_COND_AND) \
3423 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3424 T (BIT_XOR_EXPR, IFN_COND_XOR) \
3425 T (LSHIFT_EXPR, IFN_COND_SHL) \
3426 T (RSHIFT_EXPR, IFN_COND_SHR)
3428 /* Return a function that only performs CODE when a certain condition is met
3429 and that uses a given fallback value otherwise. For example, if CODE is
3430 a binary operation associated with conditional function FN:
3432 LHS = FN (COND, A, B, ELSE)
3434 is equivalent to the C expression:
3436 LHS = COND ? A CODE B : ELSE;
3438 operating elementwise if the operands are vectors.
3440 Return IFN_LAST if no such function exists. */
3443 get_conditional_internal_fn (tree_code code
)
3447 #define CASE(CODE, IFN) case CODE: return IFN;
3448 FOR_EACH_CODE_MAPPING(CASE
)
3455 /* If IFN implements the conditional form of a tree code, return that
3456 tree code, otherwise return ERROR_MARK. */
3459 conditional_internal_fn_code (internal_fn ifn
)
3463 #define CASE(CODE, IFN) case IFN: return CODE;
3464 FOR_EACH_CODE_MAPPING(CASE
)
3471 /* Invoke T(IFN) for each internal function IFN that also has an
3473 #define FOR_EACH_COND_FN_PAIR(T) \
3479 /* Return a function that only performs internal function FN when a
3480 certain condition is met and that uses a given fallback value otherwise.
3481 In other words, the returned function FN' is such that:
3483 LHS = FN' (COND, A1, ... An, ELSE)
3485 is equivalent to the C expression:
3487 LHS = COND ? FN (A1, ..., An) : ELSE;
3489 operating elementwise if the operands are vectors.
3491 Return IFN_LAST if no such function exists. */
3494 get_conditional_internal_fn (internal_fn fn
)
3498 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
3499 FOR_EACH_COND_FN_PAIR(CASE
)
3506 /* If IFN implements the conditional form of an unconditional internal
3507 function, return that unconditional function, otherwise return IFN_LAST. */
3510 get_unconditional_internal_fn (internal_fn ifn
)
3514 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
3515 FOR_EACH_COND_FN_PAIR(CASE
)
3522 /* Return true if STMT can be interpreted as a conditional tree code
3523 operation of the form:
3525 LHS = COND ? OP (RHS1, ...) : ELSE;
3527 operating elementwise if the operands are vectors. This includes
3528 the case of an all-true COND, so that the operation always happens.
3530 When returning true, set:
3532 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
3533 is known to be all-true
3534 - *CODE_OUT to the tree code
3535 - OPS[I] to operand I of *CODE_OUT
3536 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
3537 condition is known to be all true. */
3540 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
3541 tree_code
*code_out
,
3542 tree (&ops
)[3], tree
*else_out
)
3544 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
3546 *cond_out
= NULL_TREE
;
3547 *code_out
= gimple_assign_rhs_code (assign
);
3548 ops
[0] = gimple_assign_rhs1 (assign
);
3549 ops
[1] = gimple_assign_rhs2 (assign
);
3550 ops
[2] = gimple_assign_rhs3 (assign
);
3551 *else_out
= NULL_TREE
;
3554 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
3555 if (gimple_call_internal_p (call
))
3557 internal_fn ifn
= gimple_call_internal_fn (call
);
3558 tree_code code
= conditional_internal_fn_code (ifn
);
3559 if (code
!= ERROR_MARK
)
3561 *cond_out
= gimple_call_arg (call
, 0);
3563 unsigned int nops
= gimple_call_num_args (call
) - 2;
3564 for (unsigned int i
= 0; i
< 3; ++i
)
3565 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
3566 *else_out
= gimple_call_arg (call
, nops
+ 1);
3567 if (integer_truep (*cond_out
))
3569 *cond_out
= NULL_TREE
;
3570 *else_out
= NULL_TREE
;
3578 /* Return true if IFN is some form of load from memory. */
3581 internal_load_fn_p (internal_fn fn
)
3586 case IFN_LOAD_LANES
:
3587 case IFN_MASK_LOAD_LANES
:
3588 case IFN_GATHER_LOAD
:
3589 case IFN_MASK_GATHER_LOAD
:
3597 /* Return true if IFN is some form of store to memory. */
3600 internal_store_fn_p (internal_fn fn
)
3604 case IFN_MASK_STORE
:
3605 case IFN_STORE_LANES
:
3606 case IFN_MASK_STORE_LANES
:
3607 case IFN_SCATTER_STORE
:
3608 case IFN_MASK_SCATTER_STORE
:
3616 /* Return true if IFN is some form of gather load or scatter store. */
3619 internal_gather_scatter_fn_p (internal_fn fn
)
3623 case IFN_GATHER_LOAD
:
3624 case IFN_MASK_GATHER_LOAD
:
3625 case IFN_SCATTER_STORE
:
3626 case IFN_MASK_SCATTER_STORE
:
3634 /* If FN takes a vector mask argument, return the index of that argument,
3635 otherwise return -1. */
3638 internal_fn_mask_index (internal_fn fn
)
3643 case IFN_MASK_LOAD_LANES
:
3644 case IFN_MASK_STORE
:
3645 case IFN_MASK_STORE_LANES
:
3648 case IFN_MASK_GATHER_LOAD
:
3649 case IFN_MASK_SCATTER_STORE
:
3653 return (conditional_internal_fn_code (fn
) != ERROR_MARK
3654 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
3658 /* If FN takes a value that should be stored to memory, return the index
3659 of that argument, otherwise return -1. */
3662 internal_fn_stored_value_index (internal_fn fn
)
3666 case IFN_MASK_STORE
:
3667 case IFN_SCATTER_STORE
:
3668 case IFN_MASK_SCATTER_STORE
:
3676 /* Return true if the target supports gather load or scatter store function
3677 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
3678 while for stores it is the vector type of the stored data argument.
3679 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
3680 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
3681 offset from the shared base address of each loaded or stored element.
3682 SCALE is the amount by which these offsets should be multiplied
3683 *after* they have been extended to address width. */
3686 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
3687 tree memory_element_type
,
3688 tree offset_vector_type
, int scale
)
3690 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
3691 TYPE_SIZE (memory_element_type
)))
3693 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
3694 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
3696 optab optab
= direct_internal_fn_optab (ifn
);
3697 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
3698 TYPE_MODE (offset_vector_type
));
3699 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
3700 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
3701 return (icode
!= CODE_FOR_nothing
3702 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
3703 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
3706 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
3707 for pointers of type TYPE when the accesses have LENGTH bytes and their
3708 common byte alignment is ALIGN. */
3711 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
3712 poly_uint64 length
, unsigned int align
)
3714 machine_mode mode
= TYPE_MODE (type
);
3715 optab optab
= direct_internal_fn_optab (ifn
);
3716 insn_code icode
= direct_optab_handler (optab
, mode
);
3717 if (icode
== CODE_FOR_nothing
)
3719 rtx length_rtx
= immed_wide_int_const (length
, mode
);
3720 return (insn_operand_matches (icode
, 3, length_rtx
)
3721 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
3724 /* Expand STMT as though it were a call to internal function FN. */
3727 expand_internal_call (internal_fn fn
, gcall
*stmt
)
3729 internal_fn_expanders
[fn
] (fn
, stmt
);
3732 /* Expand STMT, which is a call to internal function FN. */
3735 expand_internal_call (gcall
*stmt
)
3737 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
3740 /* If TYPE is a vector type, return true if IFN is a direct internal
3741 function that is supported for that type. If TYPE is a scalar type,
3742 return true if IFN is a direct internal function that is supported for
3743 the target's preferred vector version of TYPE. */
3746 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
3749 if (!VECTOR_TYPE_P (type
) && is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
3751 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
3752 if (VECTOR_MODE_P (vmode
))
3753 type
= build_vector_type_for_mode (type
, vmode
);
3756 return (VECTOR_MODE_P (TYPE_MODE (type
))
3757 && direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
));
3761 expand_PHI (internal_fn
, gcall
*)