2 Copyright (C) 2011-2020 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
55 /* The names of each internal function, indexed by function number. */
56 const char *const internal_fn_name_array
[] = {
57 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
58 #include "internal-fn.def"
62 /* The ECF_* flags of each internal function, indexed by function number. */
63 const int internal_fn_flags_array
[] = {
64 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
65 #include "internal-fn.def"
69 /* Return the internal function called NAME, or IFN_LAST if there's
73 lookup_internal_fn (const char *name
)
75 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
76 static name_to_fn_map_type
*name_to_fn_map
;
80 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
81 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
82 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
85 internal_fn
*entry
= name_to_fn_map
->get (name
);
86 return entry
? *entry
: IFN_LAST
;
89 /* Fnspec of each internal function, indexed by function number. */
90 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
95 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
96 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
97 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
98 #include "internal-fn.def"
99 internal_fn_fnspec_array
[IFN_LAST
] = 0;
102 /* Create static initializers for the information returned by
103 direct_internal_fn. */
104 #define not_direct { -2, -2, false }
105 #define mask_load_direct { -1, 2, false }
106 #define load_lanes_direct { -1, -1, false }
107 #define mask_load_lanes_direct { -1, -1, false }
108 #define gather_load_direct { 3, 1, false }
109 #define len_load_direct { -1, -1, false }
110 #define mask_store_direct { 3, 2, false }
111 #define store_lanes_direct { 0, 0, false }
112 #define mask_store_lanes_direct { 0, 0, false }
113 #define vec_cond_mask_direct { 0, 0, false }
114 #define vec_cond_direct { 0, 0, false }
115 #define vec_condu_direct { 0, 0, false }
116 #define vec_condeq_direct { 0, 0, false }
117 #define scatter_store_direct { 3, 1, false }
118 #define len_store_direct { 3, 3, false }
119 #define vec_set_direct { 3, 3, false }
120 #define unary_direct { 0, 0, true }
121 #define binary_direct { 0, 0, true }
122 #define ternary_direct { 0, 0, true }
123 #define cond_unary_direct { 1, 1, true }
124 #define cond_binary_direct { 1, 1, true }
125 #define cond_ternary_direct { 1, 1, true }
126 #define while_direct { 0, 2, false }
127 #define fold_extract_direct { 2, 2, false }
128 #define fold_left_direct { 1, 1, false }
129 #define mask_fold_left_direct { 1, 1, false }
130 #define check_ptrs_direct { 0, 0, false }
132 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
133 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
134 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
135 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
136 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
137 #include "internal-fn.def"
141 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
142 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
144 static enum insn_code
145 get_multi_vector_move (tree array_type
, convert_optab optab
)
150 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
151 imode
= TYPE_MODE (array_type
);
152 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
154 return convert_optab_handler (optab
, imode
, vmode
);
157 /* Expand LOAD_LANES call STMT using optab OPTAB. */
160 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
162 class expand_operand ops
[2];
166 lhs
= gimple_call_lhs (stmt
);
167 rhs
= gimple_call_arg (stmt
, 0);
168 type
= TREE_TYPE (lhs
);
170 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
171 mem
= expand_normal (rhs
);
173 gcc_assert (MEM_P (mem
));
174 PUT_MODE (mem
, TYPE_MODE (type
));
176 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
177 create_fixed_operand (&ops
[1], mem
);
178 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
179 if (!rtx_equal_p (target
, ops
[0].value
))
180 emit_move_insn (target
, ops
[0].value
);
183 /* Expand STORE_LANES call STMT using optab OPTAB. */
186 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
188 class expand_operand ops
[2];
192 lhs
= gimple_call_lhs (stmt
);
193 rhs
= gimple_call_arg (stmt
, 0);
194 type
= TREE_TYPE (rhs
);
196 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
197 reg
= expand_normal (rhs
);
199 gcc_assert (MEM_P (target
));
200 PUT_MODE (target
, TYPE_MODE (type
));
202 create_fixed_operand (&ops
[0], target
);
203 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
204 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
208 expand_ANNOTATE (internal_fn
, gcall
*)
213 /* This should get expanded in omp_device_lower pass. */
216 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
221 /* This should get expanded in omp_device_lower pass. */
224 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
229 /* Allocate per-lane storage and begin non-uniform execution region. */
232 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
235 tree lhs
= gimple_call_lhs (stmt
);
237 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
239 target
= gen_reg_rtx (Pmode
);
240 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
241 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
242 class expand_operand ops
[3];
243 create_output_operand (&ops
[0], target
, Pmode
);
244 create_input_operand (&ops
[1], size
, Pmode
);
245 create_input_operand (&ops
[2], align
, Pmode
);
246 gcc_assert (targetm
.have_omp_simt_enter ());
247 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
250 /* Deallocate per-lane storage and leave non-uniform execution region. */
253 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
255 gcc_checking_assert (!gimple_call_lhs (stmt
));
256 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
257 class expand_operand ops
[1];
258 create_input_operand (&ops
[0], arg
, Pmode
);
259 gcc_assert (targetm
.have_omp_simt_exit ());
260 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
263 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
264 without SIMT execution this should be expanded in omp_device_lower pass. */
267 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
269 tree lhs
= gimple_call_lhs (stmt
);
273 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
274 gcc_assert (targetm
.have_omp_simt_lane ());
275 emit_insn (targetm
.gen_omp_simt_lane (target
));
278 /* This should get expanded in omp_device_lower pass. */
281 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
286 /* Lane index of the first SIMT lane that supplies a non-zero argument.
287 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
288 lane that executed the last iteration for handling OpenMP lastprivate. */
291 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
293 tree lhs
= gimple_call_lhs (stmt
);
297 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
298 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
299 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
300 class expand_operand ops
[2];
301 create_output_operand (&ops
[0], target
, mode
);
302 create_input_operand (&ops
[1], cond
, mode
);
303 gcc_assert (targetm
.have_omp_simt_last_lane ());
304 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
307 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
310 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
312 tree lhs
= gimple_call_lhs (stmt
);
316 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
317 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
318 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
319 class expand_operand ops
[2];
320 create_output_operand (&ops
[0], target
, mode
);
321 create_input_operand (&ops
[1], ctr
, mode
);
322 gcc_assert (targetm
.have_omp_simt_ordered ());
323 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
326 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
327 any lane supplies a non-zero argument. */
330 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
332 tree lhs
= gimple_call_lhs (stmt
);
336 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
337 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
338 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
339 class expand_operand ops
[2];
340 create_output_operand (&ops
[0], target
, mode
);
341 create_input_operand (&ops
[1], cond
, mode
);
342 gcc_assert (targetm
.have_omp_simt_vote_any ());
343 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
346 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
347 is destination lane index XOR given offset. */
350 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
352 tree lhs
= gimple_call_lhs (stmt
);
356 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
357 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
358 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
359 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
360 class expand_operand ops
[3];
361 create_output_operand (&ops
[0], target
, mode
);
362 create_input_operand (&ops
[1], src
, mode
);
363 create_input_operand (&ops
[2], idx
, SImode
);
364 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
365 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
368 /* Exchange between SIMT lanes according to given source lane index. */
371 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
373 tree lhs
= gimple_call_lhs (stmt
);
377 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
378 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
379 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
380 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
381 class expand_operand ops
[3];
382 create_output_operand (&ops
[0], target
, mode
);
383 create_input_operand (&ops
[1], src
, mode
);
384 create_input_operand (&ops
[2], idx
, SImode
);
385 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
386 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
389 /* This should get expanded in adjust_simduid_builtins. */
392 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
397 /* This should get expanded in adjust_simduid_builtins. */
400 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
405 /* This should get expanded in adjust_simduid_builtins. */
408 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
413 /* This should get expanded in adjust_simduid_builtins. */
416 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
421 /* This should get expanded in adjust_simduid_builtins. */
424 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
429 /* This should get expanded in the sanopt pass. */
432 expand_UBSAN_NULL (internal_fn
, gcall
*)
437 /* This should get expanded in the sanopt pass. */
440 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
445 /* This should get expanded in the sanopt pass. */
448 expand_UBSAN_VPTR (internal_fn
, gcall
*)
453 /* This should get expanded in the sanopt pass. */
456 expand_UBSAN_PTR (internal_fn
, gcall
*)
461 /* This should get expanded in the sanopt pass. */
464 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
469 /* This should get expanded in the sanopt pass. */
472 expand_HWASAN_CHECK (internal_fn
, gcall
*)
477 /* For hwasan stack tagging:
478 Clear tags on the dynamically allocated space.
479 For use after an object dynamically allocated on the stack goes out of
482 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
484 gcc_assert (Pmode
== ptr_mode
);
485 tree restored_position
= gimple_call_arg (gc
, 0);
486 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
488 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
489 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
490 stack_pointer_rtx
, NULL_RTX
, 0,
492 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
493 virtual_stack_dynamic_rtx
, Pmode
,
494 HWASAN_STACK_BACKGROUND
, QImode
,
498 /* For hwasan stack tagging:
499 Return a tag to be used for a dynamic allocation. */
501 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
503 tree tag
= gimple_call_lhs (gc
);
504 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
505 machine_mode mode
= GET_MODE (target
);
506 gcc_assert (mode
== QImode
);
508 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
509 gcc_assert (base_tag
);
510 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
511 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
512 target
, /* unsignedp = */1,
514 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
516 /* Really need to put the tag into the `target` RTX. */
517 if (chosen_tag
!= target
)
519 rtx temp
= chosen_tag
;
520 gcc_assert (GET_MODE (chosen_tag
) == mode
);
521 emit_move_insn (target
, temp
);
524 hwasan_increment_frame_tag ();
527 /* For hwasan stack tagging:
528 Tag a region of space in the shadow stack according to the base pointer of
529 an object on the stack. N.b. the length provided in the internal call is
530 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
532 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
534 gcc_assert (ptr_mode
== Pmode
);
535 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
536 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
538 tree base
= gimple_call_arg (gc
, 1);
539 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
540 rtx base_rtx
= expand_normal (base
);
542 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
543 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
544 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
546 tree len
= gimple_call_arg (gc
, 2);
547 rtx r_len
= expand_normal (len
);
549 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
550 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
551 tag
, QImode
, r_len
, Pmode
);
554 /* For hwasan stack tagging:
555 Store a tag into a pointer. */
557 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
559 gcc_assert (ptr_mode
== Pmode
);
560 tree g_target
= gimple_call_lhs (gc
);
561 tree g_ptr
= gimple_call_arg (gc
, 0);
562 tree g_tag
= gimple_call_arg (gc
, 1);
564 rtx ptr
= expand_normal (g_ptr
);
565 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
566 rtx target
= expand_normal (g_target
);
568 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
569 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
570 if (tagged_value
!= target
)
571 emit_move_insn (target
, tagged_value
);
574 /* This should get expanded in the sanopt pass. */
577 expand_ASAN_CHECK (internal_fn
, gcall
*)
582 /* This should get expanded in the sanopt pass. */
585 expand_ASAN_MARK (internal_fn
, gcall
*)
590 /* This should get expanded in the sanopt pass. */
593 expand_ASAN_POISON (internal_fn
, gcall
*)
598 /* This should get expanded in the sanopt pass. */
601 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
606 /* This should get expanded in the tsan pass. */
609 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
614 /* This should get expanded in the lower pass. */
617 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
619 error_at (gimple_location (call
),
620 "invalid use of attribute %<fallthrough%>");
623 /* Return minimum precision needed to represent all values
624 of ARG in SIGNed integral type. */
627 get_min_precision (tree arg
, signop sign
)
629 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
631 signop orig_sign
= sign
;
632 if (TREE_CODE (arg
) == INTEGER_CST
)
635 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
637 widest_int w
= wi::to_widest (arg
);
638 w
= wi::ext (w
, prec
, sign
);
639 p
= wi::min_precision (w
, sign
);
642 p
= wi::min_precision (wi::to_wide (arg
), sign
);
643 return MIN (p
, prec
);
645 while (CONVERT_EXPR_P (arg
)
646 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
647 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
649 arg
= TREE_OPERAND (arg
, 0);
650 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
652 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
654 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
655 return prec
+ (orig_sign
!= sign
);
656 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
659 return prec
+ (orig_sign
!= sign
);
661 if (CONVERT_EXPR_P (arg
)
662 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
663 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
665 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
666 If y_2's min precision is smaller than prec, return that. */
667 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
669 return oprec
+ (orig_sign
!= sign
);
671 if (TREE_CODE (arg
) != SSA_NAME
)
672 return prec
+ (orig_sign
!= sign
);
673 wide_int arg_min
, arg_max
;
674 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
676 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
677 if (is_gimple_assign (g
)
678 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
680 tree t
= gimple_assign_rhs1 (g
);
681 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
682 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
685 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
687 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
689 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
690 return prec
+ (orig_sign
!= sign
);
691 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
694 return prec
+ (orig_sign
!= sign
);
698 return prec
+ (orig_sign
!= sign
);
700 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
702 int p1
= wi::min_precision (arg_min
, sign
);
703 int p2
= wi::min_precision (arg_max
, sign
);
705 prec
= MIN (prec
, p1
);
707 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
709 int p
= wi::min_precision (arg_max
, UNSIGNED
);
710 prec
= MIN (prec
, p
);
712 return prec
+ (orig_sign
!= sign
);
715 /* Helper for expand_*_overflow. Set the __imag__ part to true
716 (1 except for signed:1 type, in which case store -1). */
719 expand_arith_set_overflow (tree lhs
, rtx target
)
721 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
722 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
723 write_complex_part (target
, constm1_rtx
, true);
725 write_complex_part (target
, const1_rtx
, true);
728 /* Helper for expand_*_overflow. Store RES into the __real__ part
729 of TARGET. If RES has larger MODE than __real__ part of TARGET,
730 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
731 if LHS has smaller precision than its mode. */
734 expand_arith_overflow_result_store (tree lhs
, rtx target
,
735 scalar_int_mode mode
, rtx res
)
737 scalar_int_mode tgtmode
738 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
742 rtx_code_label
*done_label
= gen_label_rtx ();
743 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
744 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
745 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
746 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
747 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
748 profile_probability::very_likely ());
749 expand_arith_set_overflow (lhs
, target
);
750 emit_label (done_label
);
752 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
753 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
756 rtx_code_label
*done_label
= gen_label_rtx ();
757 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
762 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
764 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
765 true, OPTAB_LIB_WIDEN
);
769 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
771 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
774 do_compare_rtx_and_jump (res
, lres
,
775 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
776 profile_probability::very_likely ());
777 expand_arith_set_overflow (lhs
, target
);
778 emit_label (done_label
);
780 write_complex_part (target
, lres
, false);
783 /* Helper for expand_*_overflow. Store RES into TARGET. */
786 expand_ubsan_result_store (rtx target
, rtx res
)
788 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
789 /* If this is a scalar in a register that is stored in a wider mode
790 than the declared mode, compute the result into its declared mode
791 and then convert to the wider mode. Our value is the computed
793 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
795 emit_move_insn (target
, res
);
798 /* Add sub/add overflow checking to the statement STMT.
799 CODE says whether the operation is +, or -. */
802 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
803 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
804 bool uns1_p
, bool is_ubsan
, tree
*datap
)
806 rtx res
, target
= NULL_RTX
;
808 rtx_code_label
*done_label
= gen_label_rtx ();
809 rtx_code_label
*do_error
= gen_label_rtx ();
810 do_pending_stack_adjust ();
811 rtx op0
= expand_normal (arg0
);
812 rtx op1
= expand_normal (arg1
);
813 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
814 int prec
= GET_MODE_PRECISION (mode
);
815 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
819 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
823 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
825 write_complex_part (target
, const0_rtx
, true);
828 /* We assume both operands and result have the same precision
829 here (GET_MODE_BITSIZE (mode)), S stands for signed type
830 with that precision, U for unsigned type with that precision,
831 sgn for unsigned most significant bit in that precision.
832 s1 is signed first operand, u1 is unsigned first operand,
833 s2 is signed second operand, u2 is unsigned second operand,
834 sr is signed result, ur is unsigned result and the following
835 rules say how to compute result (which is always result of
836 the operands as if both were unsigned, cast to the right
837 signedness) and how to compute whether operation overflowed.
840 res = (S) ((U) s1 + (U) s2)
841 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
843 res = (S) ((U) s1 - (U) s2)
844 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
847 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
850 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
852 res = (S) ((U) s1 + u2)
853 ovf = ((U) res ^ sgn) < u2
858 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
860 res = (S) ((U) s1 - u2)
861 ovf = u2 > ((U) s1 ^ sgn)
864 ovf = s1 < 0 || u2 > (U) s1
867 ovf = u1 >= ((U) s2 ^ sgn)
872 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
874 res = (U) s1 + (U) s2
875 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
878 ovf = (U) res < u2 || res < 0
881 ovf = u1 >= u2 ? res < 0 : res >= 0
883 res = (U) s1 - (U) s2
884 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
886 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
888 /* PLUS_EXPR is commutative, if operand signedness differs,
889 canonicalize to the first operand being signed and second
890 unsigned to simplify following code. */
891 std::swap (op0
, op1
);
892 std::swap (arg0
, arg1
);
898 if (uns0_p
&& uns1_p
&& unsr_p
)
900 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
901 : usubv4_optab
, mode
);
902 if (icode
!= CODE_FOR_nothing
)
904 class expand_operand ops
[4];
905 rtx_insn
*last
= get_last_insn ();
907 res
= gen_reg_rtx (mode
);
908 create_output_operand (&ops
[0], res
, mode
);
909 create_input_operand (&ops
[1], op0
, mode
);
910 create_input_operand (&ops
[2], op1
, mode
);
911 create_fixed_operand (&ops
[3], do_error
);
912 if (maybe_expand_insn (icode
, 4, ops
))
914 last
= get_last_insn ();
915 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
917 && any_condjump_p (last
)
918 && !find_reg_note (last
, REG_BR_PROB
, 0))
919 add_reg_br_prob_note (last
,
920 profile_probability::very_unlikely ());
921 emit_jump (done_label
);
925 delete_insns_since (last
);
928 /* Compute the operation. On RTL level, the addition is always
930 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
931 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
933 /* For PLUS_EXPR, the operation is commutative, so we can pick
934 operand to compare against. For prec <= BITS_PER_WORD, I think
935 preferring REG operand is better over CONST_INT, because
936 the CONST_INT might enlarge the instruction or CSE would need
937 to figure out we'd already loaded it into a register before.
938 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
939 as then the multi-word comparison can be perhaps simplified. */
940 if (code
== PLUS_EXPR
941 && (prec
<= BITS_PER_WORD
942 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
943 : CONST_SCALAR_INT_P (op1
)))
945 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
946 true, mode
, NULL_RTX
, NULL
, done_label
,
947 profile_probability::very_likely ());
952 if (!uns0_p
&& uns1_p
&& !unsr_p
)
954 /* Compute the operation. On RTL level, the addition is always
956 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
957 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
958 rtx tem
= expand_binop (mode
, add_optab
,
959 code
== PLUS_EXPR
? res
: op0
, sgn
,
960 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
961 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
962 done_label
, profile_probability::very_likely ());
967 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
969 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
971 /* As we've changed op1, we have to avoid using the value range
972 for the original argument. */
973 arg1
= error_mark_node
;
979 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
981 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
983 /* As we've changed op0, we have to avoid using the value range
984 for the original argument. */
985 arg0
= error_mark_node
;
991 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
993 /* Compute the operation. On RTL level, the addition is always
995 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
997 int pos_neg
= get_range_pos_neg (arg0
);
999 /* If ARG0 is known to be always negative, this is always overflow. */
1000 emit_jump (do_error
);
1001 else if (pos_neg
== 3)
1002 /* If ARG0 is not known to be always positive, check at runtime. */
1003 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1004 NULL
, do_error
, profile_probability::very_unlikely ());
1005 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1006 done_label
, profile_probability::very_likely ());
1007 goto do_error_label
;
1011 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1013 /* Compute the operation. On RTL level, the addition is always
1015 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1017 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1019 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1020 done_label
, profile_probability::very_likely ());
1021 goto do_error_label
;
1025 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1027 /* Compute the operation. On RTL level, the addition is always
1029 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1031 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1032 NULL
, do_error
, profile_probability::very_unlikely ());
1034 /* The operation is commutative, so we can pick operand to compare
1035 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1036 is better over CONST_INT, because the CONST_INT might enlarge the
1037 instruction or CSE would need to figure out we'd already loaded it
1038 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1039 might be more beneficial, as then the multi-word comparison can be
1040 perhaps simplified. */
1041 if (prec
<= BITS_PER_WORD
1042 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1043 : CONST_SCALAR_INT_P (op0
))
1045 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1046 done_label
, profile_probability::very_likely ());
1047 goto do_error_label
;
1050 /* s1 +- s2 -> ur */
1051 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1053 /* Compute the operation. On RTL level, the addition is always
1055 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1056 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1057 int pos_neg
= get_range_pos_neg (arg1
);
1058 if (code
== PLUS_EXPR
)
1060 int pos_neg0
= get_range_pos_neg (arg0
);
1061 if (pos_neg0
!= 3 && pos_neg
== 3)
1063 std::swap (op0
, op1
);
1070 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1071 ? and_optab
: ior_optab
,
1072 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1073 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1074 NULL
, done_label
, profile_probability::very_likely ());
1078 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1079 do_compare_rtx_and_jump (op1
, const0_rtx
,
1080 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1081 NULL_RTX
, NULL
, do_ior_label
,
1082 profile_probability::even ());
1083 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1085 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1086 NULL
, done_label
, profile_probability::very_likely ());
1087 emit_jump (do_error
);
1088 emit_label (do_ior_label
);
1089 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1091 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1092 NULL
, done_label
, profile_probability::very_likely ());
1094 goto do_error_label
;
1098 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1100 /* Compute the operation. On RTL level, the addition is always
1102 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1104 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1105 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1106 op0_geu_op1
, profile_probability::even ());
1107 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1108 NULL
, done_label
, profile_probability::very_likely ());
1109 emit_jump (do_error
);
1110 emit_label (op0_geu_op1
);
1111 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1112 NULL
, done_label
, profile_probability::very_likely ());
1113 goto do_error_label
;
1116 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1118 /* s1 +- s2 -> sr */
1121 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1122 : subv4_optab
, mode
);
1123 if (icode
!= CODE_FOR_nothing
)
1125 class expand_operand ops
[4];
1126 rtx_insn
*last
= get_last_insn ();
1128 res
= gen_reg_rtx (mode
);
1129 create_output_operand (&ops
[0], res
, mode
);
1130 create_input_operand (&ops
[1], op0
, mode
);
1131 create_input_operand (&ops
[2], op1
, mode
);
1132 create_fixed_operand (&ops
[3], do_error
);
1133 if (maybe_expand_insn (icode
, 4, ops
))
1135 last
= get_last_insn ();
1136 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1138 && any_condjump_p (last
)
1139 && !find_reg_note (last
, REG_BR_PROB
, 0))
1140 add_reg_br_prob_note (last
,
1141 profile_probability::very_unlikely ());
1142 emit_jump (done_label
);
1143 goto do_error_label
;
1146 delete_insns_since (last
);
1149 /* Compute the operation. On RTL level, the addition is always
1151 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1152 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1154 /* If we can prove that one of the arguments (for MINUS_EXPR only
1155 the second operand, as subtraction is not commutative) is always
1156 non-negative or always negative, we can do just one comparison
1157 and conditional jump. */
1158 int pos_neg
= get_range_pos_neg (arg1
);
1159 if (code
== PLUS_EXPR
)
1161 int pos_neg0
= get_range_pos_neg (arg0
);
1162 if (pos_neg0
!= 3 && pos_neg
== 3)
1164 std::swap (op0
, op1
);
1169 /* Addition overflows if and only if the two operands have the same sign,
1170 and the result has the opposite sign. Subtraction overflows if and
1171 only if the two operands have opposite sign, and the subtrahend has
1172 the same sign as the result. Here 0 is counted as positive. */
1175 /* Compute op0 ^ op1 (operands have opposite sign). */
1176 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1179 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1180 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1184 if (code
== PLUS_EXPR
)
1186 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1187 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1188 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1193 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1194 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1195 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1199 /* No overflow if the result has bit sign cleared. */
1200 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1201 NULL
, done_label
, profile_probability::very_likely ());
1204 /* Compare the result of the operation with the first operand.
1205 No overflow for addition if second operand is positive and result
1206 is larger or second operand is negative and result is smaller.
1207 Likewise for subtraction with sign of second operand flipped. */
1209 do_compare_rtx_and_jump (res
, op0
,
1210 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1211 false, mode
, NULL_RTX
, NULL
, done_label
,
1212 profile_probability::very_likely ());
1216 emit_label (do_error
);
1219 /* Expand the ubsan builtin call. */
1221 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1225 do_pending_stack_adjust ();
1228 expand_arith_set_overflow (lhs
, target
);
1231 emit_label (done_label
);
1236 expand_ubsan_result_store (target
, res
);
1240 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1243 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1248 /* Add negate overflow checking to the statement STMT. */
1251 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1256 rtx_code_label
*done_label
, *do_error
;
1257 rtx target
= NULL_RTX
;
1259 done_label
= gen_label_rtx ();
1260 do_error
= gen_label_rtx ();
1262 do_pending_stack_adjust ();
1263 op1
= expand_normal (arg1
);
1265 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1268 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1270 write_complex_part (target
, const0_rtx
, true);
1273 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1274 if (icode
!= CODE_FOR_nothing
)
1276 class expand_operand ops
[3];
1277 rtx_insn
*last
= get_last_insn ();
1279 res
= gen_reg_rtx (mode
);
1280 create_output_operand (&ops
[0], res
, mode
);
1281 create_input_operand (&ops
[1], op1
, mode
);
1282 create_fixed_operand (&ops
[2], do_error
);
1283 if (maybe_expand_insn (icode
, 3, ops
))
1285 last
= get_last_insn ();
1286 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1288 && any_condjump_p (last
)
1289 && !find_reg_note (last
, REG_BR_PROB
, 0))
1290 add_reg_br_prob_note (last
,
1291 profile_probability::very_unlikely ());
1292 emit_jump (done_label
);
1296 delete_insns_since (last
);
1297 icode
= CODE_FOR_nothing
;
1301 if (icode
== CODE_FOR_nothing
)
1303 /* Compute the operation. On RTL level, the addition is always
1305 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1307 /* Compare the operand with the most negative value. */
1308 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1309 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1310 done_label
, profile_probability::very_likely ());
1313 emit_label (do_error
);
1316 /* Expand the ubsan builtin call. */
1318 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1319 arg1
, NULL_TREE
, datap
);
1322 do_pending_stack_adjust ();
1325 expand_arith_set_overflow (lhs
, target
);
1328 emit_label (done_label
);
1333 expand_ubsan_result_store (target
, res
);
1335 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1339 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1340 mode MODE can be expanded without using a libcall. */
1343 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1344 rtx op0
, rtx op1
, bool uns
)
1346 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1347 != CODE_FOR_nothing
)
1350 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1351 != CODE_FOR_nothing
)
1354 rtx_insn
*last
= get_last_insn ();
1355 if (CONSTANT_P (op0
))
1356 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1358 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1359 if (CONSTANT_P (op1
))
1360 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1362 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1363 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1364 delete_insns_since (last
);
1365 return ret
!= NULL_RTX
;
1368 /* Add mul overflow checking to the statement STMT. */
1371 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1372 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1377 rtx_code_label
*done_label
, *do_error
;
1378 rtx target
= NULL_RTX
;
1380 enum insn_code icode
;
1382 done_label
= gen_label_rtx ();
1383 do_error
= gen_label_rtx ();
1385 do_pending_stack_adjust ();
1386 op0
= expand_normal (arg0
);
1387 op1
= expand_normal (arg1
);
1389 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1393 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1395 write_complex_part (target
, const0_rtx
, true);
1399 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1401 /* We assume both operands and result have the same precision
1402 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1403 with that precision, U for unsigned type with that precision,
1404 sgn for unsigned most significant bit in that precision.
1405 s1 is signed first operand, u1 is unsigned first operand,
1406 s2 is signed second operand, u2 is unsigned second operand,
1407 sr is signed result, ur is unsigned result and the following
1408 rules say how to compute result (which is always result of
1409 the operands as if both were unsigned, cast to the right
1410 signedness) and how to compute whether operation overflowed.
1411 main_ovf (false) stands for jump on signed multiplication
1412 overflow or the main algorithm with uns == false.
1413 main_ovf (true) stands for jump on unsigned multiplication
1414 overflow or the main algorithm with uns == true.
1417 res = (S) ((U) s1 * (U) s2)
1418 ovf = main_ovf (false)
1421 ovf = main_ovf (true)
1424 ovf = (s1 < 0 && u2) || main_ovf (true)
1427 ovf = res < 0 || main_ovf (true)
1429 res = (S) ((U) s1 * u2)
1430 ovf = (S) u2 >= 0 ? main_ovf (false)
1431 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1433 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1434 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1436 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1438 if (uns0_p
&& !uns1_p
)
1440 /* Multiplication is commutative, if operand signedness differs,
1441 canonicalize to the first operand being signed and second
1442 unsigned to simplify following code. */
1443 std::swap (op0
, op1
);
1444 std::swap (arg0
, arg1
);
1449 int pos_neg0
= get_range_pos_neg (arg0
);
1450 int pos_neg1
= get_range_pos_neg (arg1
);
1453 if (!uns0_p
&& uns1_p
&& unsr_p
)
1458 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1461 /* If s1 is negative, avoid the main code, just multiply and
1462 signal overflow if op1 is not 0. */
1463 struct separate_ops ops
;
1464 ops
.code
= MULT_EXPR
;
1465 ops
.type
= TREE_TYPE (arg1
);
1466 ops
.op0
= make_tree (ops
.type
, op0
);
1467 ops
.op1
= make_tree (ops
.type
, op1
);
1468 ops
.op2
= NULL_TREE
;
1470 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1471 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1472 NULL
, done_label
, profile_probability::very_likely ());
1473 goto do_error_label
;
1475 if (get_min_precision (arg1
, UNSIGNED
)
1476 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1478 /* If the first operand is sign extended from narrower type, the
1479 second operand is zero extended from narrower type and
1480 the sum of the two precisions is smaller or equal to the
1481 result precision: if the first argument is at runtime
1482 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1483 and there will be no overflow, if the first argument is
1484 negative and the second argument zero, the result will be
1485 0 and there will be no overflow, if the first argument is
1486 negative and the second argument positive, the result when
1487 treated as signed will be negative (minimum -0x7f80 or
1488 -0x7f..f80..0) there there will be always overflow. So, do
1490 ovf = (S) res < 0 */
1491 struct separate_ops ops
;
1492 ops
.code
= MULT_EXPR
;
1494 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1496 ops
.op0
= make_tree (ops
.type
, op0
);
1497 ops
.op1
= make_tree (ops
.type
, op1
);
1498 ops
.op2
= NULL_TREE
;
1500 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1501 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1502 mode
, NULL_RTX
, NULL
, done_label
,
1503 profile_probability::very_likely ());
1504 goto do_error_label
;
1506 rtx_code_label
*do_main_label
;
1507 do_main_label
= gen_label_rtx ();
1508 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1509 NULL
, do_main_label
, profile_probability::very_likely ());
1510 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1511 NULL
, do_main_label
, profile_probability::very_likely ());
1512 expand_arith_set_overflow (lhs
, target
);
1513 emit_label (do_main_label
);
1521 if (uns0_p
&& uns1_p
&& !unsr_p
)
1523 if ((pos_neg0
| pos_neg1
) == 1)
1525 /* If both arguments are zero extended from narrower types,
1526 the MSB will be clear on both and so we can pretend it is
1527 a normal s1 * s2 -> sr multiplication. */
1533 /* Rest of handling of this case after res is computed. */
1538 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1545 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1546 avoid the main code, just multiply and signal overflow
1547 unless 0 * u2 or -1 * ((U) Smin). */
1548 struct separate_ops ops
;
1549 ops
.code
= MULT_EXPR
;
1550 ops
.type
= TREE_TYPE (arg1
);
1551 ops
.op0
= make_tree (ops
.type
, op0
);
1552 ops
.op1
= make_tree (ops
.type
, op1
);
1553 ops
.op2
= NULL_TREE
;
1555 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1556 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1557 NULL
, done_label
, profile_probability::very_likely ());
1558 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1559 NULL
, do_error
, profile_probability::very_unlikely ());
1561 prec
= GET_MODE_PRECISION (mode
);
1563 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1564 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1565 NULL
, done_label
, profile_probability::very_likely ());
1566 goto do_error_label
;
1568 /* Rest of handling of this case after res is computed. */
1576 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1579 switch (pos_neg0
| pos_neg1
)
1581 case 1: /* Both operands known to be non-negative. */
1583 case 2: /* Both operands known to be negative. */
1584 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1585 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1586 /* Avoid looking at arg0/arg1 ranges, as we've changed
1588 arg0
= error_mark_node
;
1589 arg1
= error_mark_node
;
1592 if ((pos_neg0
^ pos_neg1
) == 3)
1594 /* If one operand is known to be negative and the other
1595 non-negative, this overflows always, unless the non-negative
1596 one is 0. Just do normal multiply and set overflow
1597 unless one of the operands is 0. */
1598 struct separate_ops ops
;
1599 ops
.code
= MULT_EXPR
;
1601 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1603 ops
.op0
= make_tree (ops
.type
, op0
);
1604 ops
.op1
= make_tree (ops
.type
, op1
);
1605 ops
.op2
= NULL_TREE
;
1607 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1608 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1609 true, mode
, NULL_RTX
, NULL
, done_label
,
1610 profile_probability::very_likely ());
1611 goto do_error_label
;
1613 if (get_min_precision (arg0
, SIGNED
)
1614 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1616 /* If both operands are sign extended from narrower types and
1617 the sum of the two precisions is smaller or equal to the
1618 result precision: if both arguments are at runtime
1619 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1620 and there will be no overflow, if both arguments are negative,
1621 maximum result will be 0x40..00 and there will be no overflow
1622 either, if one argument is positive and the other argument
1623 negative, the result when treated as signed will be negative
1624 and there will be always overflow, and if one argument is
1625 zero and the other negative the result will be zero and no
1628 ovf = (S) res < 0 */
1629 struct separate_ops ops
;
1630 ops
.code
= MULT_EXPR
;
1632 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1634 ops
.op0
= make_tree (ops
.type
, op0
);
1635 ops
.op1
= make_tree (ops
.type
, op1
);
1636 ops
.op2
= NULL_TREE
;
1638 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1639 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1640 mode
, NULL_RTX
, NULL
, done_label
,
1641 profile_probability::very_likely ());
1642 goto do_error_label
;
1644 /* The general case, do all the needed comparisons at runtime. */
1645 rtx_code_label
*do_main_label
, *after_negate_label
;
1647 rop0
= gen_reg_rtx (mode
);
1648 rop1
= gen_reg_rtx (mode
);
1649 emit_move_insn (rop0
, op0
);
1650 emit_move_insn (rop1
, op1
);
1653 do_main_label
= gen_label_rtx ();
1654 after_negate_label
= gen_label_rtx ();
1655 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1657 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1658 NULL
, after_negate_label
, profile_probability::very_likely ());
1659 /* Both arguments negative here, negate them and continue with
1660 normal unsigned overflow checking multiplication. */
1661 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1663 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1665 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1667 arg0
= error_mark_node
;
1668 arg1
= error_mark_node
;
1669 emit_jump (do_main_label
);
1670 emit_label (after_negate_label
);
1671 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1673 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1674 NULL
, do_main_label
,
1675 profile_probability::very_likely ());
1676 /* One argument is negative here, the other positive. This
1677 overflows always, unless one of the arguments is 0. But
1678 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1679 is, thus we can keep do_main code oring in overflow as is. */
1681 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1682 NULL
, do_main_label
,
1683 profile_probability::very_unlikely ());
1685 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1686 NULL
, do_main_label
,
1687 profile_probability::very_unlikely ());
1688 expand_arith_set_overflow (lhs
, target
);
1689 emit_label (do_main_label
);
1697 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1698 sign
= uns
? UNSIGNED
: SIGNED
;
1699 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1701 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1702 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1704 /* Optimize unsigned multiplication by power of 2 constant
1705 using 2 shifts, one for result, one to extract the shifted
1706 out bits to see if they are all zero.
1707 Don't do this if optimizing for size and we have umulv4_optab,
1708 in that case assume multiplication will be shorter.
1709 This is heuristics based on the single target that provides
1710 umulv4 right now (i?86/x86_64), if further targets add it, this
1711 might need to be revisited.
1712 Cases where both operands are constant should be folded already
1713 during GIMPLE, and cases where one operand is constant but not
1714 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1715 below can be done without multiplication, just by shifts and adds,
1716 or we'd need to divide the result (and hope it actually doesn't
1717 really divide nor multiply) and compare the result of the division
1718 with the original operand. */
1723 if (integer_pow2p (arg0
))
1725 std::swap (opn0
, opn1
);
1726 std::swap (argn0
, argn1
);
1728 int cnt
= tree_log2 (argn1
);
1729 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1731 rtx upper
= const0_rtx
;
1732 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1734 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1735 GET_MODE_PRECISION (mode
) - cnt
,
1737 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1738 NULL_RTX
, NULL
, done_label
,
1739 profile_probability::very_likely ());
1740 goto do_error_label
;
1743 if (icode
!= CODE_FOR_nothing
)
1745 class expand_operand ops
[4];
1746 rtx_insn
*last
= get_last_insn ();
1748 res
= gen_reg_rtx (mode
);
1749 create_output_operand (&ops
[0], res
, mode
);
1750 create_input_operand (&ops
[1], op0
, mode
);
1751 create_input_operand (&ops
[2], op1
, mode
);
1752 create_fixed_operand (&ops
[3], do_error
);
1753 if (maybe_expand_insn (icode
, 4, ops
))
1755 last
= get_last_insn ();
1756 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1758 && any_condjump_p (last
)
1759 && !find_reg_note (last
, REG_BR_PROB
, 0))
1760 add_reg_br_prob_note (last
,
1761 profile_probability::very_unlikely ());
1762 emit_jump (done_label
);
1766 delete_insns_since (last
);
1767 icode
= CODE_FOR_nothing
;
1771 if (icode
== CODE_FOR_nothing
)
1773 struct separate_ops ops
;
1774 int prec
= GET_MODE_PRECISION (mode
);
1775 scalar_int_mode hmode
, wmode
;
1776 ops
.op0
= make_tree (type
, op0
);
1777 ops
.op1
= make_tree (type
, op1
);
1778 ops
.op2
= NULL_TREE
;
1781 /* Optimize unsigned overflow check where we don't use the
1782 multiplication result, just whether overflow happened.
1783 If we can do MULT_HIGHPART_EXPR, that followed by
1784 comparison of the result against zero is cheapest.
1785 We'll still compute res, but it should be DCEd later. */
1791 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1792 && can_mult_highpart_p (mode
, uns
) == 1
1793 && single_imm_use (lhs
, &use
, &use_stmt
)
1794 && is_gimple_assign (use_stmt
)
1795 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1798 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1799 && targetm
.scalar_mode_supported_p (wmode
)
1800 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1803 ops
.code
= WIDEN_MULT_EXPR
;
1805 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1807 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1808 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1810 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1811 res
= convert_modes (mode
, wmode
, res
, uns
);
1813 /* For the unsigned multiplication, there was overflow if
1814 HIPART is non-zero. */
1815 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1816 NULL_RTX
, NULL
, done_label
,
1817 profile_probability::very_likely ());
1820 /* RES is used more than once, place it in a pseudo. */
1821 res
= force_reg (mode
, res
);
1823 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1825 /* RES is low half of the double width result, HIPART
1826 the high half. There was overflow if
1827 HIPART is different from RES < 0 ? -1 : 0. */
1828 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1829 NULL_RTX
, NULL
, done_label
,
1830 profile_probability::very_likely ());
1833 else if (can_mult_highpart_p (mode
, uns
) == 1)
1836 ops
.code
= MULT_HIGHPART_EXPR
;
1839 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1841 ops
.code
= MULT_EXPR
;
1842 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1844 /* For the unsigned multiplication, there was overflow if
1845 HIPART is non-zero. */
1846 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1847 NULL_RTX
, NULL
, done_label
,
1848 profile_probability::very_likely ());
1851 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1853 /* RES is low half of the double width result, HIPART
1854 the high half. There was overflow if
1855 HIPART is different from RES < 0 ? -1 : 0. */
1856 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1857 NULL_RTX
, NULL
, done_label
,
1858 profile_probability::very_likely ());
1862 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1863 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1865 rtx_code_label
*large_op0
= gen_label_rtx ();
1866 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1867 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1868 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1869 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1870 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1871 rtx_code_label
*do_overflow
= gen_label_rtx ();
1872 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1874 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1875 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1877 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1878 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1879 rtx signbit0
= const0_rtx
;
1881 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1883 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1885 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1886 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1887 rtx signbit1
= const0_rtx
;
1889 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1892 res
= gen_reg_rtx (mode
);
1894 /* True if op0 resp. op1 are known to be in the range of
1896 bool op0_small_p
= false;
1897 bool op1_small_p
= false;
1898 /* True if op0 resp. op1 are known to have all zeros or all ones
1899 in the upper half of bits, but are not known to be
1901 bool op0_medium_p
= false;
1902 bool op1_medium_p
= false;
1903 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1904 nonnegative, 1 if unknown. */
1910 else if (pos_neg0
== 2)
1914 else if (pos_neg1
== 2)
1917 unsigned int mprec0
= prec
;
1918 if (arg0
!= error_mark_node
)
1919 mprec0
= get_min_precision (arg0
, sign
);
1920 if (mprec0
<= hprec
)
1922 else if (!uns
&& mprec0
<= hprec
+ 1)
1923 op0_medium_p
= true;
1924 unsigned int mprec1
= prec
;
1925 if (arg1
!= error_mark_node
)
1926 mprec1
= get_min_precision (arg1
, sign
);
1927 if (mprec1
<= hprec
)
1929 else if (!uns
&& mprec1
<= hprec
+ 1)
1930 op1_medium_p
= true;
1932 int smaller_sign
= 1;
1933 int larger_sign
= 1;
1936 smaller_sign
= op0_sign
;
1937 larger_sign
= op1_sign
;
1939 else if (op1_small_p
)
1941 smaller_sign
= op1_sign
;
1942 larger_sign
= op0_sign
;
1944 else if (op0_sign
== op1_sign
)
1946 smaller_sign
= op0_sign
;
1947 larger_sign
= op0_sign
;
1951 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1952 NULL_RTX
, NULL
, large_op0
,
1953 profile_probability::unlikely ());
1956 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1957 NULL_RTX
, NULL
, small_op0_large_op1
,
1958 profile_probability::unlikely ());
1960 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1961 hmode to mode, the multiplication will never overflow. We can
1962 do just one hmode x hmode => mode widening multiplication. */
1963 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1964 ops
.op0
= make_tree (halfstype
, lopart0
);
1965 ops
.op1
= make_tree (halfstype
, lopart1
);
1966 ops
.code
= WIDEN_MULT_EXPR
;
1969 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1970 emit_move_insn (res
, thisres
);
1971 emit_jump (done_label
);
1973 emit_label (small_op0_large_op1
);
1975 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1976 but op1 is not, just swap the arguments and handle it as op1
1977 sign/zero extended, op0 not. */
1978 rtx larger
= gen_reg_rtx (mode
);
1979 rtx hipart
= gen_reg_rtx (hmode
);
1980 rtx lopart
= gen_reg_rtx (hmode
);
1981 emit_move_insn (larger
, op1
);
1982 emit_move_insn (hipart
, hipart1
);
1983 emit_move_insn (lopart
, lopart0
);
1984 emit_jump (one_small_one_large
);
1986 emit_label (large_op0
);
1989 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1990 NULL_RTX
, NULL
, both_ops_large
,
1991 profile_probability::unlikely ());
1993 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1994 but op0 is not, prepare larger, hipart and lopart pseudos and
1995 handle it together with small_op0_large_op1. */
1996 emit_move_insn (larger
, op0
);
1997 emit_move_insn (hipart
, hipart0
);
1998 emit_move_insn (lopart
, lopart1
);
2000 emit_label (one_small_one_large
);
2002 /* lopart is the low part of the operand that is sign extended
2003 to mode, larger is the other operand, hipart is the
2004 high part of larger and lopart0 and lopart1 are the low parts
2006 We perform lopart0 * lopart1 and lopart * hipart widening
2008 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2009 ops
.op0
= make_tree (halfutype
, lopart0
);
2010 ops
.op1
= make_tree (halfutype
, lopart1
);
2012 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2014 ops
.op0
= make_tree (halfutype
, lopart
);
2015 ops
.op1
= make_tree (halfutype
, hipart
);
2016 rtx loxhi
= gen_reg_rtx (mode
);
2017 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2018 emit_move_insn (loxhi
, tem
);
2022 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2023 if (larger_sign
== 0)
2024 emit_jump (after_hipart_neg
);
2025 else if (larger_sign
!= -1)
2026 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2027 NULL_RTX
, NULL
, after_hipart_neg
,
2028 profile_probability::even ());
2030 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2031 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2032 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2034 emit_move_insn (loxhi
, tem
);
2036 emit_label (after_hipart_neg
);
2038 /* if (lopart < 0) loxhi -= larger; */
2039 if (smaller_sign
== 0)
2040 emit_jump (after_lopart_neg
);
2041 else if (smaller_sign
!= -1)
2042 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2043 NULL_RTX
, NULL
, after_lopart_neg
,
2044 profile_probability::even ());
2046 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2048 emit_move_insn (loxhi
, tem
);
2050 emit_label (after_lopart_neg
);
2053 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2054 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2055 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2057 emit_move_insn (loxhi
, tem
);
2059 /* if (loxhi >> (bitsize / 2)
2060 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2061 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2062 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2064 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2065 rtx signbitloxhi
= const0_rtx
;
2067 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2068 convert_modes (hmode
, mode
,
2070 hprec
- 1, NULL_RTX
, 0);
2072 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2073 NULL_RTX
, NULL
, do_overflow
,
2074 profile_probability::very_unlikely ());
2076 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2077 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2079 tem
= convert_modes (mode
, hmode
,
2080 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2082 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2085 emit_move_insn (res
, tem
);
2086 emit_jump (done_label
);
2088 emit_label (both_ops_large
);
2090 /* If both operands are large (not sign (!uns) or zero (uns)
2091 extended from hmode), then perform the full multiplication
2092 which will be the result of the operation.
2093 The only cases which don't overflow are for signed multiplication
2094 some cases where both hipart0 and highpart1 are 0 or -1.
2095 For unsigned multiplication when high parts are both non-zero
2096 this overflows always. */
2097 ops
.code
= MULT_EXPR
;
2098 ops
.op0
= make_tree (type
, op0
);
2099 ops
.op1
= make_tree (type
, op1
);
2100 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2101 emit_move_insn (res
, tem
);
2107 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2108 NULL_RTX
, 1, OPTAB_WIDEN
);
2109 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2110 NULL_RTX
, NULL
, do_error
,
2111 profile_probability::very_unlikely ());
2116 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2117 NULL_RTX
, 1, OPTAB_WIDEN
);
2118 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2119 NULL_RTX
, NULL
, do_error
,
2120 profile_probability::very_unlikely ());
2123 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2124 the same, overflow happened if res is non-positive, if they
2125 are different, overflow happened if res is positive. */
2126 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2127 emit_jump (hipart_different
);
2128 else if (op0_sign
== 1 || op1_sign
== 1)
2129 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2130 NULL_RTX
, NULL
, hipart_different
,
2131 profile_probability::even ());
2133 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2134 NULL_RTX
, NULL
, do_error
,
2135 profile_probability::very_unlikely ());
2136 emit_jump (done_label
);
2138 emit_label (hipart_different
);
2140 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2141 NULL_RTX
, NULL
, do_error
,
2142 profile_probability::very_unlikely ());
2143 emit_jump (done_label
);
2146 emit_label (do_overflow
);
2148 /* Overflow, do full multiplication and fallthru into do_error. */
2149 ops
.op0
= make_tree (type
, op0
);
2150 ops
.op1
= make_tree (type
, op1
);
2151 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2152 emit_move_insn (res
, tem
);
2154 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2155 && targetm
.scalar_mode_supported_p (wmode
))
2156 /* Even emitting a libcall is better than not detecting overflow
2161 gcc_assert (!is_ubsan
);
2162 ops
.code
= MULT_EXPR
;
2164 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2165 emit_jump (done_label
);
2170 emit_label (do_error
);
2173 /* Expand the ubsan builtin call. */
2175 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2179 do_pending_stack_adjust ();
2182 expand_arith_set_overflow (lhs
, target
);
2185 emit_label (done_label
);
2188 if (uns0_p
&& uns1_p
&& !unsr_p
)
2190 rtx_code_label
*all_done_label
= gen_label_rtx ();
2191 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2192 NULL
, all_done_label
, profile_probability::very_likely ());
2193 expand_arith_set_overflow (lhs
, target
);
2194 emit_label (all_done_label
);
2198 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2200 rtx_code_label
*all_done_label
= gen_label_rtx ();
2201 rtx_code_label
*set_noovf
= gen_label_rtx ();
2202 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2203 NULL
, all_done_label
, profile_probability::very_likely ());
2204 expand_arith_set_overflow (lhs
, target
);
2205 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2206 NULL
, set_noovf
, profile_probability::very_likely ());
2207 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2208 NULL
, all_done_label
, profile_probability::very_unlikely ());
2209 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2210 all_done_label
, profile_probability::very_unlikely ());
2211 emit_label (set_noovf
);
2212 write_complex_part (target
, const0_rtx
, true);
2213 emit_label (all_done_label
);
2219 expand_ubsan_result_store (target
, res
);
2221 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2225 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2228 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2229 tree arg0
, tree arg1
)
2231 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2232 rtx_code_label
*loop_lab
= NULL
;
2233 rtx cntvar
= NULL_RTX
;
2234 tree cntv
= NULL_TREE
;
2235 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2236 tree sz
= TYPE_SIZE (eltype
);
2237 tree data
= NULL_TREE
;
2238 tree resv
= NULL_TREE
;
2239 rtx lhsr
= NULL_RTX
;
2240 rtx resvr
= NULL_RTX
;
2241 unsigned HOST_WIDE_INT const_cnt
= 0;
2242 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2247 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2248 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2249 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2250 optab_default
)) == unknown_optab
2251 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2252 == CODE_FOR_nothing
))
2255 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2258 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2259 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2265 do_pending_stack_adjust ();
2266 loop_lab
= gen_label_rtx ();
2267 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2268 cntv
= make_tree (sizetype
, cntvar
);
2269 emit_move_insn (cntvar
, const0_rtx
);
2270 emit_label (loop_lab
);
2272 if (TREE_CODE (arg0
) != VECTOR_CST
)
2274 rtx arg0r
= expand_normal (arg0
);
2275 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2277 if (TREE_CODE (arg1
) != VECTOR_CST
)
2279 rtx arg1r
= expand_normal (arg1
);
2280 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2282 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2284 tree op0
, op1
, res
= NULL_TREE
;
2287 tree atype
= build_array_type_nelts (eltype
, cnt
);
2288 op0
= uniform_vector_p (arg0
);
2289 if (op0
== NULL_TREE
)
2291 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2292 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2293 NULL_TREE
, NULL_TREE
);
2295 op1
= uniform_vector_p (arg1
);
2296 if (op1
== NULL_TREE
)
2298 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2299 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2300 NULL_TREE
, NULL_TREE
);
2304 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2305 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2306 NULL_TREE
, NULL_TREE
);
2311 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2312 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2313 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2315 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2321 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2322 false, false, false, true, &data
);
2325 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2326 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2328 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2329 false, false, false, true, &data
);
2332 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2341 struct separate_ops ops
;
2342 ops
.code
= PLUS_EXPR
;
2343 ops
.type
= TREE_TYPE (cntv
);
2345 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2346 ops
.op2
= NULL_TREE
;
2348 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2351 emit_move_insn (cntvar
, ret
);
2352 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2353 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2354 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2355 profile_probability::very_likely ());
2357 if (lhs
&& resv
== NULL_TREE
)
2359 struct separate_ops ops
;
2361 ops
.type
= TREE_TYPE (arg0
);
2364 ops
.op2
= NULL_TREE
;
2366 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2369 emit_move_insn (lhsr
, ret
);
2372 emit_move_insn (lhsr
, resvr
);
2375 /* Expand UBSAN_CHECK_ADD call STMT. */
2378 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2380 location_t loc
= gimple_location (stmt
);
2381 tree lhs
= gimple_call_lhs (stmt
);
2382 tree arg0
= gimple_call_arg (stmt
, 0);
2383 tree arg1
= gimple_call_arg (stmt
, 1);
2384 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2385 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2387 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2388 false, false, false, true, NULL
);
2391 /* Expand UBSAN_CHECK_SUB call STMT. */
2394 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2396 location_t loc
= gimple_location (stmt
);
2397 tree lhs
= gimple_call_lhs (stmt
);
2398 tree arg0
= gimple_call_arg (stmt
, 0);
2399 tree arg1
= gimple_call_arg (stmt
, 1);
2400 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2401 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2402 else if (integer_zerop (arg0
))
2403 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2405 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2406 false, false, false, true, NULL
);
2409 /* Expand UBSAN_CHECK_MUL call STMT. */
2412 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2414 location_t loc
= gimple_location (stmt
);
2415 tree lhs
= gimple_call_lhs (stmt
);
2416 tree arg0
= gimple_call_arg (stmt
, 0);
2417 tree arg1
= gimple_call_arg (stmt
, 1);
2418 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2419 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2421 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2425 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2428 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2430 tree lhs
= gimple_call_lhs (stmt
);
2431 if (lhs
== NULL_TREE
)
2433 tree arg0
= gimple_call_arg (stmt
, 0);
2434 tree arg1
= gimple_call_arg (stmt
, 1);
2435 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2436 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2437 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2438 int unsr_p
= TYPE_UNSIGNED (type
);
2439 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2440 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2441 int precres
= TYPE_PRECISION (type
);
2442 location_t loc
= gimple_location (stmt
);
2443 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2445 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2447 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2448 prec0
= MIN (prec0
, pr
);
2449 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2450 prec1
= MIN (prec1
, pr
);
2452 /* If uns0_p && uns1_p, precop is minimum needed precision
2453 of unsigned type to hold the exact result, otherwise
2454 precop is minimum needed precision of signed type to
2455 hold the exact result. */
2457 if (code
== MULT_EXPR
)
2458 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2461 if (uns0_p
== uns1_p
)
2462 precop
= MAX (prec0
, prec1
) + 1;
2464 precop
= MAX (prec0
+ 1, prec1
) + 1;
2466 precop
= MAX (prec0
, prec1
+ 1) + 1;
2468 int orig_precres
= precres
;
2472 if ((uns0_p
&& uns1_p
)
2473 ? ((precop
+ !unsr_p
) <= precres
2474 /* u1 - u2 -> ur can overflow, no matter what precision
2476 && (code
!= MINUS_EXPR
|| !unsr_p
))
2477 : (!unsr_p
&& precop
<= precres
))
2479 /* The infinity precision result will always fit into result. */
2480 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2481 write_complex_part (target
, const0_rtx
, true);
2482 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2483 struct separate_ops ops
;
2486 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2487 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2488 ops
.op2
= NULL_TREE
;
2490 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2491 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2495 /* For operations with low precision, if target doesn't have them, start
2496 with precres widening right away, otherwise do it only if the most
2497 simple cases can't be used. */
2498 const int min_precision
= targetm
.min_arithmetic_precision ();
2499 if (orig_precres
== precres
&& precres
< min_precision
)
2501 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2502 && prec1
<= precres
)
2503 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2504 && prec0
+ uns0_p
<= precres
2505 && prec1
+ uns1_p
<= precres
))
2507 arg0
= fold_convert_loc (loc
, type
, arg0
);
2508 arg1
= fold_convert_loc (loc
, type
, arg1
);
2512 if (integer_zerop (arg0
) && !unsr_p
)
2514 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2519 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2520 unsr_p
, unsr_p
, false, NULL
);
2523 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2524 unsr_p
, unsr_p
, false, NULL
);
2531 /* For sub-word operations, retry with a wider type first. */
2532 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2534 int p
= MAX (min_precision
, precop
);
2535 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2536 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2539 p
= TYPE_PRECISION (optype
);
2543 unsr_p
= TYPE_UNSIGNED (optype
);
2549 if (prec0
<= precres
&& prec1
<= precres
)
2554 types
[0] = build_nonstandard_integer_type (precres
, 0);
2560 types
[1] = build_nonstandard_integer_type (precres
, 1);
2562 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2563 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2564 if (code
!= MULT_EXPR
)
2565 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2566 uns0_p
, uns1_p
, false, NULL
);
2568 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2569 uns0_p
, uns1_p
, false, NULL
);
2573 /* Retry with a wider type. */
2574 if (orig_precres
== precres
)
2576 int p
= MAX (prec0
, prec1
);
2577 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2578 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2581 p
= TYPE_PRECISION (optype
);
2585 unsr_p
= TYPE_UNSIGNED (optype
);
2596 /* Expand ADD_OVERFLOW STMT. */
2599 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2601 expand_arith_overflow (PLUS_EXPR
, stmt
);
2604 /* Expand SUB_OVERFLOW STMT. */
2607 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2609 expand_arith_overflow (MINUS_EXPR
, stmt
);
2612 /* Expand MUL_OVERFLOW STMT. */
2615 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2617 expand_arith_overflow (MULT_EXPR
, stmt
);
2620 /* This should get folded in tree-vectorizer.c. */
2623 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2628 /* This should get folded in tree-vectorizer.c. */
2631 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2636 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2637 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2640 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2642 tree addr
= gimple_call_arg (stmt
, index
);
2643 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2644 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2645 if (TYPE_ALIGN (type
) != align
)
2646 type
= build_aligned_type (type
, align
);
2649 if (TREE_CODE (tmp
) == SSA_NAME
)
2651 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2652 if (gimple_assign_single_p (def
))
2653 tmp
= gimple_assign_rhs1 (def
);
2656 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2658 tree mem
= TREE_OPERAND (tmp
, 0);
2659 if (TREE_CODE (mem
) == TARGET_MEM_REF
2660 && types_compatible_p (TREE_TYPE (mem
), type
))
2662 tree offset
= TMR_OFFSET (mem
);
2663 if (type
!= TREE_TYPE (mem
)
2664 || alias_ptr_type
!= TREE_TYPE (offset
)
2665 || !integer_zerop (offset
))
2667 mem
= copy_node (mem
);
2668 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2669 wi::to_poly_wide (offset
));
2670 TREE_TYPE (mem
) = type
;
2676 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2679 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB. */
2682 expand_partial_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2684 class expand_operand ops
[3];
2685 tree type
, lhs
, rhs
, maskt
;
2686 rtx mem
, target
, mask
;
2689 maskt
= gimple_call_arg (stmt
, 2);
2690 lhs
= gimple_call_lhs (stmt
);
2691 if (lhs
== NULL_TREE
)
2693 type
= TREE_TYPE (lhs
);
2694 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2696 if (optab
== vec_mask_load_lanes_optab
)
2697 icode
= get_multi_vector_move (type
, optab
);
2698 else if (optab
== len_load_optab
)
2699 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2701 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2702 TYPE_MODE (TREE_TYPE (maskt
)));
2704 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2705 gcc_assert (MEM_P (mem
));
2706 mask
= expand_normal (maskt
);
2707 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2708 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2709 create_fixed_operand (&ops
[1], mem
);
2710 if (optab
== len_load_optab
)
2711 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2712 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2714 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2715 expand_insn (icode
, 3, ops
);
2716 if (!rtx_equal_p (target
, ops
[0].value
))
2717 emit_move_insn (target
, ops
[0].value
);
2720 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2721 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2722 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2724 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB. */
2727 expand_partial_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2729 class expand_operand ops
[3];
2730 tree type
, lhs
, rhs
, maskt
;
2734 maskt
= gimple_call_arg (stmt
, 2);
2735 rhs
= gimple_call_arg (stmt
, 3);
2736 type
= TREE_TYPE (rhs
);
2737 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2739 if (optab
== vec_mask_store_lanes_optab
)
2740 icode
= get_multi_vector_move (type
, optab
);
2741 else if (optab
== len_store_optab
)
2742 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2744 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2745 TYPE_MODE (TREE_TYPE (maskt
)));
2747 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2748 gcc_assert (MEM_P (mem
));
2749 mask
= expand_normal (maskt
);
2750 reg
= expand_normal (rhs
);
2751 create_fixed_operand (&ops
[0], mem
);
2752 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2753 if (optab
== len_store_optab
)
2754 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2755 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2757 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2758 expand_insn (icode
, 3, ops
);
2761 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2762 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2763 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2765 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2766 The expansion of STMT happens based on OPTAB table associated. */
2769 expand_vect_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2771 class expand_operand ops
[6];
2773 tree lhs
= gimple_call_lhs (stmt
);
2774 tree op0a
= gimple_call_arg (stmt
, 0);
2775 tree op0b
= gimple_call_arg (stmt
, 1);
2776 tree op1
= gimple_call_arg (stmt
, 2);
2777 tree op2
= gimple_call_arg (stmt
, 3);
2778 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
2780 tree vec_cond_type
= TREE_TYPE (lhs
);
2781 tree op_mode
= TREE_TYPE (op0a
);
2782 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
2784 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2785 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
2787 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
2789 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
2790 rtx rtx_op1
= expand_normal (op1
);
2791 rtx rtx_op2
= expand_normal (op2
);
2793 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2794 create_output_operand (&ops
[0], target
, mode
);
2795 create_input_operand (&ops
[1], rtx_op1
, mode
);
2796 create_input_operand (&ops
[2], rtx_op2
, mode
);
2797 create_fixed_operand (&ops
[3], comparison
);
2798 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
2799 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
2800 expand_insn (icode
, 6, ops
);
2801 if (!rtx_equal_p (ops
[0].value
, target
))
2802 emit_move_insn (target
, ops
[0].value
);
2805 #define expand_vec_cond_optab_fn expand_vect_cond_optab_fn
2806 #define expand_vec_condu_optab_fn expand_vect_cond_optab_fn
2807 #define expand_vec_condeq_optab_fn expand_vect_cond_optab_fn
2809 /* Expand VCOND_MASK optab internal function.
2810 The expansion of STMT happens based on OPTAB table associated. */
2813 expand_vect_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2815 class expand_operand ops
[4];
2817 tree lhs
= gimple_call_lhs (stmt
);
2818 tree op0
= gimple_call_arg (stmt
, 0);
2819 tree op1
= gimple_call_arg (stmt
, 1);
2820 tree op2
= gimple_call_arg (stmt
, 2);
2821 tree vec_cond_type
= TREE_TYPE (lhs
);
2823 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2824 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
2825 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
2826 rtx mask
, rtx_op1
, rtx_op2
;
2828 gcc_assert (icode
!= CODE_FOR_nothing
);
2830 mask
= expand_normal (op0
);
2831 rtx_op1
= expand_normal (op1
);
2832 rtx_op2
= expand_normal (op2
);
2834 mask
= force_reg (mask_mode
, mask
);
2835 rtx_op1
= force_reg (mode
, rtx_op1
);
2837 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2838 create_output_operand (&ops
[0], target
, mode
);
2839 create_input_operand (&ops
[1], rtx_op1
, mode
);
2840 create_input_operand (&ops
[2], rtx_op2
, mode
);
2841 create_input_operand (&ops
[3], mask
, mask_mode
);
2842 expand_insn (icode
, 4, ops
);
2843 if (!rtx_equal_p (ops
[0].value
, target
))
2844 emit_move_insn (target
, ops
[0].value
);
2847 #define expand_vec_cond_mask_optab_fn expand_vect_cond_mask_optab_fn
2849 /* Expand VEC_SET internal functions. */
2852 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2854 tree lhs
= gimple_call_lhs (stmt
);
2855 tree op0
= gimple_call_arg (stmt
, 0);
2856 tree op1
= gimple_call_arg (stmt
, 1);
2857 tree op2
= gimple_call_arg (stmt
, 2);
2858 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2859 rtx src
= expand_normal (op0
);
2861 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
2862 scalar_mode innermode
= GET_MODE_INNER (outermode
);
2864 rtx value
= expand_normal (op1
);
2865 rtx pos
= expand_normal (op2
);
2867 class expand_operand ops
[3];
2868 enum insn_code icode
= optab_handler (optab
, outermode
);
2870 if (icode
!= CODE_FOR_nothing
)
2872 rtx temp
= gen_reg_rtx (outermode
);
2873 emit_move_insn (temp
, src
);
2875 create_fixed_operand (&ops
[0], temp
);
2876 create_input_operand (&ops
[1], value
, innermode
);
2877 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
2879 if (maybe_expand_insn (icode
, 3, ops
))
2881 emit_move_insn (target
, temp
);
2889 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2894 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2896 /* When guessing was done, the hints should be already stripped away. */
2897 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2900 tree lhs
= gimple_call_lhs (stmt
);
2902 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2904 target
= const0_rtx
;
2905 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2906 if (lhs
&& val
!= target
)
2907 emit_move_insn (target
, val
);
2910 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2911 should never be called. */
2914 expand_VA_ARG (internal_fn
, gcall
*)
2919 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2920 dummy function should never be called. */
2923 expand_VEC_CONVERT (internal_fn
, gcall
*)
2928 /* Expand the IFN_UNIQUE function according to its first argument. */
2931 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2933 rtx pattern
= NULL_RTX
;
2934 enum ifn_unique_kind kind
2935 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2942 case IFN_UNIQUE_UNSPEC
:
2943 if (targetm
.have_unique ())
2944 pattern
= targetm
.gen_unique ();
2947 case IFN_UNIQUE_OACC_FORK
:
2948 case IFN_UNIQUE_OACC_JOIN
:
2949 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2951 tree lhs
= gimple_call_lhs (stmt
);
2952 rtx target
= const0_rtx
;
2955 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2957 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2958 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2960 if (kind
== IFN_UNIQUE_OACC_FORK
)
2961 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2963 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2971 emit_insn (pattern
);
2974 /* The size of an OpenACC compute dimension. */
2977 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2979 tree lhs
= gimple_call_lhs (stmt
);
2984 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2985 if (targetm
.have_oacc_dim_size ())
2987 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2988 VOIDmode
, EXPAND_NORMAL
);
2989 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2992 emit_move_insn (target
, GEN_INT (1));
2995 /* The position of an OpenACC execution engine along one compute axis. */
2998 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3000 tree lhs
= gimple_call_lhs (stmt
);
3005 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3006 if (targetm
.have_oacc_dim_pos ())
3008 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3009 VOIDmode
, EXPAND_NORMAL
);
3010 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3013 emit_move_insn (target
, const0_rtx
);
3016 /* This is expanded by oacc_device_lower pass. */
3019 expand_GOACC_LOOP (internal_fn
, gcall
*)
3024 /* This is expanded by oacc_device_lower pass. */
3027 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3032 /* This is expanded by oacc_device_lower pass. */
3035 expand_GOACC_TILE (internal_fn
, gcall
*)
3040 /* Set errno to EDOM. */
3043 expand_SET_EDOM (internal_fn
, gcall
*)
3046 #ifdef GEN_ERRNO_RTX
3047 rtx errno_rtx
= GEN_ERRNO_RTX
;
3049 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3051 emit_move_insn (errno_rtx
,
3052 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3058 /* Expand atomic bit test and set. */
3061 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3063 expand_ifn_atomic_bit_test_and (call
);
3066 /* Expand atomic bit test and complement. */
3069 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3071 expand_ifn_atomic_bit_test_and (call
);
3074 /* Expand atomic bit test and reset. */
3077 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3079 expand_ifn_atomic_bit_test_and (call
);
3082 /* Expand atomic bit test and set. */
3085 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3087 expand_ifn_atomic_compare_exchange (call
);
3090 /* Expand LAUNDER to assignment, lhs = arg0. */
3093 expand_LAUNDER (internal_fn
, gcall
*call
)
3095 tree lhs
= gimple_call_lhs (call
);
3100 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3103 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3106 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3108 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3109 int rhs_index
= internal_fn_stored_value_index (ifn
);
3110 int mask_index
= internal_fn_mask_index (ifn
);
3111 tree base
= gimple_call_arg (stmt
, 0);
3112 tree offset
= gimple_call_arg (stmt
, 1);
3113 tree scale
= gimple_call_arg (stmt
, 2);
3114 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3116 rtx base_rtx
= expand_normal (base
);
3117 rtx offset_rtx
= expand_normal (offset
);
3118 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3119 rtx rhs_rtx
= expand_normal (rhs
);
3121 class expand_operand ops
[6];
3123 create_address_operand (&ops
[i
++], base_rtx
);
3124 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3125 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3126 create_integer_operand (&ops
[i
++], scale_int
);
3127 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3128 if (mask_index
>= 0)
3130 tree mask
= gimple_call_arg (stmt
, mask_index
);
3131 rtx mask_rtx
= expand_normal (mask
);
3132 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3135 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3136 TYPE_MODE (TREE_TYPE (offset
)));
3137 expand_insn (icode
, i
, ops
);
3140 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3143 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3145 tree lhs
= gimple_call_lhs (stmt
);
3146 tree base
= gimple_call_arg (stmt
, 0);
3147 tree offset
= gimple_call_arg (stmt
, 1);
3148 tree scale
= gimple_call_arg (stmt
, 2);
3150 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3151 rtx base_rtx
= expand_normal (base
);
3152 rtx offset_rtx
= expand_normal (offset
);
3153 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3156 class expand_operand ops
[6];
3157 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3158 create_address_operand (&ops
[i
++], base_rtx
);
3159 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3160 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3161 create_integer_operand (&ops
[i
++], scale_int
);
3162 if (optab
== mask_gather_load_optab
)
3164 tree mask
= gimple_call_arg (stmt
, 4);
3165 rtx mask_rtx
= expand_normal (mask
);
3166 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3168 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3169 TYPE_MODE (TREE_TYPE (offset
)));
3170 expand_insn (icode
, i
, ops
);
3171 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3172 emit_move_insn (lhs_rtx
, ops
[0].value
);
3175 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3176 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3179 contains_call_div_mod (rtx_insn
*insn
)
3181 subrtx_iterator::array_type array
;
3182 for (; insn
; insn
= NEXT_INSN (insn
))
3185 else if (INSN_P (insn
))
3186 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3187 switch (GET_CODE (*iter
))
3201 /* Expand DIVMOD() using:
3202 a) optab handler for udivmod/sdivmod if it is available.
3203 b) If optab_handler doesn't exist, generate call to
3204 target-specific divmod libfunc. */
3207 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3209 tree lhs
= gimple_call_lhs (call_stmt
);
3210 tree arg0
= gimple_call_arg (call_stmt
, 0);
3211 tree arg1
= gimple_call_arg (call_stmt
, 1);
3213 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3214 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3215 machine_mode mode
= TYPE_MODE (type
);
3216 bool unsignedp
= TYPE_UNSIGNED (type
);
3217 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3219 rtx op0
= expand_normal (arg0
);
3220 rtx op1
= expand_normal (arg1
);
3221 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3223 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3224 rtx_insn
*insns
= NULL
;
3226 if (TREE_CODE (arg1
) == INTEGER_CST
)
3228 /* For DIVMOD by integral constants, there could be efficient code
3229 expanded inline e.g. using shifts and plus/minus. Try to expand
3230 the division and modulo and if it emits any library calls or any
3231 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3233 scalar_int_mode int_mode
;
3234 if (remainder
== NULL_RTX
3236 && CONST_INT_P (op1
)
3237 && !pow2p_hwi (INTVAL (op1
))
3238 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3239 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3240 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3241 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3242 && optimize_insn_for_speed_p ())
3244 rtx_insn
*last
= get_last_insn ();
3245 remainder
= NULL_RTX
;
3246 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3247 TYPE_UNSIGNED (type
));
3248 if (quotient
!= NULL_RTX
)
3250 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3252 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3253 set_dst_reg_note (move
, REG_EQUAL
,
3254 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3255 ? UDIV
: DIV
, int_mode
,
3256 copy_rtx (op0
), op1
),
3258 move
= emit_move_insn (remainder
, remainder
);
3259 set_dst_reg_note (move
, REG_EQUAL
,
3260 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3261 ? UMOD
: MOD
, int_mode
,
3262 copy_rtx (op0
), op1
),
3267 delete_insns_since (last
);
3270 if (remainder
== NULL_RTX
)
3272 struct separate_ops ops
;
3273 ops
.code
= TRUNC_DIV_EXPR
;
3275 ops
.op0
= make_tree (ops
.type
, op0
);
3277 ops
.op2
= NULL_TREE
;
3278 ops
.location
= gimple_location (call_stmt
);
3280 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3281 if (contains_call_div_mod (get_insns ()))
3282 quotient
= NULL_RTX
;
3285 ops
.code
= TRUNC_MOD_EXPR
;
3286 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3288 if (contains_call_div_mod (get_insns ()))
3289 remainder
= NULL_RTX
;
3292 insns
= get_insns ();
3300 /* Check if optab_handler exists for divmod_optab for given mode. */
3301 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3303 quotient
= gen_reg_rtx (mode
);
3304 remainder
= gen_reg_rtx (mode
);
3305 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3308 /* Generate call to divmod libfunc if it exists. */
3309 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3310 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3311 "ient
, &remainder
);
3316 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3317 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3318 make_tree (TREE_TYPE (arg0
), quotient
),
3319 make_tree (TREE_TYPE (arg1
), remainder
)),
3320 target
, VOIDmode
, EXPAND_NORMAL
);
3326 expand_NOP (internal_fn
, gcall
*)
3328 /* Nothing. But it shouldn't really prevail. */
3331 /* Coroutines, all should have been processed at this stage. */
3334 expand_CO_FRAME (internal_fn
, gcall
*)
3340 expand_CO_YIELD (internal_fn
, gcall
*)
3346 expand_CO_SUSPN (internal_fn
, gcall
*)
3352 expand_CO_ACTOR (internal_fn
, gcall
*)
3357 /* Expand a call to FN using the operands in STMT. FN has a single
3358 output operand and NARGS input operands. */
3361 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3364 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
3366 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3367 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3368 gcc_assert (icode
!= CODE_FOR_nothing
);
3370 tree lhs
= gimple_call_lhs (stmt
);
3371 rtx lhs_rtx
= NULL_RTX
;
3373 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3375 /* Do not assign directly to a promoted subreg, since there is no
3376 guarantee that the instruction will leave the upper bits of the
3377 register in the state required by SUBREG_PROMOTED_SIGN. */
3379 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
3382 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
3384 for (unsigned int i
= 0; i
< nargs
; ++i
)
3386 tree rhs
= gimple_call_arg (stmt
, i
);
3387 tree rhs_type
= TREE_TYPE (rhs
);
3388 rtx rhs_rtx
= expand_normal (rhs
);
3389 if (INTEGRAL_TYPE_P (rhs_type
))
3390 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
3391 TYPE_MODE (rhs_type
),
3392 TYPE_UNSIGNED (rhs_type
));
3394 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
3397 expand_insn (icode
, nargs
+ 1, ops
);
3398 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
3400 /* If the return value has an integral type, convert the instruction
3401 result to that type. This is useful for things that return an
3402 int regardless of the size of the input. If the instruction result
3403 is smaller than required, assume that it is signed.
3405 If the return value has a nonintegral type, its mode must match
3406 the instruction result. */
3407 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
3409 /* If this is a scalar in a register that is stored in a wider
3410 mode than the declared mode, compute the result into its
3411 declared mode and then convert to the wider mode. */
3412 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3413 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
3414 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
3415 SUBREG_PROMOTED_SIGN (lhs_rtx
));
3417 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
3418 emit_move_insn (lhs_rtx
, ops
[0].value
);
3421 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3422 convert_move (lhs_rtx
, ops
[0].value
, 0);
3427 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3430 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3432 expand_operand ops
[3];
3435 tree lhs
= gimple_call_lhs (stmt
);
3436 tree lhs_type
= TREE_TYPE (lhs
);
3437 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3438 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3440 for (unsigned int i
= 0; i
< 2; ++i
)
3442 tree rhs
= gimple_call_arg (stmt
, i
);
3443 rhs_type
[i
] = TREE_TYPE (rhs
);
3444 rtx rhs_rtx
= expand_normal (rhs
);
3445 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3448 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3449 TYPE_MODE (lhs_type
));
3451 expand_insn (icode
, 3, ops
);
3452 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3453 emit_move_insn (lhs_rtx
, ops
[0].value
);
3456 /* Expanders for optabs that can use expand_direct_optab_fn. */
3458 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3459 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3461 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3462 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3464 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3465 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3467 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3468 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3470 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3471 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3473 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3474 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3476 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3477 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3479 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3480 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3482 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3483 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3485 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3486 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3488 /* RETURN_TYPE and ARGS are a return type and argument list that are
3489 in principle compatible with FN (which satisfies direct_internal_fn_p).
3490 Return the types that should be used to determine whether the
3491 target supports FN. */
3494 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3496 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3497 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3498 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3499 return tree_pair (type0
, type1
);
3502 /* CALL is a call whose return type and arguments are in principle
3503 compatible with FN (which satisfies direct_internal_fn_p). Return the
3504 types that should be used to determine whether the target supports FN. */
3507 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3509 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3510 tree op0
= (info
.type0
< 0
3511 ? gimple_call_lhs (call
)
3512 : gimple_call_arg (call
, info
.type0
));
3513 tree op1
= (info
.type1
< 0
3514 ? gimple_call_lhs (call
)
3515 : gimple_call_arg (call
, info
.type1
));
3516 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3519 /* Return true if OPTAB is supported for TYPES (whose modes should be
3520 the same) when the optimization type is OPT_TYPE. Used for simple
3524 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3525 optimization_type opt_type
)
3527 machine_mode mode
= TYPE_MODE (types
.first
);
3528 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3529 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3532 /* Return true if OPTAB is supported for TYPES, where the first type
3533 is the destination and the second type is the source. Used for
3537 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3538 optimization_type opt_type
)
3540 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3541 TYPE_MODE (types
.second
), opt_type
)
3542 != CODE_FOR_nothing
);
3545 /* Return true if load/store lanes optab OPTAB is supported for
3546 array type TYPES.first when the optimization type is OPT_TYPE. */
3549 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3550 optimization_type opt_type
)
3552 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3553 machine_mode imode
= TYPE_MODE (types
.first
);
3554 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3555 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3556 != CODE_FOR_nothing
);
3559 #define direct_unary_optab_supported_p direct_optab_supported_p
3560 #define direct_binary_optab_supported_p direct_optab_supported_p
3561 #define direct_ternary_optab_supported_p direct_optab_supported_p
3562 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3563 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3564 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3565 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3566 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3567 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3568 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3569 #define direct_len_load_optab_supported_p direct_optab_supported_p
3570 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3571 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3572 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3573 #define direct_vec_cond_mask_optab_supported_p multi_vector_optab_supported_p
3574 #define direct_vec_cond_optab_supported_p multi_vector_optab_supported_p
3575 #define direct_vec_condu_optab_supported_p multi_vector_optab_supported_p
3576 #define direct_vec_condeq_optab_supported_p multi_vector_optab_supported_p
3577 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3578 #define direct_len_store_optab_supported_p direct_optab_supported_p
3579 #define direct_while_optab_supported_p convert_optab_supported_p
3580 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3581 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3582 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3583 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3584 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3586 /* Return the optab used by internal function FN. */
3589 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3593 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3594 case IFN_##CODE: break;
3595 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3596 case IFN_##CODE: return OPTAB##_optab;
3597 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3598 UNSIGNED_OPTAB, TYPE) \
3599 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3600 ? UNSIGNED_OPTAB ## _optab \
3601 : SIGNED_OPTAB ## _optab);
3602 #include "internal-fn.def"
3610 /* Return the optab used by internal function FN. */
3613 direct_internal_fn_optab (internal_fn fn
)
3617 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3618 case IFN_##CODE: break;
3619 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3620 case IFN_##CODE: return OPTAB##_optab;
3621 #include "internal-fn.def"
3629 /* Return true if FN is supported for the types in TYPES when the
3630 optimization type is OPT_TYPE. The types are those associated with
3631 the "type0" and "type1" fields of FN's direct_internal_fn_info
3635 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3636 optimization_type opt_type
)
3640 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3641 case IFN_##CODE: break;
3642 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3644 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3646 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3647 UNSIGNED_OPTAB, TYPE) \
3650 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3651 ? UNSIGNED_OPTAB ## _optab \
3652 : SIGNED_OPTAB ## _optab); \
3653 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3656 #include "internal-fn.def"
3664 /* Return true if FN is supported for type TYPE when the optimization
3665 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3666 fields of FN's direct_internal_fn_info structure are the same. */
3669 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3670 optimization_type opt_type
)
3672 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3673 gcc_checking_assert (info
.type0
== info
.type1
);
3674 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3677 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3678 given that STMT is a call to a direct internal function. */
3681 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
3683 internal_fn fn
= gimple_call_internal_fn (stmt
);
3684 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3685 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
3688 /* If FN is commutative in two consecutive arguments, return the
3689 index of the first, otherwise return -1. */
3692 first_commutative_argument (internal_fn fn
)
3726 /* Return true if IFN_SET_EDOM is supported. */
3729 set_edom_supported_p (void)
3738 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3740 expand_##CODE (internal_fn fn, gcall *stmt) \
3742 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3744 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3745 UNSIGNED_OPTAB, TYPE) \
3747 expand_##CODE (internal_fn fn, gcall *stmt) \
3749 tree_pair types = direct_internal_fn_types (fn, stmt); \
3750 optab which_optab = direct_internal_fn_optab (fn, types); \
3751 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3753 #include "internal-fn.def"
3755 /* Routines to expand each internal function, indexed by function number.
3756 Each routine has the prototype:
3758 expand_<NAME> (gcall *stmt)
3760 where STMT is the statement that performs the call. */
3761 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3762 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3763 #include "internal-fn.def"
3767 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
3769 #define FOR_EACH_CODE_MAPPING(T) \
3770 T (PLUS_EXPR, IFN_COND_ADD) \
3771 T (MINUS_EXPR, IFN_COND_SUB) \
3772 T (MULT_EXPR, IFN_COND_MUL) \
3773 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
3774 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
3775 T (RDIV_EXPR, IFN_COND_RDIV) \
3776 T (MIN_EXPR, IFN_COND_MIN) \
3777 T (MAX_EXPR, IFN_COND_MAX) \
3778 T (BIT_AND_EXPR, IFN_COND_AND) \
3779 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3780 T (BIT_XOR_EXPR, IFN_COND_XOR) \
3781 T (LSHIFT_EXPR, IFN_COND_SHL) \
3782 T (RSHIFT_EXPR, IFN_COND_SHR)
3784 /* Return a function that only performs CODE when a certain condition is met
3785 and that uses a given fallback value otherwise. For example, if CODE is
3786 a binary operation associated with conditional function FN:
3788 LHS = FN (COND, A, B, ELSE)
3790 is equivalent to the C expression:
3792 LHS = COND ? A CODE B : ELSE;
3794 operating elementwise if the operands are vectors.
3796 Return IFN_LAST if no such function exists. */
3799 get_conditional_internal_fn (tree_code code
)
3803 #define CASE(CODE, IFN) case CODE: return IFN;
3804 FOR_EACH_CODE_MAPPING(CASE
)
3811 /* If IFN implements the conditional form of a tree code, return that
3812 tree code, otherwise return ERROR_MARK. */
3815 conditional_internal_fn_code (internal_fn ifn
)
3819 #define CASE(CODE, IFN) case IFN: return CODE;
3820 FOR_EACH_CODE_MAPPING(CASE
)
3827 /* Invoke T(IFN) for each internal function IFN that also has an
3829 #define FOR_EACH_COND_FN_PAIR(T) \
3835 /* Return a function that only performs internal function FN when a
3836 certain condition is met and that uses a given fallback value otherwise.
3837 In other words, the returned function FN' is such that:
3839 LHS = FN' (COND, A1, ... An, ELSE)
3841 is equivalent to the C expression:
3843 LHS = COND ? FN (A1, ..., An) : ELSE;
3845 operating elementwise if the operands are vectors.
3847 Return IFN_LAST if no such function exists. */
3850 get_conditional_internal_fn (internal_fn fn
)
3854 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
3855 FOR_EACH_COND_FN_PAIR(CASE
)
3862 /* If IFN implements the conditional form of an unconditional internal
3863 function, return that unconditional function, otherwise return IFN_LAST. */
3866 get_unconditional_internal_fn (internal_fn ifn
)
3870 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
3871 FOR_EACH_COND_FN_PAIR(CASE
)
3878 /* Return true if STMT can be interpreted as a conditional tree code
3879 operation of the form:
3881 LHS = COND ? OP (RHS1, ...) : ELSE;
3883 operating elementwise if the operands are vectors. This includes
3884 the case of an all-true COND, so that the operation always happens.
3886 When returning true, set:
3888 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
3889 is known to be all-true
3890 - *CODE_OUT to the tree code
3891 - OPS[I] to operand I of *CODE_OUT
3892 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
3893 condition is known to be all true. */
3896 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
3897 tree_code
*code_out
,
3898 tree (&ops
)[3], tree
*else_out
)
3900 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
3902 *cond_out
= NULL_TREE
;
3903 *code_out
= gimple_assign_rhs_code (assign
);
3904 ops
[0] = gimple_assign_rhs1 (assign
);
3905 ops
[1] = gimple_assign_rhs2 (assign
);
3906 ops
[2] = gimple_assign_rhs3 (assign
);
3907 *else_out
= NULL_TREE
;
3910 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
3911 if (gimple_call_internal_p (call
))
3913 internal_fn ifn
= gimple_call_internal_fn (call
);
3914 tree_code code
= conditional_internal_fn_code (ifn
);
3915 if (code
!= ERROR_MARK
)
3917 *cond_out
= gimple_call_arg (call
, 0);
3919 unsigned int nops
= gimple_call_num_args (call
) - 2;
3920 for (unsigned int i
= 0; i
< 3; ++i
)
3921 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
3922 *else_out
= gimple_call_arg (call
, nops
+ 1);
3923 if (integer_truep (*cond_out
))
3925 *cond_out
= NULL_TREE
;
3926 *else_out
= NULL_TREE
;
3934 /* Return true if IFN is some form of load from memory. */
3937 internal_load_fn_p (internal_fn fn
)
3942 case IFN_LOAD_LANES
:
3943 case IFN_MASK_LOAD_LANES
:
3944 case IFN_GATHER_LOAD
:
3945 case IFN_MASK_GATHER_LOAD
:
3954 /* Return true if IFN is some form of store to memory. */
3957 internal_store_fn_p (internal_fn fn
)
3961 case IFN_MASK_STORE
:
3962 case IFN_STORE_LANES
:
3963 case IFN_MASK_STORE_LANES
:
3964 case IFN_SCATTER_STORE
:
3965 case IFN_MASK_SCATTER_STORE
:
3974 /* Return true if IFN is some form of gather load or scatter store. */
3977 internal_gather_scatter_fn_p (internal_fn fn
)
3981 case IFN_GATHER_LOAD
:
3982 case IFN_MASK_GATHER_LOAD
:
3983 case IFN_SCATTER_STORE
:
3984 case IFN_MASK_SCATTER_STORE
:
3992 /* If FN takes a vector mask argument, return the index of that argument,
3993 otherwise return -1. */
3996 internal_fn_mask_index (internal_fn fn
)
4001 case IFN_MASK_LOAD_LANES
:
4002 case IFN_MASK_STORE
:
4003 case IFN_MASK_STORE_LANES
:
4006 case IFN_MASK_GATHER_LOAD
:
4007 case IFN_MASK_SCATTER_STORE
:
4011 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4012 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4016 /* If FN takes a value that should be stored to memory, return the index
4017 of that argument, otherwise return -1. */
4020 internal_fn_stored_value_index (internal_fn fn
)
4024 case IFN_MASK_STORE
:
4025 case IFN_MASK_STORE_LANES
:
4026 case IFN_SCATTER_STORE
:
4027 case IFN_MASK_SCATTER_STORE
:
4036 /* Return true if the target supports gather load or scatter store function
4037 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4038 while for stores it is the vector type of the stored data argument.
4039 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4040 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4041 offset from the shared base address of each loaded or stored element.
4042 SCALE is the amount by which these offsets should be multiplied
4043 *after* they have been extended to address width. */
4046 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4047 tree memory_element_type
,
4048 tree offset_vector_type
, int scale
)
4050 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4051 TYPE_SIZE (memory_element_type
)))
4053 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4054 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4056 optab optab
= direct_internal_fn_optab (ifn
);
4057 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4058 TYPE_MODE (offset_vector_type
));
4059 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4060 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4061 return (icode
!= CODE_FOR_nothing
4062 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4063 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4066 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4067 for pointers of type TYPE when the accesses have LENGTH bytes and their
4068 common byte alignment is ALIGN. */
4071 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4072 poly_uint64 length
, unsigned int align
)
4074 machine_mode mode
= TYPE_MODE (type
);
4075 optab optab
= direct_internal_fn_optab (ifn
);
4076 insn_code icode
= direct_optab_handler (optab
, mode
);
4077 if (icode
== CODE_FOR_nothing
)
4079 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4080 return (insn_operand_matches (icode
, 3, length_rtx
)
4081 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4084 /* Expand STMT as though it were a call to internal function FN. */
4087 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4089 internal_fn_expanders
[fn
] (fn
, stmt
);
4092 /* Expand STMT, which is a call to internal function FN. */
4095 expand_internal_call (gcall
*stmt
)
4097 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4100 /* If TYPE is a vector type, return true if IFN is a direct internal
4101 function that is supported for that type. If TYPE is a scalar type,
4102 return true if IFN is a direct internal function that is supported for
4103 the target's preferred vector version of TYPE. */
4106 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4109 if (!VECTOR_TYPE_P (type
) && is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4111 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4112 if (VECTOR_MODE_P (vmode
))
4113 type
= build_vector_type_for_mode (type
, vmode
);
4116 return (VECTOR_MODE_P (TYPE_MODE (type
))
4117 && direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
));
4121 expand_PHI (internal_fn
, gcall
*)