]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/internal-fn.c
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[AArch64] Implement <su><maxmin>v2di3 pattern
[thirdparty/gcc.git] / gcc / internal-fn.c
CommitLineData
25583c4f 1/* Internal functions.
23a5b65a 2 Copyright (C) 2011-2014 Free Software Foundation, Inc.
25583c4f
RS		 3
4This file is part of GCC.
5
6GCC is free software; you can redistribute it and/or modify it under
7the terms of the GNU General Public License as published by the Free
8Software Foundation; either version 3, or (at your option) any later
9version.
10
11GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12WARRANTY; without even the implied warranty of MERCHANTABILITY or
13FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14for more details.
15
16You should have received a copy of the GNU General Public License
17along with GCC; see the file COPYING3. If not see
18<http://www.gnu.org/licenses/>. */
19
20#include "config.h"
21#include "system.h"
22#include "coretypes.h"
25583c4f 23#include "tree.h"
0e37a2f3 24#include "internal-fn.h"
d8a2d370 25#include "stor-layout.h"
25583c4f 26#include "expr.h"
b0710fe1 27#include "insn-codes.h"
25583c4f 28#include "optabs.h"
60393bbc
AM		 29#include "predict.h"
30#include "vec.h"
31#include "hashtab.h"
32#include "hash-set.h"
33#include "machmode.h"
34#include "tm.h"
35#include "hard-reg-set.h"
36#include "input.h"
37#include "function.h"
38#include "dominance.h"
39#include "cfg.h"
2fb9a547
AM		 40#include "basic-block.h"
41#include "tree-ssa-alias.h"
42#include "internal-fn.h"
43#include "gimple-expr.h"
44#include "is-a.h"
25583c4f 45#include "gimple.h"
31e071ae
MP		 46#include "ubsan.h"
47#include "target.h"
97286431
JJ		 48#include "stringpool.h"
49#include "tree-ssanames.h"
ed9c79e1 50#include "diagnostic-core.h"
25583c4f
RS		 51
52/* The names of each internal function, indexed by function number. */
53const char *const internal_fn_name_array[] = {
b78475cf 54#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
25583c4f
RS		 55#include "internal-fn.def"
56#undef DEF_INTERNAL_FN
57 "<invalid-fn>"
58};
59
60/* The ECF_* flags of each internal function, indexed by function number. */
61const int internal_fn_flags_array[] = {
b78475cf 62#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
25583c4f
RS		 63#include "internal-fn.def"
64#undef DEF_INTERNAL_FN
65 0
66};
67
b78475cf
YG		 68/* Fnspec of each internal function, indexed by function number. */
69const_tree internal_fn_fnspec_array[IFN_LAST + 1];
70
71void
72init_internal_fns ()
73{
74#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
75 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
63a4184f 76 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
b78475cf
YG		 77#include "internal-fn.def"
78#undef DEF_INTERNAL_FN
79 internal_fn_fnspec_array[IFN_LAST] = 0;
80}
81
272c6793
RS		 82/* ARRAY_TYPE is an array of vector modes. Return the associated insn
83 for load-lanes-style optab OPTAB. The insn must exist. */
84
85static enum insn_code
86get_multi_vector_move (tree array_type, convert_optab optab)
87{
88 enum insn_code icode;
ef4bddc2
RS		 89 machine_mode imode;
90 machine_mode vmode;
272c6793
RS		 91
92 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
93 imode = TYPE_MODE (array_type);
94 vmode = TYPE_MODE (TREE_TYPE (array_type));
95
96 icode = convert_optab_handler (optab, imode, vmode);
97 gcc_assert (icode != CODE_FOR_nothing);
98 return icode;
99}
100
101/* Expand LOAD_LANES call STMT. */
102
103static void
104expand_LOAD_LANES (gimple stmt)
105{
106 struct expand_operand ops[2];
107 tree type, lhs, rhs;
108 rtx target, mem;
109
110 lhs = gimple_call_lhs (stmt);
111 rhs = gimple_call_arg (stmt, 0);
112 type = TREE_TYPE (lhs);
113
114 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
115 mem = expand_normal (rhs);
116
117 gcc_assert (MEM_P (mem));
118 PUT_MODE (mem, TYPE_MODE (type));
119
120 create_output_operand (&ops[0], target, TYPE_MODE (type));
121 create_fixed_operand (&ops[1], mem);
122 expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
123}
124
125/* Expand STORE_LANES call STMT. */
126
127static void
128expand_STORE_LANES (gimple stmt)
129{
130 struct expand_operand ops[2];
131 tree type, lhs, rhs;
132 rtx target, reg;
133
134 lhs = gimple_call_lhs (stmt);
135 rhs = gimple_call_arg (stmt, 0);
136 type = TREE_TYPE (rhs);
137
138 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
139 reg = expand_normal (rhs);
140
141 gcc_assert (MEM_P (target));
142 PUT_MODE (target, TYPE_MODE (type));
143
144 create_fixed_operand (&ops[0], target);
145 create_input_operand (&ops[1], reg, TYPE_MODE (type));
146 expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
147}
148
8170608b
TB		 149static void
150expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED)
151{
152 gcc_unreachable ();
153}
154
74bf76ed
JJ		 155/* This should get expanded in adjust_simduid_builtins. */
156
157static void
158expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED)
159{
160 gcc_unreachable ();
161}
162
163/* This should get expanded in adjust_simduid_builtins. */
164
165static void
166expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED)
167{
168 gcc_unreachable ();
169}
170
171/* This should get expanded in adjust_simduid_builtins. */
172
173static void
174expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED)
175{
176 gcc_unreachable ();
177}
178
b9a55b13
MP		 179/* This should get expanded in the sanopt pass. */
180
181static void
182expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED)
183{
184 gcc_unreachable ();
185}
186
0e37a2f3
MP		 187/* This should get expanded in the sanopt pass. */
188
189static void
190expand_UBSAN_BOUNDS (gimple stmt ATTRIBUTE_UNUSED)
0e82f089
MP		 191{
192 gcc_unreachable ();
193}
194
195/* This should get expanded in the sanopt pass. */
196
197static void
198expand_UBSAN_OBJECT_SIZE (gimple stmt ATTRIBUTE_UNUSED)
0e37a2f3
MP		 199{
200 gcc_unreachable ();
201}
202
c62ccb9a
YG		 203/* This should get expanded in the sanopt pass. */
204
205static void
206expand_ASAN_CHECK (gimple stmt ATTRIBUTE_UNUSED)
207{
208 gcc_unreachable ();
209}
210
1304953e
JJ		 211/* Helper function for expand_addsub_overflow. Return 1
212 if ARG interpreted as signed in its precision is known to be always
213 positive or 2 if ARG is known to be always negative, or 3 if ARG may
214 be positive or negative. */
215
216static int
217get_range_pos_neg (tree arg)
218{
219 if (arg == error_mark_node)
220 return 3;
221
222 int prec = TYPE_PRECISION (TREE_TYPE (arg));
223 int cnt = 0;
224 if (TREE_CODE (arg) == INTEGER_CST)
225 {
226 wide_int w = wi::sext (arg, prec);
227 if (wi::neg_p (w))
228 return 2;
229 else
230 return 1;
231 }
232 while (CONVERT_EXPR_P (arg)
233 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
234 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
235 {
236 arg = TREE_OPERAND (arg, 0);
237 /* Narrower value zero extended into wider type
238 will always result in positive values. */
239 if (TYPE_UNSIGNED (TREE_TYPE (arg))
240 && TYPE_PRECISION (TREE_TYPE (arg)) < prec)
241 return 1;
242 prec = TYPE_PRECISION (TREE_TYPE (arg));
243 if (++cnt > 30)
244 return 3;
245 }
246
247 if (TREE_CODE (arg) != SSA_NAME)
248 return 3;
249 wide_int arg_min, arg_max;
250 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
251 {
252 gimple g = SSA_NAME_DEF_STMT (arg);
253 if (is_gimple_assign (g)
254 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
255 {
256 tree t = gimple_assign_rhs1 (g);
257 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
258 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
259 {
260 if (TYPE_UNSIGNED (TREE_TYPE (t))
261 && TYPE_PRECISION (TREE_TYPE (t)) < prec)
262 return 1;
263 prec = TYPE_PRECISION (TREE_TYPE (t));
264 arg = t;
265 if (++cnt > 30)
266 return 3;
267 continue;
268 }
269 }
270 return 3;
271 }
272 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
273 {
274 /* For unsigned values, the "positive" range comes
275 below the "negative" range. */
276 if (!wi::neg_p (wi::sext (arg_max, prec), SIGNED))
277 return 1;
278 if (wi::neg_p (wi::sext (arg_min, prec), SIGNED))
279 return 2;
280 }
281 else
282 {
283 if (!wi::neg_p (wi::sext (arg_min, prec), SIGNED))
284 return 1;
285 if (wi::neg_p (wi::sext (arg_max, prec), SIGNED))
286 return 2;
287 }
288 return 3;
289}
290
291/* Return minimum precision needed to represent all values
292 of ARG in SIGNed integral type. */
293
294static int
295get_min_precision (tree arg, signop sign)
296{
297 int prec = TYPE_PRECISION (TREE_TYPE (arg));
298 int cnt = 0;
299 signop orig_sign = sign;
300 if (TREE_CODE (arg) == INTEGER_CST)
301 {
302 int p;
303 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
304 {
305 widest_int w = wi::to_widest (arg);
306 w = wi::ext (w, prec, sign);
307 p = wi::min_precision (w, sign);
308 }
309 else
310 p = wi::min_precision (arg, sign);
311 return MIN (p, prec);
312 }
313 while (CONVERT_EXPR_P (arg)
314 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
315 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
316 {
317 arg = TREE_OPERAND (arg, 0);
318 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
319 {
320 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
321 sign = UNSIGNED;
322 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
323 return prec + (orig_sign != sign);
324 prec = TYPE_PRECISION (TREE_TYPE (arg));
325 }
326 if (++cnt > 30)
327 return prec + (orig_sign != sign);
328 }
329 if (TREE_CODE (arg) != SSA_NAME)
330 return prec + (orig_sign != sign);
331 wide_int arg_min, arg_max;
332 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
333 {
334 gimple g = SSA_NAME_DEF_STMT (arg);
335 if (is_gimple_assign (g)
336 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
337 {
338 tree t = gimple_assign_rhs1 (g);
339 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
340 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
341 {
342 arg = t;
343 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
344 {
345 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
346 sign = UNSIGNED;
347 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
348 return prec + (orig_sign != sign);
349 prec = TYPE_PRECISION (TREE_TYPE (arg));
350 }
351 if (++cnt > 30)
352 return prec + (orig_sign != sign);
353 continue;
354 }
355 }
356 return prec + (orig_sign != sign);
357 }
358 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
359 {
360 int p1 = wi::min_precision (arg_min, sign);
361 int p2 = wi::min_precision (arg_max, sign);
362 p1 = MAX (p1, p2);
363 prec = MIN (prec, p1);
364 }
365 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
366 {
367 int p = wi::min_precision (arg_max, SIGNED);
368 prec = MIN (prec, p);
369 }
370 return prec + (orig_sign != sign);
371}
372
373/* Helper for expand_*_overflow. Store RES into the __real__ part
374 of TARGET. If RES has larger MODE than __real__ part of TARGET,
375 set the __imag__ part to 1 if RES doesn't fit into it. */
376
377static void
378expand_arith_overflow_result_store (tree lhs, rtx target,
379 machine_mode mode, rtx res)
380{
381 machine_mode tgtmode = GET_MODE_INNER (GET_MODE (target));
382 rtx lres = res;
383 if (tgtmode != mode)
384 {
385 rtx_code_label *done_label = gen_label_rtx ();
386 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
387 lres = convert_modes (tgtmode, mode, res, uns);
388 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
389 emit_cmp_and_jump_insns (res, convert_modes (mode, tgtmode, lres, uns),
390 EQ, NULL_RTX, mode, false, done_label,
391 PROB_VERY_LIKELY);
392 write_complex_part (target, const1_rtx, true);
393 emit_label (done_label);
394 }
395 write_complex_part (target, lres, false);
396}
397
5620052d
JJ		 398/* Helper for expand_*_overflow. Store RES into TARGET. */
399
400static void
401expand_ubsan_result_store (rtx target, rtx res)
402{
403 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
404 /* If this is a scalar in a register that is stored in a wider mode
405 than the declared mode, compute the result into its declared mode
406 and then convert to the wider mode. Our value is the computed
407 expression. */
408 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
409 else
410 emit_move_insn (target, res);
411}
412
31e071ae
MP		 413/* Add sub/add overflow checking to the statement STMT.
414 CODE says whether the operation is +, or -. */
415
1304953e
JJ		 416static void
417expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
418 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
419 bool uns1_p, bool is_ubsan)
31e071ae 420{
1304953e
JJ		 421 rtx res, target = NULL_RTX;
422 tree fn;
423 rtx_code_label *done_label = gen_label_rtx ();
424 rtx_code_label *do_error = gen_label_rtx ();
31e071ae 425 do_pending_stack_adjust ();
1304953e
JJ		 426 rtx op0 = expand_normal (arg0);
427 rtx op1 = expand_normal (arg1);
ef4bddc2 428 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
1304953e
JJ		 429 int prec = GET_MODE_PRECISION (mode);
430 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
431 bool do_xor = false;
432
433 if (is_ubsan)
434 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
435
31e071ae 436 if (lhs)
1304953e
JJ		 437 {
438 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
439 if (!is_ubsan)
440 write_complex_part (target, const0_rtx, true);
441 }
442
443 /* We assume both operands and result have the same precision
444 here (GET_MODE_BITSIZE (mode)), S stands for signed type
445 with that precision, U for unsigned type with that precision,
446 sgn for unsigned most significant bit in that precision.
447 s1 is signed first operand, u1 is unsigned first operand,
448 s2 is signed second operand, u2 is unsigned second operand,
449 sr is signed result, ur is unsigned result and the following
450 rules say how to compute result (which is always result of
451 the operands as if both were unsigned, cast to the right
452 signedness) and how to compute whether operation overflowed.
453
454 s1 + s2 -> sr
455 res = (S) ((U) s1 + (U) s2)
456 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
457 s1 - s2 -> sr
458 res = (S) ((U) s1 - (U) s2)
459 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
460 u1 + u2 -> ur
461 res = u1 + u2
462 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
463 u1 - u2 -> ur
464 res = u1 - u2
465 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
466 s1 + u2 -> sr
467 res = (S) ((U) s1 + u2)
468 ovf = ((U) res ^ sgn) < u2
469 s1 + u2 -> ur
470 t1 = (S) (u2 ^ sgn)
471 t2 = s1 + t1
472 res = (U) t2 ^ sgn
473 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
474 s1 - u2 -> sr
475 res = (S) ((U) s1 - u2)
476 ovf = u2 > ((U) s1 ^ sgn)
477 s1 - u2 -> ur
478 res = (U) s1 - u2
479 ovf = s1 < 0 || u2 > (U) s1
480 u1 - s2 -> sr
481 res = u1 - (U) s2
482 ovf = u1 >= ((U) s2 ^ sgn)
483 u1 - s2 -> ur
484 t1 = u1 ^ sgn
485 t2 = t1 - (U) s2
486 res = t2 ^ sgn
487 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
488 s1 + s2 -> ur
489 res = (U) s1 + (U) s2
490 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
491 u1 + u2 -> sr
492 res = (S) (u1 + u2)
493 ovf = (U) res < u2 || res < 0
494 u1 - u2 -> sr
495 res = (S) (u1 - u2)
496 ovf = u1 >= u2 ? res < 0 : res >= 0
497 s1 - s2 -> ur
498 res = (U) s1 - (U) s2
499 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
500
501 if (code == PLUS_EXPR && uns0_p && !uns1_p)
502 {
503 /* PLUS_EXPR is commutative, if operand signedness differs,
504 canonicalize to the first operand being signed and second
505 unsigned to simplify following code. */
506 rtx tem = op1;
507 op1 = op0;
508 op0 = tem;
509 tree t = arg1;
510 arg1 = arg0;
511 arg0 = t;
512 uns0_p = 0;
513 uns1_p = 1;
514 }
515
516 /* u1 +- u2 -> ur */
517 if (uns0_p && uns1_p && unsr_p)
518 {
519 /* Compute the operation. On RTL level, the addition is always
520 unsigned. */
521 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
522 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
523 rtx tem = op0;
524 /* For PLUS_EXPR, the operation is commutative, so we can pick
525 operand to compare against. For prec <= BITS_PER_WORD, I think
526 preferring REG operand is better over CONST_INT, because
527 the CONST_INT might enlarge the instruction or CSE would need
528 to figure out we'd already loaded it into a register before.
529 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
530 as then the multi-word comparison can be perhaps simplified. */
531 if (code == PLUS_EXPR
532 && (prec <= BITS_PER_WORD
533 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
534 : CONST_SCALAR_INT_P (op1)))
535 tem = op1;
536 emit_cmp_and_jump_insns (res, tem, code == PLUS_EXPR ? GEU : LEU,
537 NULL_RTX, mode, false, done_label,
538 PROB_VERY_LIKELY);
539 goto do_error_label;
540 }
541
542 /* s1 +- u2 -> sr */
543 if (!uns0_p && uns1_p && !unsr_p)
544 {
545 /* Compute the operation. On RTL level, the addition is always
546 unsigned. */
547 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
548 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
549 rtx tem = expand_binop (mode, add_optab,
550 code == PLUS_EXPR ? res : op0, sgn,
551 NULL_RTX, false, OPTAB_LIB_WIDEN);
552 emit_cmp_and_jump_insns (tem, op1, GEU, NULL_RTX, mode, false,
553 done_label, PROB_VERY_LIKELY);
554 goto do_error_label;
555 }
556
557 /* s1 + u2 -> ur */
558 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
559 {
560 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
561 OPTAB_LIB_WIDEN);
562 /* As we've changed op1, we have to avoid using the value range
563 for the original argument. */
564 arg1 = error_mark_node;
565 do_xor = true;
566 goto do_signed;
567 }
568
569 /* u1 - s2 -> ur */
570 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
571 {
572 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
573 OPTAB_LIB_WIDEN);
574 /* As we've changed op0, we have to avoid using the value range
575 for the original argument. */
576 arg0 = error_mark_node;
577 do_xor = true;
578 goto do_signed;
579 }
580
581 /* s1 - u2 -> ur */
582 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
583 {
584 /* Compute the operation. On RTL level, the addition is always
585 unsigned. */
586 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
587 OPTAB_LIB_WIDEN);
588 int pos_neg = get_range_pos_neg (arg0);
589 if (pos_neg == 2)
590 /* If ARG0 is known to be always negative, this is always overflow. */
591 emit_jump (do_error);
592 else if (pos_neg == 3)
593 /* If ARG0 is not known to be always positive, check at runtime. */
594 emit_cmp_and_jump_insns (op0, const0_rtx, LT, NULL_RTX, mode, false,
595 do_error, PROB_VERY_UNLIKELY);
596 emit_cmp_and_jump_insns (op1, op0, LEU, NULL_RTX, mode, false,
597 done_label, PROB_VERY_LIKELY);
598 goto do_error_label;
599 }
600
601 /* u1 - s2 -> sr */
602 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
603 {
604 /* Compute the operation. On RTL level, the addition is always
605 unsigned. */
606 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
607 OPTAB_LIB_WIDEN);
608 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
609 OPTAB_LIB_WIDEN);
610 emit_cmp_and_jump_insns (op0, tem, LTU, NULL_RTX, mode, false,
611 done_label, PROB_VERY_LIKELY);
612 goto do_error_label;
613 }
614
615 /* u1 + u2 -> sr */
616 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
617 {
618 /* Compute the operation. On RTL level, the addition is always
619 unsigned. */
620 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
621 OPTAB_LIB_WIDEN);
622 emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false,
623 do_error, PROB_VERY_UNLIKELY);
624 rtx tem = op1;
625 /* The operation is commutative, so we can pick operand to compare
626 against. For prec <= BITS_PER_WORD, I think preferring REG operand
627 is better over CONST_INT, because the CONST_INT might enlarge the
628 instruction or CSE would need to figure out we'd already loaded it
629 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
630 might be more beneficial, as then the multi-word comparison can be
631 perhaps simplified. */
632 if (prec <= BITS_PER_WORD
633 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
634 : CONST_SCALAR_INT_P (op0))
635 tem = op0;
636 emit_cmp_and_jump_insns (res, tem, GEU, NULL_RTX, mode, false,
637 done_label, PROB_VERY_LIKELY);
638 goto do_error_label;
639 }
640
641 /* s1 +- s2 -> ur */
642 if (!uns0_p && !uns1_p && unsr_p)
643 {
644 /* Compute the operation. On RTL level, the addition is always
645 unsigned. */
646 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
647 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
648 int pos_neg = get_range_pos_neg (arg1);
649 if (code == PLUS_EXPR)
650 {
651 int pos_neg0 = get_range_pos_neg (arg0);
652 if (pos_neg0 != 3 && pos_neg == 3)
653 {
654 rtx tem = op1;
655 op1 = op0;
656 op0 = tem;
657 pos_neg = pos_neg0;
658 }
659 }
660 rtx tem;
661 if (pos_neg != 3)
662 {
663 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
664 ? and_optab : ior_optab,
665 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
666 emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false,
667 done_label, PROB_VERY_LIKELY);
668 }
669 else
670 {
671 rtx_code_label *do_ior_label = gen_label_rtx ();
672 emit_cmp_and_jump_insns (op1, const0_rtx,
673 code == MINUS_EXPR ? GE : LT, NULL_RTX,
674 mode, false, do_ior_label, PROB_EVEN);
675 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
676 OPTAB_LIB_WIDEN);
677 emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false,
678 done_label, PROB_VERY_LIKELY);
679 emit_jump (do_error);
680 emit_label (do_ior_label);
681 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
682 OPTAB_LIB_WIDEN);
683 emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false,
684 done_label, PROB_VERY_LIKELY);
685 }
686 goto do_error_label;
687 }
688
689 /* u1 - u2 -> sr */
690 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
691 {
692 /* Compute the operation. On RTL level, the addition is always
693 unsigned. */
694 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
695 OPTAB_LIB_WIDEN);
696 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
697 emit_cmp_and_jump_insns (op0, op1, GEU, NULL_RTX, mode, false,
698 op0_geu_op1, PROB_EVEN);
699 emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false,
700 done_label, PROB_VERY_LIKELY);
701 emit_jump (do_error);
702 emit_label (op0_geu_op1);
703 emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false,
704 done_label, PROB_VERY_LIKELY);
705 goto do_error_label;
706 }
31e071ae 707
1304953e
JJ		 708 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
709
710 /* s1 +- s2 -> sr */
711 do_signed: ;
712 enum insn_code icode;
713 icode = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
31e071ae
MP		 714 if (icode != CODE_FOR_nothing)
715 {
716 struct expand_operand ops[4];
da664544 717 rtx_insn *last = get_last_insn ();
31e071ae
MP		 718
719 res = gen_reg_rtx (mode);
720 create_output_operand (&ops[0], res, mode);
721 create_input_operand (&ops[1], op0, mode);
722 create_input_operand (&ops[2], op1, mode);
723 create_fixed_operand (&ops[3], do_error);
724 if (maybe_expand_insn (icode, 4, ops))
725 {
726 last = get_last_insn ();
0a6a6ac9 727 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
31e071ae
MP		 728 && JUMP_P (last)
729 && any_condjump_p (last)
730 && !find_reg_note (last, REG_BR_PROB, 0))
731 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
732 emit_jump (done_label);
733 }
734 else
735 {
736 delete_insns_since (last);
737 icode = CODE_FOR_nothing;
738 }
739 }
740
741 if (icode == CODE_FOR_nothing)
742 {
da664544 743 rtx_code_label *sub_check = gen_label_rtx ();
97286431 744 int pos_neg = 3;
31e071ae
MP		 745
746 /* Compute the operation. On RTL level, the addition is always
747 unsigned. */
7ddf4d5a
MP		 748 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
749 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
31e071ae 750
f451d3a8
JJ		 751 /* If we can prove one of the arguments (for MINUS_EXPR only
752 the second operand, as subtraction is not commutative) is always
753 non-negative or always negative, we can do just one comparison
754 and conditional jump instead of 2 at runtime, 3 present in the
97286431
JJ		 755 emitted code. If one of the arguments is CONST_INT, all we
756 need is to make sure it is op1, then the first
757 emit_cmp_and_jump_insns will be just folded. Otherwise try
758 to use range info if available. */
f451d3a8 759 if (code == PLUS_EXPR && CONST_INT_P (op0))
97286431
JJ		 760 {
761 rtx tem = op0;
762 op0 = op1;
763 op1 = tem;
764 }
765 else if (CONST_INT_P (op1))
766 ;
f451d3a8 767 else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME)
97286431 768 {
1304953e 769 pos_neg = get_range_pos_neg (arg0);
97286431
JJ		 770 if (pos_neg != 3)
771 {
772 rtx tem = op0;
773 op0 = op1;
774 op1 = tem;
775 }
776 }
777 if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME)
1304953e 778 pos_neg = get_range_pos_neg (arg1);
97286431 779
31e071ae 780 /* If the op1 is negative, we have to use a different check. */
97286431
JJ		 781 if (pos_neg == 3)
782 emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode,
783 false, sub_check, PROB_EVEN);
31e071ae 784
7ddf4d5a 785 /* Compare the result of the operation with one of the operands. */
97286431
JJ		 786 if (pos_neg & 1)
787 emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE,
788 NULL_RTX, mode, false, done_label,
789 PROB_VERY_LIKELY);
790
31e071ae 791 /* If we get here, we have to print the error. */
97286431
JJ		 792 if (pos_neg == 3)
793 {
794 emit_jump (do_error);
795
796 emit_label (sub_check);
797 }
31e071ae 798
31e071ae 799 /* We have k = a + b for b < 0 here. k <= a must hold. */
97286431
JJ		 800 if (pos_neg & 2)
801 emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE,
802 NULL_RTX, mode, false, done_label,
803 PROB_VERY_LIKELY);
31e071ae
MP		 804 }
805
1304953e 806 do_error_label:
1769415d 807 emit_label (do_error);
1304953e
JJ		 808 if (is_ubsan)
809 {
810 /* Expand the ubsan builtin call. */
811 push_temp_slots ();
812 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
813 arg0, arg1);
814 expand_normal (fn);
815 pop_temp_slots ();
816 do_pending_stack_adjust ();
817 }
818 else if (lhs)
819 write_complex_part (target, const1_rtx, true);
31e071ae 820
1769415d
MP		 821 /* We're done. */
822 emit_label (done_label);
31e071ae
MP		 823
824 if (lhs)
1304953e
JJ		 825 {
826 if (is_ubsan)
5620052d 827 expand_ubsan_result_store (target, res);
1304953e
JJ		 828 else
829 {
830 if (do_xor)
831 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
832 OPTAB_LIB_WIDEN);
833
834 expand_arith_overflow_result_store (lhs, target, mode, res);
835 }
836 }
31e071ae
MP		 837}
838
839/* Add negate overflow checking to the statement STMT. */
840
1304953e
JJ		 841static void
842expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan)
31e071ae
MP		 843{
844 rtx res, op1;
1304953e 845 tree fn;
da664544
DM		 846 rtx_code_label *done_label, *do_error;
847 rtx target = NULL_RTX;
31e071ae 848
31e071ae
MP		 849 done_label = gen_label_rtx ();
850 do_error = gen_label_rtx ();
31e071ae
MP		 851
852 do_pending_stack_adjust ();
853 op1 = expand_normal (arg1);
854
ef4bddc2 855 machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
31e071ae 856 if (lhs)
1304953e
JJ		 857 {
858 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
859 if (!is_ubsan)
860 write_complex_part (target, const0_rtx, true);
861 }
31e071ae
MP		 862
863 enum insn_code icode = optab_handler (negv3_optab, mode);
864 if (icode != CODE_FOR_nothing)
865 {
866 struct expand_operand ops[3];
da664544 867 rtx_insn *last = get_last_insn ();
31e071ae
MP		 868
869 res = gen_reg_rtx (mode);
870 create_output_operand (&ops[0], res, mode);
871 create_input_operand (&ops[1], op1, mode);
872 create_fixed_operand (&ops[2], do_error);
873 if (maybe_expand_insn (icode, 3, ops))
874 {
875 last = get_last_insn ();
0a6a6ac9 876 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
31e071ae
MP		 877 && JUMP_P (last)
878 && any_condjump_p (last)
879 && !find_reg_note (last, REG_BR_PROB, 0))
880 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
881 emit_jump (done_label);
882 }
883 else
884 {
885 delete_insns_since (last);
886 icode = CODE_FOR_nothing;
887 }
888 }
889
890 if (icode == CODE_FOR_nothing)
891 {
892 /* Compute the operation. On RTL level, the addition is always
893 unsigned. */
894 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
895
896 /* Compare the operand with the most negative value. */
897 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
898 emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false,
899 done_label, PROB_VERY_LIKELY);
900 }
901
902 emit_label (do_error);
1304953e
JJ		 903 if (is_ubsan)
904 {
905 /* Expand the ubsan builtin call. */
906 push_temp_slots ();
907 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
908 arg1, NULL_TREE);
909 expand_normal (fn);
910 pop_temp_slots ();
911 do_pending_stack_adjust ();
912 }
913 else if (lhs)
914 write_complex_part (target, const1_rtx, true);
31e071ae
MP		 915
916 /* We're done. */
917 emit_label (done_label);
918
919 if (lhs)
1304953e
JJ		 920 {
921 if (is_ubsan)
5620052d 922 expand_ubsan_result_store (target, res);
1304953e
JJ		 923 else
924 expand_arith_overflow_result_store (lhs, target, mode, res);
925 }
31e071ae
MP		 926}
927
928/* Add mul overflow checking to the statement STMT. */
929
1304953e
JJ		 930static void
931expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
932 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan)
31e071ae
MP		 933{
934 rtx res, op0, op1;
1304953e 935 tree fn, type;
da664544
DM		 936 rtx_code_label *done_label, *do_error;
937 rtx target = NULL_RTX;
1304953e
JJ		 938 signop sign;
939 enum insn_code icode;
31e071ae 940
31e071ae
MP		 941 done_label = gen_label_rtx ();
942 do_error = gen_label_rtx ();
31e071ae
MP		 943
944 do_pending_stack_adjust ();
945 op0 = expand_normal (arg0);
946 op1 = expand_normal (arg1);
947
ef4bddc2 948 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
1304953e 949 bool uns = unsr_p;
31e071ae 950 if (lhs)
1304953e
JJ		 951 {
952 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
953 if (!is_ubsan)
954 write_complex_part (target, const0_rtx, true);
955 }
956
957 if (is_ubsan)
958 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
959
960 /* We assume both operands and result have the same precision
961 here (GET_MODE_BITSIZE (mode)), S stands for signed type
962 with that precision, U for unsigned type with that precision,
963 sgn for unsigned most significant bit in that precision.
964 s1 is signed first operand, u1 is unsigned first operand,
965 s2 is signed second operand, u2 is unsigned second operand,
966 sr is signed result, ur is unsigned result and the following
967 rules say how to compute result (which is always result of
968 the operands as if both were unsigned, cast to the right
969 signedness) and how to compute whether operation overflowed.
970 main_ovf (false) stands for jump on signed multiplication
971 overflow or the main algorithm with uns == false.
972 main_ovf (true) stands for jump on unsigned multiplication
973 overflow or the main algorithm with uns == true.
974
975 s1 * s2 -> sr
976 res = (S) ((U) s1 * (U) s2)
977 ovf = main_ovf (false)
978 u1 * u2 -> ur
979 res = u1 * u2
980 ovf = main_ovf (true)
981 s1 * u2 -> ur
982 res = (U) s1 * u2
983 ovf = (s1 < 0 && u2) || main_ovf (true)
984 u1 * u2 -> sr
985 res = (S) (u1 * u2)
986 ovf = res < 0 || main_ovf (true)
987 s1 * u2 -> sr
988 res = (S) ((U) s1 * u2)
989 ovf = (S) u2 >= 0 ? main_ovf (false)
990 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
991 s1 * s2 -> ur
992 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
993 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
994 res = t1 * t2
995 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
996
997 if (uns0_p && !uns1_p)
998 {
999 /* Multiplication is commutative, if operand signedness differs,
1000 canonicalize to the first operand being signed and second
1001 unsigned to simplify following code. */
1002 rtx tem = op1;
1003 op1 = op0;
1004 op0 = tem;
1005 tree t = arg1;
1006 arg1 = arg0;
1007 arg0 = t;
1008 uns0_p = 0;
1009 uns1_p = 1;
1010 }
1011
1012 int pos_neg0 = get_range_pos_neg (arg0);
1013 int pos_neg1 = get_range_pos_neg (arg1);
1014
1015 /* s1 * u2 -> ur */
1016 if (!uns0_p && uns1_p && unsr_p)
1017 {
1018 switch (pos_neg0)
1019 {
1020 case 1:
1021 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1022 goto do_main;
1023 case 2:
1024 /* If s1 is negative, avoid the main code, just multiply and
1025 signal overflow if op1 is not 0. */
1026 struct separate_ops ops;
1027 ops.code = MULT_EXPR;
1028 ops.type = TREE_TYPE (arg1);
1029 ops.op0 = make_tree (ops.type, op0);
1030 ops.op1 = make_tree (ops.type, op1);
1031 ops.op2 = NULL_TREE;
1032 ops.location = loc;
1033 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1034 emit_cmp_and_jump_insns (op1, const0_rtx, EQ, NULL_RTX, mode,
1035 false, done_label, PROB_VERY_LIKELY);
1036 goto do_error_label;
1037 case 3:
1038 rtx_code_label *do_main_label;
1039 do_main_label = gen_label_rtx ();
1040 emit_cmp_and_jump_insns (op0, const0_rtx, GE, NULL_RTX, mode,
1041 false, do_main_label, PROB_VERY_LIKELY);
1042 emit_cmp_and_jump_insns (op1, const0_rtx, EQ, NULL_RTX, mode,
1043 false, do_main_label, PROB_VERY_LIKELY);
1044 write_complex_part (target, const1_rtx, true);
1045 emit_label (do_main_label);
1046 goto do_main;
1047 default:
1048 gcc_unreachable ();
1049 }
1050 }
1051
1052 /* u1 * u2 -> sr */
1053 if (uns0_p && uns1_p && !unsr_p)
1054 {
1055 uns = true;
1056 /* Rest of handling of this case after res is computed. */
1057 goto do_main;
1058 }
1059
1060 /* s1 * u2 -> sr */
1061 if (!uns0_p && uns1_p && !unsr_p)
1062 {
1063 switch (pos_neg1)
1064 {
1065 case 1:
1066 goto do_main;
1067 case 2:
1068 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1069 avoid the main code, just multiply and signal overflow
1070 unless 0 * u2 or -1 * ((U) Smin). */
1071 struct separate_ops ops;
1072 ops.code = MULT_EXPR;
1073 ops.type = TREE_TYPE (arg1);
1074 ops.op0 = make_tree (ops.type, op0);
1075 ops.op1 = make_tree (ops.type, op1);
1076 ops.op2 = NULL_TREE;
1077 ops.location = loc;
1078 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1079 emit_cmp_and_jump_insns (op0, const0_rtx, EQ, NULL_RTX, mode,
1080 false, done_label, PROB_VERY_LIKELY);
1081 emit_cmp_and_jump_insns (op0, constm1_rtx, NE, NULL_RTX, mode,
1082 false, do_error, PROB_VERY_UNLIKELY);
1083 int prec;
1084 prec = GET_MODE_PRECISION (mode);
1085 rtx sgn;
1086 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1087 emit_cmp_and_jump_insns (op1, sgn, EQ, NULL_RTX, mode,
1088 false, done_label, PROB_VERY_LIKELY);
1089 goto do_error_label;
1090 case 3:
1091 /* Rest of handling of this case after res is computed. */
1092 goto do_main;
1093 default:
1094 gcc_unreachable ();
1095 }
1096 }
31e071ae 1097
1304953e
JJ		 1098 /* s1 * s2 -> ur */
1099 if (!uns0_p && !uns1_p && unsr_p)
1100 {
1101 rtx tem, tem2;
1102 switch (pos_neg0 | pos_neg1)
1103 {
1104 case 1: /* Both operands known to be non-negative. */
1105 goto do_main;
1106 case 2: /* Both operands known to be negative. */
1107 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1108 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1109 /* Avoid looking at arg0/arg1 ranges, as we've changed
1110 the arguments. */
1111 arg0 = error_mark_node;
1112 arg1 = error_mark_node;
1113 goto do_main;
1114 case 3:
1115 if ((pos_neg0 ^ pos_neg1) == 3)
1116 {
1117 /* If one operand is known to be negative and the other
1118 non-negative, this overflows always, unless the non-negative
1119 one is 0. Just do normal multiply and set overflow
1120 unless one of the operands is 0. */
1121 struct separate_ops ops;
1122 ops.code = MULT_EXPR;
1123 ops.type
1124 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1125 1);
1126 ops.op0 = make_tree (ops.type, op0);
1127 ops.op1 = make_tree (ops.type, op1);
1128 ops.op2 = NULL_TREE;
1129 ops.location = loc;
1130 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1131 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1132 OPTAB_LIB_WIDEN);
1133 emit_cmp_and_jump_insns (tem, const0_rtx, EQ, NULL_RTX, mode,
1134 false, done_label, PROB_VERY_LIKELY);
1135 goto do_error_label;
1136 }
1137 /* The general case, do all the needed comparisons at runtime. */
1138 rtx_code_label *do_main_label, *after_negate_label;
1139 rtx rop0, rop1;
1140 rop0 = gen_reg_rtx (mode);
1141 rop1 = gen_reg_rtx (mode);
1142 emit_move_insn (rop0, op0);
1143 emit_move_insn (rop1, op1);
1144 op0 = rop0;
1145 op1 = rop1;
1146 do_main_label = gen_label_rtx ();
1147 after_negate_label = gen_label_rtx ();
1148 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1149 OPTAB_LIB_WIDEN);
1150 emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false,
1151 after_negate_label, PROB_VERY_LIKELY);
1152 /* Both arguments negative here, negate them and continue with
1153 normal unsigned overflow checking multiplication. */
1154 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1155 NULL_RTX, false));
1156 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1157 NULL_RTX, false));
1158 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1159 the arguments. */
1160 arg0 = error_mark_node;
1161 arg1 = error_mark_node;
1162 emit_jump (do_main_label);
1163 emit_label (after_negate_label);
1164 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1165 OPTAB_LIB_WIDEN);
1166 emit_cmp_and_jump_insns (tem2, const0_rtx, GE, NULL_RTX, mode, false,
1167 do_main_label, PROB_VERY_LIKELY);
1168 /* One argument is negative here, the other positive. This
1169 overflows always, unless one of the arguments is 0. But
1170 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1171 is, thus we can keep do_main code oring in overflow as is. */
1172 emit_cmp_and_jump_insns (tem, const0_rtx, EQ, NULL_RTX, mode, false,
1173 do_main_label, PROB_VERY_LIKELY);
1174 write_complex_part (target, const1_rtx, true);
1175 emit_label (do_main_label);
1176 goto do_main;
1177 default:
1178 gcc_unreachable ();
1179 }
1180 }
1181
1182 do_main:
1183 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1184 sign = uns ? UNSIGNED : SIGNED;
1185 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
31e071ae
MP		 1186 if (icode != CODE_FOR_nothing)
1187 {
1188 struct expand_operand ops[4];
da664544 1189 rtx_insn *last = get_last_insn ();
31e071ae
MP		 1190
1191 res = gen_reg_rtx (mode);
1192 create_output_operand (&ops[0], res, mode);
1193 create_input_operand (&ops[1], op0, mode);
1194 create_input_operand (&ops[2], op1, mode);
1195 create_fixed_operand (&ops[3], do_error);
1196 if (maybe_expand_insn (icode, 4, ops))
1197 {
1198 last = get_last_insn ();
0a6a6ac9 1199 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
31e071ae
MP		 1200 && JUMP_P (last)
1201 && any_condjump_p (last)
1202 && !find_reg_note (last, REG_BR_PROB, 0))
1203 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
1204 emit_jump (done_label);
1205 }
1206 else
1207 {
1208 delete_insns_since (last);
1209 icode = CODE_FOR_nothing;
1210 }
1211 }
1212
1213 if (icode == CODE_FOR_nothing)
1214 {
1215 struct separate_ops ops;
1304953e
JJ		 1216 int prec = GET_MODE_PRECISION (mode);
1217 machine_mode hmode = mode_for_size (prec / 2, MODE_INT, 1);
1218 ops.op0 = make_tree (type, op0);
1219 ops.op1 = make_tree (type, op1);
31e071ae 1220 ops.op2 = NULL_TREE;
1304953e 1221 ops.location = loc;
31e071ae
MP		 1222 if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1223 && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
1224 {
ef4bddc2 1225 machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
31e071ae
MP		 1226 ops.code = WIDEN_MULT_EXPR;
1227 ops.type
1304953e 1228 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
31e071ae
MP		 1229
1230 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1304953e
JJ		 1231 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1232 NULL_RTX, uns);
31e071ae
MP		 1233 hipart = gen_lowpart (mode, hipart);
1234 res = gen_lowpart (mode, res);
1304953e
JJ		 1235 if (uns)
1236 /* For the unsigned multiplication, there was overflow if
1237 HIPART is non-zero. */
1238 emit_cmp_and_jump_insns (hipart, const0_rtx, EQ, NULL_RTX, mode,
1239 false, done_label, PROB_VERY_LIKELY);
1240 else
1241 {
1242 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1243 NULL_RTX, 0);
1244 /* RES is low half of the double width result, HIPART
1245 the high half. There was overflow if
1246 HIPART is different from RES < 0 ? -1 : 0. */
1247 emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
1248 false, done_label, PROB_VERY_LIKELY);
1249 }
31e071ae 1250 }
1304953e 1251 else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec)
d5fa9cc9 1252 {
da664544
DM		 1253 rtx_code_label *large_op0 = gen_label_rtx ();
1254 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1255 rtx_code_label *one_small_one_large = gen_label_rtx ();
1256 rtx_code_label *both_ops_large = gen_label_rtx ();
1304953e
JJ		 1257 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1258 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
da664544 1259 rtx_code_label *do_overflow = gen_label_rtx ();
1304953e 1260 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
d5fa9cc9 1261
807e902e 1262 unsigned int hprec = GET_MODE_PRECISION (hmode);
d5fa9cc9 1263 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1304953e 1264 NULL_RTX, uns);
d5fa9cc9
JJ		 1265 hipart0 = gen_lowpart (hmode, hipart0);
1266 rtx lopart0 = gen_lowpart (hmode, op0);
1304953e
JJ		 1267 rtx signbit0 = const0_rtx;
1268 if (!uns)
1269 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1270 NULL_RTX, 0);
d5fa9cc9 1271 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1304953e 1272 NULL_RTX, uns);
d5fa9cc9
JJ		 1273 hipart1 = gen_lowpart (hmode, hipart1);
1274 rtx lopart1 = gen_lowpart (hmode, op1);
1304953e
JJ		 1275 rtx signbit1 = const0_rtx;
1276 if (!uns)
1277 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1278 NULL_RTX, 0);
d5fa9cc9
JJ		 1279
1280 res = gen_reg_rtx (mode);
1281
1282 /* True if op0 resp. op1 are known to be in the range of
1283 halfstype. */
1284 bool op0_small_p = false;
1285 bool op1_small_p = false;
1286 /* True if op0 resp. op1 are known to have all zeros or all ones
1287 in the upper half of bits, but are not known to be
1288 op{0,1}_small_p. */
1289 bool op0_medium_p = false;
1290 bool op1_medium_p = false;
1291 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1292 nonnegative, 1 if unknown. */
1293 int op0_sign = 1;
1294 int op1_sign = 1;
1295
1304953e
JJ		 1296 if (pos_neg0 == 1)
1297 op0_sign = 0;
1298 else if (pos_neg0 == 2)
1299 op0_sign = -1;
1300 if (pos_neg1 == 1)
1301 op1_sign = 0;
1302 else if (pos_neg1 == 2)
1303 op1_sign = -1;
1304
1305 unsigned int mprec0 = prec;
1306 if (arg0 != error_mark_node)
1307 mprec0 = get_min_precision (arg0, sign);
1308 if (mprec0 <= hprec)
1309 op0_small_p = true;
1310 else if (!uns && mprec0 <= hprec + 1)
1311 op0_medium_p = true;
1312 unsigned int mprec1 = prec;
1313 if (arg1 != error_mark_node)
1314 mprec1 = get_min_precision (arg1, sign);
1315 if (mprec1 <= hprec)
1316 op1_small_p = true;
1317 else if (!uns && mprec1 <= hprec + 1)
1318 op1_medium_p = true;
d5fa9cc9
JJ		 1319
1320 int smaller_sign = 1;
1321 int larger_sign = 1;
1322 if (op0_small_p)
1323 {
1324 smaller_sign = op0_sign;
1325 larger_sign = op1_sign;
1326 }
1327 else if (op1_small_p)
1328 {
1329 smaller_sign = op1_sign;
1330 larger_sign = op0_sign;
1331 }
1332 else if (op0_sign == op1_sign)
1333 {
1334 smaller_sign = op0_sign;
1335 larger_sign = op0_sign;
1336 }
1337
1338 if (!op0_small_p)
1339 emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode,
1340 false, large_op0, PROB_UNLIKELY);
1341
1342 if (!op1_small_p)
1343 emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
1344 false, small_op0_large_op1,
1345 PROB_UNLIKELY);
1346
1304953e
JJ		 1347 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1348 hmode to mode, the multiplication will never overflow. We can
1349 do just one hmode x hmode => mode widening multiplication. */
1350 rtx lopart0s = lopart0, lopart1s = lopart1;
d5fa9cc9
JJ		 1351 if (GET_CODE (lopart0) == SUBREG)
1352 {
1304953e
JJ		 1353 lopart0s = shallow_copy_rtx (lopart0);
1354 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
1355 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
d5fa9cc9
JJ		 1356 }
1357 if (GET_CODE (lopart1) == SUBREG)
1358 {
1304953e
JJ		 1359 lopart1s = shallow_copy_rtx (lopart1);
1360 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
1361 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
d5fa9cc9 1362 }
1304953e
JJ		 1363 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1364 ops.op0 = make_tree (halfstype, lopart0s);
1365 ops.op1 = make_tree (halfstype, lopart1s);
d5fa9cc9 1366 ops.code = WIDEN_MULT_EXPR;
1304953e 1367 ops.type = type;
d5fa9cc9
JJ		 1368 rtx thisres
1369 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1370 emit_move_insn (res, thisres);
1371 emit_jump (done_label);
1372
1373 emit_label (small_op0_large_op1);
1374
1304953e
JJ		 1375 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1376 but op1 is not, just swap the arguments and handle it as op1
1377 sign/zero extended, op0 not. */
d5fa9cc9
JJ		 1378 rtx larger = gen_reg_rtx (mode);
1379 rtx hipart = gen_reg_rtx (hmode);
1380 rtx lopart = gen_reg_rtx (hmode);
1381 emit_move_insn (larger, op1);
1382 emit_move_insn (hipart, hipart1);
1383 emit_move_insn (lopart, lopart0);
1384 emit_jump (one_small_one_large);
1385
1386 emit_label (large_op0);
1387
1388 if (!op1_small_p)
1389 emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
1390 false, both_ops_large, PROB_UNLIKELY);
1391
1304953e
JJ		 1392 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1393 but op0 is not, prepare larger, hipart and lopart pseudos and
1394 handle it together with small_op0_large_op1. */
d5fa9cc9
JJ		 1395 emit_move_insn (larger, op0);
1396 emit_move_insn (hipart, hipart0);
1397 emit_move_insn (lopart, lopart1);
1398
1399 emit_label (one_small_one_large);
1400
1401 /* lopart is the low part of the operand that is sign extended
1402 to mode, larger is the the other operand, hipart is the
1403 high part of larger and lopart0 and lopart1 are the low parts
1404 of both operands.
1405 We perform lopart0 * lopart1 and lopart * hipart widening
1406 multiplications. */
1407 tree halfutype = build_nonstandard_integer_type (hprec, 1);
1408 ops.op0 = make_tree (halfutype, lopart0);
1409 ops.op1 = make_tree (halfutype, lopart1);
1410 rtx lo0xlo1
1411 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1412
1413 ops.op0 = make_tree (halfutype, lopart);
1414 ops.op1 = make_tree (halfutype, hipart);
1415 rtx loxhi = gen_reg_rtx (mode);
1416 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1417 emit_move_insn (loxhi, tem);
1418
1304953e
JJ		 1419 if (!uns)
1420 {
1421 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1422 if (larger_sign == 0)
1423 emit_jump (after_hipart_neg);
1424 else if (larger_sign != -1)
1425 emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX,
1426 hmode, false, after_hipart_neg,
1427 PROB_EVEN);
1428
1429 tem = convert_modes (mode, hmode, lopart, 1);
1430 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
1431 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
1432 1, OPTAB_DIRECT);
1433 emit_move_insn (loxhi, tem);
1434
1435 emit_label (after_hipart_neg);
1436
1437 /* if (lopart < 0) loxhi -= larger; */
1438 if (smaller_sign == 0)
1439 emit_jump (after_lopart_neg);
1440 else if (smaller_sign != -1)
1441 emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX,
1442 hmode, false, after_lopart_neg,
1443 PROB_EVEN);
1444
1445 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1446 1, OPTAB_DIRECT);
1447 emit_move_insn (loxhi, tem);
1448
1449 emit_label (after_lopart_neg);
1450 }
d5fa9cc9
JJ		 1451
1452 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1453 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
1454 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
1455 1, OPTAB_DIRECT);
1456 emit_move_insn (loxhi, tem);
1457
1458 /* if (loxhi >> (bitsize / 2)
1304953e
JJ		 1459 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1460 if (loxhi >> (bitsize / 2) == 0 (if uns). */
d5fa9cc9
JJ		 1461 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
1462 NULL_RTX, 0);
1463 hipartloxhi = gen_lowpart (hmode, hipartloxhi);
1304953e
JJ		 1464 rtx signbitloxhi = const0_rtx;
1465 if (!uns)
1466 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
1467 gen_lowpart (hmode, loxhi),
1468 hprec - 1, NULL_RTX, 0);
d5fa9cc9
JJ		 1469
1470 emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX,
1471 hmode, false, do_overflow,
1472 PROB_VERY_UNLIKELY);
1473
1474 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1475 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
1476 NULL_RTX, 1);
1477 tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1);
1478
1479 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
1480 1, OPTAB_DIRECT);
1481 if (tem != res)
1482 emit_move_insn (res, tem);
1483 emit_jump (done_label);
1484
1485 emit_label (both_ops_large);
1486
1304953e
JJ		 1487 /* If both operands are large (not sign (!uns) or zero (uns)
1488 extended from hmode), then perform the full multiplication
1489 which will be the result of the operation.
1490 The only cases which don't overflow are for signed multiplication
1491 some cases where both hipart0 and highpart1 are 0 or -1.
1492 For unsigned multiplication when high parts are both non-zero
1493 this overflows always. */
d5fa9cc9 1494 ops.code = MULT_EXPR;
1304953e
JJ		 1495 ops.op0 = make_tree (type, op0);
1496 ops.op1 = make_tree (type, op1);
d5fa9cc9
JJ		 1497 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1498 emit_move_insn (res, tem);
1499
1304953e 1500 if (!uns)
d5fa9cc9 1501 {
1304953e
JJ		 1502 if (!op0_medium_p)
1503 {
1504 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
1505 NULL_RTX, 1, OPTAB_DIRECT);
1506 emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX,
1507 hmode, true, do_error,
1508 PROB_VERY_UNLIKELY);
1509 }
d5fa9cc9 1510
1304953e
JJ		 1511 if (!op1_medium_p)
1512 {
1513 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
1514 NULL_RTX, 1, OPTAB_DIRECT);
1515 emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX,
1516 hmode, true, do_error,
1517 PROB_VERY_UNLIKELY);
1518 }
d5fa9cc9 1519
1304953e
JJ		 1520 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1521 the same, overflow happened if res is negative, if they are
1522 different, overflow happened if res is positive. */
1523 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
1524 emit_jump (hipart_different);
1525 else if (op0_sign == 1 || op1_sign == 1)
1526 emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode,
1527 true, hipart_different, PROB_EVEN);
d5fa9cc9 1528
1304953e
JJ		 1529 emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode,
1530 false, do_error, PROB_VERY_UNLIKELY);
1531 emit_jump (done_label);
d5fa9cc9 1532
1304953e
JJ		 1533 emit_label (hipart_different);
1534
1535 emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode,
1536 false, do_error, PROB_VERY_UNLIKELY);
1537 emit_jump (done_label);
1538 }
d5fa9cc9
JJ		 1539
1540 emit_label (do_overflow);
1541
1542 /* Overflow, do full multiplication and fallthru into do_error. */
1304953e
JJ		 1543 ops.op0 = make_tree (type, op0);
1544 ops.op1 = make_tree (type, op1);
d5fa9cc9
JJ		 1545 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1546 emit_move_insn (res, tem);
1547 }
31e071ae
MP		 1548 else
1549 {
1304953e 1550 gcc_assert (!is_ubsan);
31e071ae 1551 ops.code = MULT_EXPR;
1304953e 1552 ops.type = type;
31e071ae
MP		 1553 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1554 emit_jump (done_label);
1555 }
1556 }
1557
1304953e 1558 do_error_label:
31e071ae 1559 emit_label (do_error);
1304953e
JJ		 1560 if (is_ubsan)
1561 {
1562 /* Expand the ubsan builtin call. */
1563 push_temp_slots ();
1564 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
1565 arg0, arg1);
1566 expand_normal (fn);
1567 pop_temp_slots ();
1568 do_pending_stack_adjust ();
1569 }
1570 else if (lhs)
1571 write_complex_part (target, const1_rtx, true);
31e071ae
MP		 1572
1573 /* We're done. */
1574 emit_label (done_label);
1575
1304953e
JJ		 1576 /* u1 * u2 -> sr */
1577 if (uns0_p && uns1_p && !unsr_p)
1578 {
1579 rtx_code_label *all_done_label = gen_label_rtx ();
1580 emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode,
1581 false, all_done_label, PROB_VERY_LIKELY);
1582 write_complex_part (target, const1_rtx, true);
1583 emit_label (all_done_label);
1584 }
1585
1586 /* s1 * u2 -> sr */
1587 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
1588 {
1589 rtx_code_label *all_done_label = gen_label_rtx ();
1590 rtx_code_label *set_noovf = gen_label_rtx ();
1591 emit_cmp_and_jump_insns (op1, const0_rtx, GE, NULL_RTX, mode,
1592 false, all_done_label, PROB_VERY_LIKELY);
1593 write_complex_part (target, const1_rtx, true);
1594 emit_cmp_and_jump_insns (op0, const0_rtx, EQ, NULL_RTX, mode,
1595 false, set_noovf, PROB_VERY_LIKELY);
1596 emit_cmp_and_jump_insns (op0, constm1_rtx, NE, NULL_RTX, mode,
1597 false, all_done_label, PROB_VERY_UNLIKELY);
1598 emit_cmp_and_jump_insns (op1, res, NE, NULL_RTX, mode,
1599 false, all_done_label, PROB_VERY_UNLIKELY);
1600 emit_label (set_noovf);
1601 write_complex_part (target, const0_rtx, true);
1602 emit_label (all_done_label);
1603 }
1604
31e071ae 1605 if (lhs)
1304953e
JJ		 1606 {
1607 if (is_ubsan)
5620052d 1608 expand_ubsan_result_store (target, res);
1304953e
JJ		 1609 else
1610 expand_arith_overflow_result_store (lhs, target, mode, res);
1611 }
31e071ae
MP		 1612}
1613
1614/* Expand UBSAN_CHECK_ADD call STMT. */
1615
1616static void
1617expand_UBSAN_CHECK_ADD (gimple stmt)
1618{
1304953e
JJ		 1619 location_t loc = gimple_location (stmt);
1620 tree lhs = gimple_call_lhs (stmt);
1621 tree arg0 = gimple_call_arg (stmt, 0);
1622 tree arg1 = gimple_call_arg (stmt, 1);
1623 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
1624 false, false, false, true);
31e071ae
MP		 1625}
1626
1627/* Expand UBSAN_CHECK_SUB call STMT. */
1628
1629static void
1630expand_UBSAN_CHECK_SUB (gimple stmt)
1631{
1304953e
JJ		 1632 location_t loc = gimple_location (stmt);
1633 tree lhs = gimple_call_lhs (stmt);
1634 tree arg0 = gimple_call_arg (stmt, 0);
1635 tree arg1 = gimple_call_arg (stmt, 1);
1636 if (integer_zerop (arg0))
1637 expand_neg_overflow (loc, lhs, arg1, true);
31e071ae 1638 else
1304953e
JJ		 1639 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
1640 false, false, false, true);
31e071ae
MP		 1641}
1642
1643/* Expand UBSAN_CHECK_MUL call STMT. */
1644
1645static void
1646expand_UBSAN_CHECK_MUL (gimple stmt)
1647{
1304953e
JJ		 1648 location_t loc = gimple_location (stmt);
1649 tree lhs = gimple_call_lhs (stmt);
1650 tree arg0 = gimple_call_arg (stmt, 0);
1651 tree arg1 = gimple_call_arg (stmt, 1);
1652 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true);
1653}
1654
1655/* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1656
1657static void
1658expand_arith_overflow (enum tree_code code, gimple stmt)
1659{
1660 tree lhs = gimple_call_lhs (stmt);
1661 if (lhs == NULL_TREE)
1662 return;
1663 tree arg0 = gimple_call_arg (stmt, 0);
1664 tree arg1 = gimple_call_arg (stmt, 1);
1665 tree type = TREE_TYPE (TREE_TYPE (lhs));
1666 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
1667 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
1668 int unsr_p = TYPE_UNSIGNED (type);
1669 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
1670 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
1671 int precres = TYPE_PRECISION (type);
1672 location_t loc = gimple_location (stmt);
1673 if (!uns0_p && get_range_pos_neg (arg0) == 1)
1674 uns0_p = true;
1675 if (!uns1_p && get_range_pos_neg (arg1) == 1)
1676 uns1_p = true;
1677 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
1678 prec0 = MIN (prec0, pr);
1679 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
1680 prec1 = MIN (prec1, pr);
1681
1682 /* If uns0_p && uns1_p, precop is minimum needed precision
1683 of unsigned type to hold the exact result, otherwise
1684 precop is minimum needed precision of signed type to
1685 hold the exact result. */
1686 int precop;
1687 if (code == MULT_EXPR)
1688 precop = prec0 + prec1 + (uns0_p != uns1_p);
1689 else
1690 {
1691 if (uns0_p == uns1_p)
1692 precop = MAX (prec0, prec1) + 1;
1693 else if (uns0_p)
1694 precop = MAX (prec0 + 1, prec1) + 1;
1695 else
1696 precop = MAX (prec0, prec1 + 1) + 1;
1697 }
1698 int orig_precres = precres;
1699
1700 do
1701 {
1702 if ((uns0_p && uns1_p)
1703 ? ((precop + !unsr_p) <= precres
1704 /* u1 - u2 -> ur can overflow, no matter what precision
1705 the result has. */
1706 && (code != MINUS_EXPR || !unsr_p))
1707 : (!unsr_p && precop <= precres))
1708 {
1709 /* The infinity precision result will always fit into result. */
1710 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1711 write_complex_part (target, const0_rtx, true);
1712 enum machine_mode mode = TYPE_MODE (type);
1713 struct separate_ops ops;
1714 ops.code = code;
1715 ops.type = type;
1716 ops.op0 = fold_convert_loc (loc, type, arg0);
1717 ops.op1 = fold_convert_loc (loc, type, arg1);
1718 ops.op2 = NULL_TREE;
1719 ops.location = loc;
1720 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1721 expand_arith_overflow_result_store (lhs, target, mode, tem);
1722 return;
1723 }
1724
1725#ifdef WORD_REGISTER_OPERATIONS
1726 /* For sub-word operations, if target doesn't have them, start
1727 with precres widening right away, otherwise do it only
1728 if the most simple cases can't be used. */
1729 if (orig_precres == precres && precres < BITS_PER_WORD)
1730 ;
1731 else
1732#endif
1733 if ((uns0_p && uns1_p && unsr_p && prec0 <= precres && prec1 <= precres)
1734 || ((!uns0_p || !uns1_p) && !unsr_p
1735 && prec0 + uns0_p <= precres
1736 && prec1 + uns1_p <= precres))
1737 {
1738 arg0 = fold_convert_loc (loc, type, arg0);
1739 arg1 = fold_convert_loc (loc, type, arg1);
1740 switch (code)
1741 {
1742 case MINUS_EXPR:
1743 if (integer_zerop (arg0) && !unsr_p)
1744 expand_neg_overflow (loc, lhs, arg1, false);
1745 /* FALLTHRU */
1746 case PLUS_EXPR:
1747 expand_addsub_overflow (loc, code, lhs, arg0, arg1,
1748 unsr_p, unsr_p, unsr_p, false);
1749 return;
1750 case MULT_EXPR:
1751 expand_mul_overflow (loc, lhs, arg0, arg1,
1752 unsr_p, unsr_p, unsr_p, false);
1753 return;
1754 default:
1755 gcc_unreachable ();
1756 }
1757 }
1758
1759 /* For sub-word operations, retry with a wider type first. */
1760 if (orig_precres == precres && precop <= BITS_PER_WORD)
1761 {
1762#ifdef WORD_REGISTER_OPERATIONS
1763 int p = BITS_PER_WORD;
1764#else
1765 int p = precop;
1766#endif
1767 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1768 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1769 uns0_p && uns1_p
1770 && unsr_p);
1771 p = TYPE_PRECISION (optype);
1772 if (p > precres)
1773 {
1774 precres = p;
1775 unsr_p = TYPE_UNSIGNED (optype);
1776 type = optype;
1777 continue;
1778 }
1779 }
1780
1781 if (prec0 <= precres && prec1 <= precres)
1782 {
1783 tree types[2];
1784 if (unsr_p)
1785 {
1786 types[0] = build_nonstandard_integer_type (precres, 0);
1787 types[1] = type;
1788 }
1789 else
1790 {
1791 types[0] = type;
1792 types[1] = build_nonstandard_integer_type (precres, 1);
1793 }
1794 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
1795 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
1796 if (code != MULT_EXPR)
1797 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
1798 uns0_p, uns1_p, false);
1799 else
1800 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
1801 uns0_p, uns1_p, false);
1802 return;
1803 }
1804
1805 /* Retry with a wider type. */
1806 if (orig_precres == precres)
1807 {
1808 int p = MAX (prec0, prec1);
1809 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1810 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1811 uns0_p && uns1_p
1812 && unsr_p);
1813 p = TYPE_PRECISION (optype);
1814 if (p > precres)
1815 {
1816 precres = p;
1817 unsr_p = TYPE_UNSIGNED (optype);
1818 type = optype;
1819 continue;
1820 }
1821 }
1822
1823 gcc_unreachable ();
1824 }
1825 while (1);
1826}
1827
1828/* Expand ADD_OVERFLOW STMT. */
1829
1830static void
1831expand_ADD_OVERFLOW (gimple stmt)
1832{
1833 expand_arith_overflow (PLUS_EXPR, stmt);
1834}
1835
1836/* Expand SUB_OVERFLOW STMT. */
1837
1838static void
1839expand_SUB_OVERFLOW (gimple stmt)
1840{
1841 expand_arith_overflow (MINUS_EXPR, stmt);
1842}
1843
1844/* Expand MUL_OVERFLOW STMT. */
1845
1846static void
1847expand_MUL_OVERFLOW (gimple stmt)
1848{
1849 expand_arith_overflow (MULT_EXPR, stmt);
31e071ae
MP		 1850}
1851
5ce9450f
JJ		 1852/* This should get folded in tree-vectorizer.c. */
1853
1854static void
1855expand_LOOP_VECTORIZED (gimple stmt ATTRIBUTE_UNUSED)
1856{
1857 gcc_unreachable ();
1858}
1859
1860static void
1861expand_MASK_LOAD (gimple stmt)
1862{
1863 struct expand_operand ops[3];
1864 tree type, lhs, rhs, maskt;
1865 rtx mem, target, mask;
1866
1867 maskt = gimple_call_arg (stmt, 2);
1868 lhs = gimple_call_lhs (stmt);
8e91d222
JJ		 1869 if (lhs == NULL_TREE)
1870 return;
5ce9450f
JJ		 1871 type = TREE_TYPE (lhs);
1872 rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
1873 gimple_call_arg (stmt, 1));
1874
1875 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1876 gcc_assert (MEM_P (mem));
1877 mask = expand_normal (maskt);
1878 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1879 create_output_operand (&ops[0], target, TYPE_MODE (type));
1880 create_fixed_operand (&ops[1], mem);
1881 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1882 expand_insn (optab_handler (maskload_optab, TYPE_MODE (type)), 3, ops);
1883}
1884
1885static void
1886expand_MASK_STORE (gimple stmt)
1887{
1888 struct expand_operand ops[3];
1889 tree type, lhs, rhs, maskt;
1890 rtx mem, reg, mask;
1891
1892 maskt = gimple_call_arg (stmt, 2);
1893 rhs = gimple_call_arg (stmt, 3);
1894 type = TREE_TYPE (rhs);
1895 lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
1896 gimple_call_arg (stmt, 1));
1897
1898 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1899 gcc_assert (MEM_P (mem));
1900 mask = expand_normal (maskt);
1901 reg = expand_normal (rhs);
1902 create_fixed_operand (&ops[0], mem);
1903 create_input_operand (&ops[1], reg, TYPE_MODE (type));
1904 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1905 expand_insn (optab_handler (maskstore_optab, TYPE_MODE (type)), 3, ops);
1906}
1907
09b22f48
JJ		 1908static void
1909expand_ABNORMAL_DISPATCHER (gimple)
1910{
1911}
1912
ed9c79e1
JJ		 1913static void
1914expand_BUILTIN_EXPECT (gimple stmt)
1915{
1916 /* When guessing was done, the hints should be already stripped away. */
1917 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
1918
1919 rtx target;
1920 tree lhs = gimple_call_lhs (stmt);
1921 if (lhs)
1922 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1923 else
1924 target = const0_rtx;
1925 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
1926 if (lhs && val != target)
1927 emit_move_insn (target, val);
1928}
1929
25583c4f
RS		 1930/* Routines to expand each internal function, indexed by function number.
1931 Each routine has the prototype:
1932
1933 expand_<NAME> (gimple stmt)
1934
1935 where STMT is the statement that performs the call. */
1936static void (*const internal_fn_expanders[]) (gimple) = {
b78475cf 1937#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
25583c4f
RS		 1938#include "internal-fn.def"
1939#undef DEF_INTERNAL_FN
1940 0
1941};
1942
1943/* Expand STMT, which is a call to internal function FN. */
1944
1945void
1946expand_internal_call (gimple stmt)
1947{
1948 internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);
1949}
Mirror of https://gcc.gnu.org/git/gcc.git