]>
Commit | Line | Data |
---|---|---|
fb049fba | 1 | /* Internal functions. |
3aea1f79 | 2 | Copyright (C) 2011-2014 Free Software Foundation, Inc. |
fb049fba | 3 | |
4 | This file is part of GCC. | |
5 | ||
6 | GCC is free software; you can redistribute it and/or modify it under | |
7 | the terms of the GNU General Public License as published by the Free | |
8 | Software Foundation; either version 3, or (at your option) any later | |
9 | version. | |
10 | ||
11 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
14 | for more details. | |
15 | ||
16 | You should have received a copy of the GNU General Public License | |
17 | along with GCC; see the file COPYING3. If not see | |
18 | <http://www.gnu.org/licenses/>. */ | |
19 | ||
20 | #include "config.h" | |
21 | #include "system.h" | |
22 | #include "coretypes.h" | |
fb049fba | 23 | #include "tree.h" |
5ef6b660 | 24 | #include "internal-fn.h" |
9ed99284 | 25 | #include "stor-layout.h" |
fb049fba | 26 | #include "expr.h" |
34517c64 | 27 | #include "insn-codes.h" |
fb049fba | 28 | #include "optabs.h" |
94ea8568 | 29 | #include "predict.h" |
30 | #include "vec.h" | |
31 | #include "hashtab.h" | |
32 | #include "hash-set.h" | |
33 | #include "machmode.h" | |
34 | #include "tm.h" | |
35 | #include "hard-reg-set.h" | |
36 | #include "input.h" | |
37 | #include "function.h" | |
38 | #include "dominance.h" | |
39 | #include "cfg.h" | |
bc61cadb | 40 | #include "basic-block.h" |
41 | #include "tree-ssa-alias.h" | |
42 | #include "internal-fn.h" | |
43 | #include "gimple-expr.h" | |
44 | #include "is-a.h" | |
fb049fba | 45 | #include "gimple.h" |
137559b2 | 46 | #include "ubsan.h" |
47 | #include "target.h" | |
509e8fea | 48 | #include "stringpool.h" |
49 | #include "tree-ssanames.h" | |
c83059be | 50 | #include "diagnostic-core.h" |
fb049fba | 51 | |
52 | /* The names of each internal function, indexed by function number. */ | |
53 | const char *const internal_fn_name_array[] = { | |
64d5d3e8 | 54 | #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE, |
fb049fba | 55 | #include "internal-fn.def" |
56 | #undef DEF_INTERNAL_FN | |
57 | "<invalid-fn>" | |
58 | }; | |
59 | ||
60 | /* The ECF_* flags of each internal function, indexed by function number. */ | |
61 | const int internal_fn_flags_array[] = { | |
64d5d3e8 | 62 | #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS, |
fb049fba | 63 | #include "internal-fn.def" |
64 | #undef DEF_INTERNAL_FN | |
65 | 0 | |
66 | }; | |
67 | ||
64d5d3e8 | 68 | /* Fnspec of each internal function, indexed by function number. */ |
69 | const_tree internal_fn_fnspec_array[IFN_LAST + 1]; | |
70 | ||
71 | void | |
72 | init_internal_fns () | |
73 | { | |
74 | #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \ | |
75 | if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \ | |
1f788a0a | 76 | build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : ""); |
64d5d3e8 | 77 | #include "internal-fn.def" |
78 | #undef DEF_INTERNAL_FN | |
79 | internal_fn_fnspec_array[IFN_LAST] = 0; | |
80 | } | |
81 | ||
94b7b4dd | 82 | /* ARRAY_TYPE is an array of vector modes. Return the associated insn |
83 | for load-lanes-style optab OPTAB. The insn must exist. */ | |
84 | ||
85 | static enum insn_code | |
86 | get_multi_vector_move (tree array_type, convert_optab optab) | |
87 | { | |
88 | enum insn_code icode; | |
3754d046 | 89 | machine_mode imode; |
90 | machine_mode vmode; | |
94b7b4dd | 91 | |
92 | gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE); | |
93 | imode = TYPE_MODE (array_type); | |
94 | vmode = TYPE_MODE (TREE_TYPE (array_type)); | |
95 | ||
96 | icode = convert_optab_handler (optab, imode, vmode); | |
97 | gcc_assert (icode != CODE_FOR_nothing); | |
98 | return icode; | |
99 | } | |
100 | ||
101 | /* Expand LOAD_LANES call STMT. */ | |
102 | ||
103 | static void | |
1a91d914 | 104 | expand_LOAD_LANES (gcall *stmt) |
94b7b4dd | 105 | { |
106 | struct expand_operand ops[2]; | |
107 | tree type, lhs, rhs; | |
108 | rtx target, mem; | |
109 | ||
110 | lhs = gimple_call_lhs (stmt); | |
111 | rhs = gimple_call_arg (stmt, 0); | |
112 | type = TREE_TYPE (lhs); | |
113 | ||
114 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
115 | mem = expand_normal (rhs); | |
116 | ||
117 | gcc_assert (MEM_P (mem)); | |
118 | PUT_MODE (mem, TYPE_MODE (type)); | |
119 | ||
120 | create_output_operand (&ops[0], target, TYPE_MODE (type)); | |
121 | create_fixed_operand (&ops[1], mem); | |
122 | expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops); | |
123 | } | |
124 | ||
125 | /* Expand STORE_LANES call STMT. */ | |
126 | ||
127 | static void | |
1a91d914 | 128 | expand_STORE_LANES (gcall *stmt) |
94b7b4dd | 129 | { |
130 | struct expand_operand ops[2]; | |
131 | tree type, lhs, rhs; | |
132 | rtx target, reg; | |
133 | ||
134 | lhs = gimple_call_lhs (stmt); | |
135 | rhs = gimple_call_arg (stmt, 0); | |
136 | type = TREE_TYPE (rhs); | |
137 | ||
138 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
139 | reg = expand_normal (rhs); | |
140 | ||
141 | gcc_assert (MEM_P (target)); | |
142 | PUT_MODE (target, TYPE_MODE (type)); | |
143 | ||
144 | create_fixed_operand (&ops[0], target); | |
145 | create_input_operand (&ops[1], reg, TYPE_MODE (type)); | |
146 | expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops); | |
147 | } | |
148 | ||
4644b593 | 149 | static void |
1a91d914 | 150 | expand_ANNOTATE (gcall *stmt ATTRIBUTE_UNUSED) |
4644b593 | 151 | { |
152 | gcc_unreachable (); | |
153 | } | |
154 | ||
3d483a94 | 155 | /* This should get expanded in adjust_simduid_builtins. */ |
156 | ||
157 | static void | |
1a91d914 | 158 | expand_GOMP_SIMD_LANE (gcall *stmt ATTRIBUTE_UNUSED) |
3d483a94 | 159 | { |
160 | gcc_unreachable (); | |
161 | } | |
162 | ||
163 | /* This should get expanded in adjust_simduid_builtins. */ | |
164 | ||
165 | static void | |
1a91d914 | 166 | expand_GOMP_SIMD_VF (gcall *stmt ATTRIBUTE_UNUSED) |
3d483a94 | 167 | { |
168 | gcc_unreachable (); | |
169 | } | |
170 | ||
171 | /* This should get expanded in adjust_simduid_builtins. */ | |
172 | ||
173 | static void | |
1a91d914 | 174 | expand_GOMP_SIMD_LAST_LANE (gcall *stmt ATTRIBUTE_UNUSED) |
3d483a94 | 175 | { |
176 | gcc_unreachable (); | |
177 | } | |
178 | ||
19b928d9 | 179 | /* This should get expanded in the sanopt pass. */ |
180 | ||
181 | static void | |
1a91d914 | 182 | expand_UBSAN_NULL (gcall *stmt ATTRIBUTE_UNUSED) |
19b928d9 | 183 | { |
184 | gcc_unreachable (); | |
185 | } | |
186 | ||
5ef6b660 | 187 | /* This should get expanded in the sanopt pass. */ |
188 | ||
189 | static void | |
1a91d914 | 190 | expand_UBSAN_BOUNDS (gcall *stmt ATTRIBUTE_UNUSED) |
0b45f2d1 | 191 | { |
192 | gcc_unreachable (); | |
193 | } | |
194 | ||
195 | /* This should get expanded in the sanopt pass. */ | |
196 | ||
197 | static void | |
1a91d914 | 198 | expand_UBSAN_OBJECT_SIZE (gcall *stmt ATTRIBUTE_UNUSED) |
5ef6b660 | 199 | { |
200 | gcc_unreachable (); | |
201 | } | |
202 | ||
ff326078 | 203 | /* This should get expanded in the sanopt pass. */ |
204 | ||
205 | static void | |
1a91d914 | 206 | expand_ASAN_CHECK (gcall *stmt ATTRIBUTE_UNUSED) |
ff326078 | 207 | { |
208 | gcc_unreachable (); | |
209 | } | |
210 | ||
0c93c8a9 | 211 | /* Helper function for expand_addsub_overflow. Return 1 |
212 | if ARG interpreted as signed in its precision is known to be always | |
213 | positive or 2 if ARG is known to be always negative, or 3 if ARG may | |
214 | be positive or negative. */ | |
215 | ||
216 | static int | |
217 | get_range_pos_neg (tree arg) | |
218 | { | |
219 | if (arg == error_mark_node) | |
220 | return 3; | |
221 | ||
222 | int prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
223 | int cnt = 0; | |
224 | if (TREE_CODE (arg) == INTEGER_CST) | |
225 | { | |
226 | wide_int w = wi::sext (arg, prec); | |
227 | if (wi::neg_p (w)) | |
228 | return 2; | |
229 | else | |
230 | return 1; | |
231 | } | |
232 | while (CONVERT_EXPR_P (arg) | |
233 | && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0))) | |
234 | && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec) | |
235 | { | |
236 | arg = TREE_OPERAND (arg, 0); | |
237 | /* Narrower value zero extended into wider type | |
238 | will always result in positive values. */ | |
239 | if (TYPE_UNSIGNED (TREE_TYPE (arg)) | |
240 | && TYPE_PRECISION (TREE_TYPE (arg)) < prec) | |
241 | return 1; | |
242 | prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
243 | if (++cnt > 30) | |
244 | return 3; | |
245 | } | |
246 | ||
247 | if (TREE_CODE (arg) != SSA_NAME) | |
248 | return 3; | |
249 | wide_int arg_min, arg_max; | |
250 | while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE) | |
251 | { | |
252 | gimple g = SSA_NAME_DEF_STMT (arg); | |
253 | if (is_gimple_assign (g) | |
254 | && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g))) | |
255 | { | |
256 | tree t = gimple_assign_rhs1 (g); | |
257 | if (INTEGRAL_TYPE_P (TREE_TYPE (t)) | |
258 | && TYPE_PRECISION (TREE_TYPE (t)) <= prec) | |
259 | { | |
260 | if (TYPE_UNSIGNED (TREE_TYPE (t)) | |
261 | && TYPE_PRECISION (TREE_TYPE (t)) < prec) | |
262 | return 1; | |
263 | prec = TYPE_PRECISION (TREE_TYPE (t)); | |
264 | arg = t; | |
265 | if (++cnt > 30) | |
266 | return 3; | |
267 | continue; | |
268 | } | |
269 | } | |
270 | return 3; | |
271 | } | |
272 | if (TYPE_UNSIGNED (TREE_TYPE (arg))) | |
273 | { | |
274 | /* For unsigned values, the "positive" range comes | |
275 | below the "negative" range. */ | |
276 | if (!wi::neg_p (wi::sext (arg_max, prec), SIGNED)) | |
277 | return 1; | |
278 | if (wi::neg_p (wi::sext (arg_min, prec), SIGNED)) | |
279 | return 2; | |
280 | } | |
281 | else | |
282 | { | |
283 | if (!wi::neg_p (wi::sext (arg_min, prec), SIGNED)) | |
284 | return 1; | |
285 | if (wi::neg_p (wi::sext (arg_max, prec), SIGNED)) | |
286 | return 2; | |
287 | } | |
288 | return 3; | |
289 | } | |
290 | ||
291 | /* Return minimum precision needed to represent all values | |
292 | of ARG in SIGNed integral type. */ | |
293 | ||
294 | static int | |
295 | get_min_precision (tree arg, signop sign) | |
296 | { | |
297 | int prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
298 | int cnt = 0; | |
299 | signop orig_sign = sign; | |
300 | if (TREE_CODE (arg) == INTEGER_CST) | |
301 | { | |
302 | int p; | |
303 | if (TYPE_SIGN (TREE_TYPE (arg)) != sign) | |
304 | { | |
305 | widest_int w = wi::to_widest (arg); | |
306 | w = wi::ext (w, prec, sign); | |
307 | p = wi::min_precision (w, sign); | |
308 | } | |
309 | else | |
310 | p = wi::min_precision (arg, sign); | |
311 | return MIN (p, prec); | |
312 | } | |
313 | while (CONVERT_EXPR_P (arg) | |
314 | && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0))) | |
315 | && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec) | |
316 | { | |
317 | arg = TREE_OPERAND (arg, 0); | |
318 | if (TYPE_PRECISION (TREE_TYPE (arg)) < prec) | |
319 | { | |
320 | if (TYPE_UNSIGNED (TREE_TYPE (arg))) | |
321 | sign = UNSIGNED; | |
322 | else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1) | |
323 | return prec + (orig_sign != sign); | |
324 | prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
325 | } | |
326 | if (++cnt > 30) | |
327 | return prec + (orig_sign != sign); | |
328 | } | |
329 | if (TREE_CODE (arg) != SSA_NAME) | |
330 | return prec + (orig_sign != sign); | |
331 | wide_int arg_min, arg_max; | |
332 | while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE) | |
333 | { | |
334 | gimple g = SSA_NAME_DEF_STMT (arg); | |
335 | if (is_gimple_assign (g) | |
336 | && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g))) | |
337 | { | |
338 | tree t = gimple_assign_rhs1 (g); | |
339 | if (INTEGRAL_TYPE_P (TREE_TYPE (t)) | |
340 | && TYPE_PRECISION (TREE_TYPE (t)) <= prec) | |
341 | { | |
342 | arg = t; | |
343 | if (TYPE_PRECISION (TREE_TYPE (arg)) < prec) | |
344 | { | |
345 | if (TYPE_UNSIGNED (TREE_TYPE (arg))) | |
346 | sign = UNSIGNED; | |
347 | else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1) | |
348 | return prec + (orig_sign != sign); | |
349 | prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
350 | } | |
351 | if (++cnt > 30) | |
352 | return prec + (orig_sign != sign); | |
353 | continue; | |
354 | } | |
355 | } | |
356 | return prec + (orig_sign != sign); | |
357 | } | |
358 | if (sign == TYPE_SIGN (TREE_TYPE (arg))) | |
359 | { | |
360 | int p1 = wi::min_precision (arg_min, sign); | |
361 | int p2 = wi::min_precision (arg_max, sign); | |
362 | p1 = MAX (p1, p2); | |
363 | prec = MIN (prec, p1); | |
364 | } | |
365 | else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED)) | |
366 | { | |
367 | int p = wi::min_precision (arg_max, SIGNED); | |
368 | prec = MIN (prec, p); | |
369 | } | |
370 | return prec + (orig_sign != sign); | |
371 | } | |
372 | ||
373 | /* Helper for expand_*_overflow. Store RES into the __real__ part | |
374 | of TARGET. If RES has larger MODE than __real__ part of TARGET, | |
375 | set the __imag__ part to 1 if RES doesn't fit into it. */ | |
376 | ||
377 | static void | |
378 | expand_arith_overflow_result_store (tree lhs, rtx target, | |
379 | machine_mode mode, rtx res) | |
380 | { | |
381 | machine_mode tgtmode = GET_MODE_INNER (GET_MODE (target)); | |
382 | rtx lres = res; | |
383 | if (tgtmode != mode) | |
384 | { | |
385 | rtx_code_label *done_label = gen_label_rtx (); | |
386 | int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))); | |
387 | lres = convert_modes (tgtmode, mode, res, uns); | |
388 | gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode)); | |
389 | emit_cmp_and_jump_insns (res, convert_modes (mode, tgtmode, lres, uns), | |
390 | EQ, NULL_RTX, mode, false, done_label, | |
391 | PROB_VERY_LIKELY); | |
392 | write_complex_part (target, const1_rtx, true); | |
393 | emit_label (done_label); | |
394 | } | |
395 | write_complex_part (target, lres, false); | |
396 | } | |
397 | ||
920d99b4 | 398 | /* Helper for expand_*_overflow. Store RES into TARGET. */ |
399 | ||
400 | static void | |
401 | expand_ubsan_result_store (rtx target, rtx res) | |
402 | { | |
403 | if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target)) | |
404 | /* If this is a scalar in a register that is stored in a wider mode | |
405 | than the declared mode, compute the result into its declared mode | |
406 | and then convert to the wider mode. Our value is the computed | |
407 | expression. */ | |
408 | convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target)); | |
409 | else | |
410 | emit_move_insn (target, res); | |
411 | } | |
412 | ||
137559b2 | 413 | /* Add sub/add overflow checking to the statement STMT. |
414 | CODE says whether the operation is +, or -. */ | |
415 | ||
0c93c8a9 | 416 | static void |
417 | expand_addsub_overflow (location_t loc, tree_code code, tree lhs, | |
418 | tree arg0, tree arg1, bool unsr_p, bool uns0_p, | |
419 | bool uns1_p, bool is_ubsan) | |
137559b2 | 420 | { |
0c93c8a9 | 421 | rtx res, target = NULL_RTX; |
422 | tree fn; | |
423 | rtx_code_label *done_label = gen_label_rtx (); | |
424 | rtx_code_label *do_error = gen_label_rtx (); | |
137559b2 | 425 | do_pending_stack_adjust (); |
0c93c8a9 | 426 | rtx op0 = expand_normal (arg0); |
427 | rtx op1 = expand_normal (arg1); | |
3754d046 | 428 | machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); |
0c93c8a9 | 429 | int prec = GET_MODE_PRECISION (mode); |
430 | rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode); | |
431 | bool do_xor = false; | |
432 | ||
433 | if (is_ubsan) | |
434 | gcc_assert (!unsr_p && !uns0_p && !uns1_p); | |
435 | ||
137559b2 | 436 | if (lhs) |
0c93c8a9 | 437 | { |
438 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
439 | if (!is_ubsan) | |
440 | write_complex_part (target, const0_rtx, true); | |
441 | } | |
442 | ||
443 | /* We assume both operands and result have the same precision | |
444 | here (GET_MODE_BITSIZE (mode)), S stands for signed type | |
445 | with that precision, U for unsigned type with that precision, | |
446 | sgn for unsigned most significant bit in that precision. | |
447 | s1 is signed first operand, u1 is unsigned first operand, | |
448 | s2 is signed second operand, u2 is unsigned second operand, | |
449 | sr is signed result, ur is unsigned result and the following | |
450 | rules say how to compute result (which is always result of | |
451 | the operands as if both were unsigned, cast to the right | |
452 | signedness) and how to compute whether operation overflowed. | |
453 | ||
454 | s1 + s2 -> sr | |
455 | res = (S) ((U) s1 + (U) s2) | |
456 | ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow) | |
457 | s1 - s2 -> sr | |
458 | res = (S) ((U) s1 - (U) s2) | |
459 | ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow) | |
460 | u1 + u2 -> ur | |
461 | res = u1 + u2 | |
462 | ovf = res < u1 (or jump on carry, but RTL opts will handle it) | |
463 | u1 - u2 -> ur | |
464 | res = u1 - u2 | |
465 | ovf = res > u1 (or jump on carry, but RTL opts will handle it) | |
466 | s1 + u2 -> sr | |
467 | res = (S) ((U) s1 + u2) | |
468 | ovf = ((U) res ^ sgn) < u2 | |
469 | s1 + u2 -> ur | |
470 | t1 = (S) (u2 ^ sgn) | |
471 | t2 = s1 + t1 | |
472 | res = (U) t2 ^ sgn | |
473 | ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow) | |
474 | s1 - u2 -> sr | |
475 | res = (S) ((U) s1 - u2) | |
476 | ovf = u2 > ((U) s1 ^ sgn) | |
477 | s1 - u2 -> ur | |
478 | res = (U) s1 - u2 | |
479 | ovf = s1 < 0 || u2 > (U) s1 | |
480 | u1 - s2 -> sr | |
481 | res = u1 - (U) s2 | |
482 | ovf = u1 >= ((U) s2 ^ sgn) | |
483 | u1 - s2 -> ur | |
484 | t1 = u1 ^ sgn | |
485 | t2 = t1 - (U) s2 | |
486 | res = t2 ^ sgn | |
487 | ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow) | |
488 | s1 + s2 -> ur | |
489 | res = (U) s1 + (U) s2 | |
490 | ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0) | |
491 | u1 + u2 -> sr | |
492 | res = (S) (u1 + u2) | |
493 | ovf = (U) res < u2 || res < 0 | |
494 | u1 - u2 -> sr | |
495 | res = (S) (u1 - u2) | |
496 | ovf = u1 >= u2 ? res < 0 : res >= 0 | |
497 | s1 - s2 -> ur | |
498 | res = (U) s1 - (U) s2 | |
499 | ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */ | |
500 | ||
501 | if (code == PLUS_EXPR && uns0_p && !uns1_p) | |
502 | { | |
503 | /* PLUS_EXPR is commutative, if operand signedness differs, | |
504 | canonicalize to the first operand being signed and second | |
505 | unsigned to simplify following code. */ | |
506 | rtx tem = op1; | |
507 | op1 = op0; | |
508 | op0 = tem; | |
509 | tree t = arg1; | |
510 | arg1 = arg0; | |
511 | arg0 = t; | |
512 | uns0_p = 0; | |
513 | uns1_p = 1; | |
514 | } | |
515 | ||
516 | /* u1 +- u2 -> ur */ | |
517 | if (uns0_p && uns1_p && unsr_p) | |
518 | { | |
519 | /* Compute the operation. On RTL level, the addition is always | |
520 | unsigned. */ | |
521 | res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, | |
522 | op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); | |
523 | rtx tem = op0; | |
524 | /* For PLUS_EXPR, the operation is commutative, so we can pick | |
525 | operand to compare against. For prec <= BITS_PER_WORD, I think | |
526 | preferring REG operand is better over CONST_INT, because | |
527 | the CONST_INT might enlarge the instruction or CSE would need | |
528 | to figure out we'd already loaded it into a register before. | |
529 | For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial, | |
530 | as then the multi-word comparison can be perhaps simplified. */ | |
531 | if (code == PLUS_EXPR | |
532 | && (prec <= BITS_PER_WORD | |
533 | ? (CONST_SCALAR_INT_P (op0) && REG_P (op1)) | |
534 | : CONST_SCALAR_INT_P (op1))) | |
535 | tem = op1; | |
536 | emit_cmp_and_jump_insns (res, tem, code == PLUS_EXPR ? GEU : LEU, | |
537 | NULL_RTX, mode, false, done_label, | |
538 | PROB_VERY_LIKELY); | |
539 | goto do_error_label; | |
540 | } | |
541 | ||
542 | /* s1 +- u2 -> sr */ | |
543 | if (!uns0_p && uns1_p && !unsr_p) | |
544 | { | |
545 | /* Compute the operation. On RTL level, the addition is always | |
546 | unsigned. */ | |
547 | res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, | |
548 | op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); | |
549 | rtx tem = expand_binop (mode, add_optab, | |
550 | code == PLUS_EXPR ? res : op0, sgn, | |
551 | NULL_RTX, false, OPTAB_LIB_WIDEN); | |
552 | emit_cmp_and_jump_insns (tem, op1, GEU, NULL_RTX, mode, false, | |
553 | done_label, PROB_VERY_LIKELY); | |
554 | goto do_error_label; | |
555 | } | |
556 | ||
557 | /* s1 + u2 -> ur */ | |
558 | if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p) | |
559 | { | |
560 | op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false, | |
561 | OPTAB_LIB_WIDEN); | |
562 | /* As we've changed op1, we have to avoid using the value range | |
563 | for the original argument. */ | |
564 | arg1 = error_mark_node; | |
565 | do_xor = true; | |
566 | goto do_signed; | |
567 | } | |
568 | ||
569 | /* u1 - s2 -> ur */ | |
570 | if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p) | |
571 | { | |
572 | op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false, | |
573 | OPTAB_LIB_WIDEN); | |
574 | /* As we've changed op0, we have to avoid using the value range | |
575 | for the original argument. */ | |
576 | arg0 = error_mark_node; | |
577 | do_xor = true; | |
578 | goto do_signed; | |
579 | } | |
580 | ||
581 | /* s1 - u2 -> ur */ | |
582 | if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p) | |
583 | { | |
584 | /* Compute the operation. On RTL level, the addition is always | |
585 | unsigned. */ | |
586 | res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false, | |
587 | OPTAB_LIB_WIDEN); | |
588 | int pos_neg = get_range_pos_neg (arg0); | |
589 | if (pos_neg == 2) | |
590 | /* If ARG0 is known to be always negative, this is always overflow. */ | |
591 | emit_jump (do_error); | |
592 | else if (pos_neg == 3) | |
593 | /* If ARG0 is not known to be always positive, check at runtime. */ | |
594 | emit_cmp_and_jump_insns (op0, const0_rtx, LT, NULL_RTX, mode, false, | |
595 | do_error, PROB_VERY_UNLIKELY); | |
596 | emit_cmp_and_jump_insns (op1, op0, LEU, NULL_RTX, mode, false, | |
597 | done_label, PROB_VERY_LIKELY); | |
598 | goto do_error_label; | |
599 | } | |
600 | ||
601 | /* u1 - s2 -> sr */ | |
602 | if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p) | |
603 | { | |
604 | /* Compute the operation. On RTL level, the addition is always | |
605 | unsigned. */ | |
606 | res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false, | |
607 | OPTAB_LIB_WIDEN); | |
608 | rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false, | |
609 | OPTAB_LIB_WIDEN); | |
610 | emit_cmp_and_jump_insns (op0, tem, LTU, NULL_RTX, mode, false, | |
611 | done_label, PROB_VERY_LIKELY); | |
612 | goto do_error_label; | |
613 | } | |
614 | ||
615 | /* u1 + u2 -> sr */ | |
616 | if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p) | |
617 | { | |
618 | /* Compute the operation. On RTL level, the addition is always | |
619 | unsigned. */ | |
620 | res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false, | |
621 | OPTAB_LIB_WIDEN); | |
622 | emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false, | |
623 | do_error, PROB_VERY_UNLIKELY); | |
624 | rtx tem = op1; | |
625 | /* The operation is commutative, so we can pick operand to compare | |
626 | against. For prec <= BITS_PER_WORD, I think preferring REG operand | |
627 | is better over CONST_INT, because the CONST_INT might enlarge the | |
628 | instruction or CSE would need to figure out we'd already loaded it | |
629 | into a register before. For prec > BITS_PER_WORD, I think CONST_INT | |
630 | might be more beneficial, as then the multi-word comparison can be | |
631 | perhaps simplified. */ | |
632 | if (prec <= BITS_PER_WORD | |
633 | ? (CONST_SCALAR_INT_P (op1) && REG_P (op0)) | |
634 | : CONST_SCALAR_INT_P (op0)) | |
635 | tem = op0; | |
636 | emit_cmp_and_jump_insns (res, tem, GEU, NULL_RTX, mode, false, | |
637 | done_label, PROB_VERY_LIKELY); | |
638 | goto do_error_label; | |
639 | } | |
640 | ||
641 | /* s1 +- s2 -> ur */ | |
642 | if (!uns0_p && !uns1_p && unsr_p) | |
643 | { | |
644 | /* Compute the operation. On RTL level, the addition is always | |
645 | unsigned. */ | |
646 | res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, | |
647 | op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); | |
648 | int pos_neg = get_range_pos_neg (arg1); | |
649 | if (code == PLUS_EXPR) | |
650 | { | |
651 | int pos_neg0 = get_range_pos_neg (arg0); | |
652 | if (pos_neg0 != 3 && pos_neg == 3) | |
653 | { | |
654 | rtx tem = op1; | |
655 | op1 = op0; | |
656 | op0 = tem; | |
657 | pos_neg = pos_neg0; | |
658 | } | |
659 | } | |
660 | rtx tem; | |
661 | if (pos_neg != 3) | |
662 | { | |
663 | tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR)) | |
664 | ? and_optab : ior_optab, | |
665 | op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN); | |
666 | emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false, | |
667 | done_label, PROB_VERY_LIKELY); | |
668 | } | |
669 | else | |
670 | { | |
671 | rtx_code_label *do_ior_label = gen_label_rtx (); | |
672 | emit_cmp_and_jump_insns (op1, const0_rtx, | |
673 | code == MINUS_EXPR ? GE : LT, NULL_RTX, | |
674 | mode, false, do_ior_label, PROB_EVEN); | |
675 | tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false, | |
676 | OPTAB_LIB_WIDEN); | |
677 | emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false, | |
678 | done_label, PROB_VERY_LIKELY); | |
679 | emit_jump (do_error); | |
680 | emit_label (do_ior_label); | |
681 | tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false, | |
682 | OPTAB_LIB_WIDEN); | |
683 | emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false, | |
684 | done_label, PROB_VERY_LIKELY); | |
685 | } | |
686 | goto do_error_label; | |
687 | } | |
688 | ||
689 | /* u1 - u2 -> sr */ | |
690 | if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p) | |
691 | { | |
692 | /* Compute the operation. On RTL level, the addition is always | |
693 | unsigned. */ | |
694 | res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false, | |
695 | OPTAB_LIB_WIDEN); | |
696 | rtx_code_label *op0_geu_op1 = gen_label_rtx (); | |
697 | emit_cmp_and_jump_insns (op0, op1, GEU, NULL_RTX, mode, false, | |
698 | op0_geu_op1, PROB_EVEN); | |
699 | emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false, | |
700 | done_label, PROB_VERY_LIKELY); | |
701 | emit_jump (do_error); | |
702 | emit_label (op0_geu_op1); | |
703 | emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false, | |
704 | done_label, PROB_VERY_LIKELY); | |
705 | goto do_error_label; | |
706 | } | |
137559b2 | 707 | |
0c93c8a9 | 708 | gcc_assert (!uns0_p && !uns1_p && !unsr_p); |
709 | ||
710 | /* s1 +- s2 -> sr */ | |
711 | do_signed: ; | |
712 | enum insn_code icode; | |
713 | icode = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode); | |
137559b2 | 714 | if (icode != CODE_FOR_nothing) |
715 | { | |
716 | struct expand_operand ops[4]; | |
f8f996ed | 717 | rtx_insn *last = get_last_insn (); |
137559b2 | 718 | |
719 | res = gen_reg_rtx (mode); | |
720 | create_output_operand (&ops[0], res, mode); | |
721 | create_input_operand (&ops[1], op0, mode); | |
722 | create_input_operand (&ops[2], op1, mode); | |
723 | create_fixed_operand (&ops[3], do_error); | |
724 | if (maybe_expand_insn (icode, 4, ops)) | |
725 | { | |
726 | last = get_last_insn (); | |
f26d8580 | 727 | if (profile_status_for_fn (cfun) != PROFILE_ABSENT |
137559b2 | 728 | && JUMP_P (last) |
729 | && any_condjump_p (last) | |
730 | && !find_reg_note (last, REG_BR_PROB, 0)) | |
731 | add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); | |
732 | emit_jump (done_label); | |
733 | } | |
734 | else | |
735 | { | |
736 | delete_insns_since (last); | |
737 | icode = CODE_FOR_nothing; | |
738 | } | |
739 | } | |
740 | ||
741 | if (icode == CODE_FOR_nothing) | |
742 | { | |
f8f996ed | 743 | rtx_code_label *sub_check = gen_label_rtx (); |
509e8fea | 744 | int pos_neg = 3; |
137559b2 | 745 | |
746 | /* Compute the operation. On RTL level, the addition is always | |
747 | unsigned. */ | |
c8344e43 | 748 | res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, |
749 | op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); | |
137559b2 | 750 | |
c866ebee | 751 | /* If we can prove one of the arguments (for MINUS_EXPR only |
752 | the second operand, as subtraction is not commutative) is always | |
753 | non-negative or always negative, we can do just one comparison | |
754 | and conditional jump instead of 2 at runtime, 3 present in the | |
509e8fea | 755 | emitted code. If one of the arguments is CONST_INT, all we |
756 | need is to make sure it is op1, then the first | |
757 | emit_cmp_and_jump_insns will be just folded. Otherwise try | |
758 | to use range info if available. */ | |
c866ebee | 759 | if (code == PLUS_EXPR && CONST_INT_P (op0)) |
509e8fea | 760 | { |
761 | rtx tem = op0; | |
762 | op0 = op1; | |
763 | op1 = tem; | |
764 | } | |
765 | else if (CONST_INT_P (op1)) | |
766 | ; | |
c866ebee | 767 | else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME) |
509e8fea | 768 | { |
0c93c8a9 | 769 | pos_neg = get_range_pos_neg (arg0); |
509e8fea | 770 | if (pos_neg != 3) |
771 | { | |
772 | rtx tem = op0; | |
773 | op0 = op1; | |
774 | op1 = tem; | |
775 | } | |
776 | } | |
777 | if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME) | |
0c93c8a9 | 778 | pos_neg = get_range_pos_neg (arg1); |
509e8fea | 779 | |
137559b2 | 780 | /* If the op1 is negative, we have to use a different check. */ |
509e8fea | 781 | if (pos_neg == 3) |
782 | emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode, | |
783 | false, sub_check, PROB_EVEN); | |
137559b2 | 784 | |
c8344e43 | 785 | /* Compare the result of the operation with one of the operands. */ |
509e8fea | 786 | if (pos_neg & 1) |
787 | emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE, | |
788 | NULL_RTX, mode, false, done_label, | |
789 | PROB_VERY_LIKELY); | |
790 | ||
137559b2 | 791 | /* If we get here, we have to print the error. */ |
509e8fea | 792 | if (pos_neg == 3) |
793 | { | |
794 | emit_jump (do_error); | |
795 | ||
796 | emit_label (sub_check); | |
797 | } | |
137559b2 | 798 | |
137559b2 | 799 | /* We have k = a + b for b < 0 here. k <= a must hold. */ |
509e8fea | 800 | if (pos_neg & 2) |
801 | emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE, | |
802 | NULL_RTX, mode, false, done_label, | |
803 | PROB_VERY_LIKELY); | |
137559b2 | 804 | } |
805 | ||
0c93c8a9 | 806 | do_error_label: |
457e2f49 | 807 | emit_label (do_error); |
0c93c8a9 | 808 | if (is_ubsan) |
809 | { | |
810 | /* Expand the ubsan builtin call. */ | |
811 | push_temp_slots (); | |
812 | fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0), | |
813 | arg0, arg1); | |
814 | expand_normal (fn); | |
815 | pop_temp_slots (); | |
816 | do_pending_stack_adjust (); | |
817 | } | |
818 | else if (lhs) | |
819 | write_complex_part (target, const1_rtx, true); | |
137559b2 | 820 | |
457e2f49 | 821 | /* We're done. */ |
822 | emit_label (done_label); | |
137559b2 | 823 | |
824 | if (lhs) | |
0c93c8a9 | 825 | { |
826 | if (is_ubsan) | |
920d99b4 | 827 | expand_ubsan_result_store (target, res); |
0c93c8a9 | 828 | else |
829 | { | |
830 | if (do_xor) | |
831 | res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false, | |
832 | OPTAB_LIB_WIDEN); | |
833 | ||
834 | expand_arith_overflow_result_store (lhs, target, mode, res); | |
835 | } | |
836 | } | |
137559b2 | 837 | } |
838 | ||
839 | /* Add negate overflow checking to the statement STMT. */ | |
840 | ||
0c93c8a9 | 841 | static void |
842 | expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan) | |
137559b2 | 843 | { |
844 | rtx res, op1; | |
0c93c8a9 | 845 | tree fn; |
f8f996ed | 846 | rtx_code_label *done_label, *do_error; |
847 | rtx target = NULL_RTX; | |
137559b2 | 848 | |
137559b2 | 849 | done_label = gen_label_rtx (); |
850 | do_error = gen_label_rtx (); | |
137559b2 | 851 | |
852 | do_pending_stack_adjust (); | |
853 | op1 = expand_normal (arg1); | |
854 | ||
3754d046 | 855 | machine_mode mode = TYPE_MODE (TREE_TYPE (arg1)); |
137559b2 | 856 | if (lhs) |
0c93c8a9 | 857 | { |
858 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
859 | if (!is_ubsan) | |
860 | write_complex_part (target, const0_rtx, true); | |
861 | } | |
137559b2 | 862 | |
863 | enum insn_code icode = optab_handler (negv3_optab, mode); | |
864 | if (icode != CODE_FOR_nothing) | |
865 | { | |
866 | struct expand_operand ops[3]; | |
f8f996ed | 867 | rtx_insn *last = get_last_insn (); |
137559b2 | 868 | |
869 | res = gen_reg_rtx (mode); | |
870 | create_output_operand (&ops[0], res, mode); | |
871 | create_input_operand (&ops[1], op1, mode); | |
872 | create_fixed_operand (&ops[2], do_error); | |
873 | if (maybe_expand_insn (icode, 3, ops)) | |
874 | { | |
875 | last = get_last_insn (); | |
f26d8580 | 876 | if (profile_status_for_fn (cfun) != PROFILE_ABSENT |
137559b2 | 877 | && JUMP_P (last) |
878 | && any_condjump_p (last) | |
879 | && !find_reg_note (last, REG_BR_PROB, 0)) | |
880 | add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); | |
881 | emit_jump (done_label); | |
882 | } | |
883 | else | |
884 | { | |
885 | delete_insns_since (last); | |
886 | icode = CODE_FOR_nothing; | |
887 | } | |
888 | } | |
889 | ||
890 | if (icode == CODE_FOR_nothing) | |
891 | { | |
892 | /* Compute the operation. On RTL level, the addition is always | |
893 | unsigned. */ | |
894 | res = expand_unop (mode, neg_optab, op1, NULL_RTX, false); | |
895 | ||
896 | /* Compare the operand with the most negative value. */ | |
897 | rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1))); | |
898 | emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false, | |
899 | done_label, PROB_VERY_LIKELY); | |
900 | } | |
901 | ||
902 | emit_label (do_error); | |
0c93c8a9 | 903 | if (is_ubsan) |
904 | { | |
905 | /* Expand the ubsan builtin call. */ | |
906 | push_temp_slots (); | |
907 | fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1), | |
908 | arg1, NULL_TREE); | |
909 | expand_normal (fn); | |
910 | pop_temp_slots (); | |
911 | do_pending_stack_adjust (); | |
912 | } | |
913 | else if (lhs) | |
914 | write_complex_part (target, const1_rtx, true); | |
137559b2 | 915 | |
916 | /* We're done. */ | |
917 | emit_label (done_label); | |
918 | ||
919 | if (lhs) | |
0c93c8a9 | 920 | { |
921 | if (is_ubsan) | |
920d99b4 | 922 | expand_ubsan_result_store (target, res); |
0c93c8a9 | 923 | else |
924 | expand_arith_overflow_result_store (lhs, target, mode, res); | |
925 | } | |
137559b2 | 926 | } |
927 | ||
928 | /* Add mul overflow checking to the statement STMT. */ | |
929 | ||
0c93c8a9 | 930 | static void |
931 | expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, | |
932 | bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan) | |
137559b2 | 933 | { |
934 | rtx res, op0, op1; | |
0c93c8a9 | 935 | tree fn, type; |
f8f996ed | 936 | rtx_code_label *done_label, *do_error; |
937 | rtx target = NULL_RTX; | |
0c93c8a9 | 938 | signop sign; |
939 | enum insn_code icode; | |
137559b2 | 940 | |
137559b2 | 941 | done_label = gen_label_rtx (); |
942 | do_error = gen_label_rtx (); | |
137559b2 | 943 | |
944 | do_pending_stack_adjust (); | |
945 | op0 = expand_normal (arg0); | |
946 | op1 = expand_normal (arg1); | |
947 | ||
3754d046 | 948 | machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); |
0c93c8a9 | 949 | bool uns = unsr_p; |
137559b2 | 950 | if (lhs) |
0c93c8a9 | 951 | { |
952 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
953 | if (!is_ubsan) | |
954 | write_complex_part (target, const0_rtx, true); | |
955 | } | |
956 | ||
957 | if (is_ubsan) | |
958 | gcc_assert (!unsr_p && !uns0_p && !uns1_p); | |
959 | ||
960 | /* We assume both operands and result have the same precision | |
961 | here (GET_MODE_BITSIZE (mode)), S stands for signed type | |
962 | with that precision, U for unsigned type with that precision, | |
963 | sgn for unsigned most significant bit in that precision. | |
964 | s1 is signed first operand, u1 is unsigned first operand, | |
965 | s2 is signed second operand, u2 is unsigned second operand, | |
966 | sr is signed result, ur is unsigned result and the following | |
967 | rules say how to compute result (which is always result of | |
968 | the operands as if both were unsigned, cast to the right | |
969 | signedness) and how to compute whether operation overflowed. | |
970 | main_ovf (false) stands for jump on signed multiplication | |
971 | overflow or the main algorithm with uns == false. | |
972 | main_ovf (true) stands for jump on unsigned multiplication | |
973 | overflow or the main algorithm with uns == true. | |
974 | ||
975 | s1 * s2 -> sr | |
976 | res = (S) ((U) s1 * (U) s2) | |
977 | ovf = main_ovf (false) | |
978 | u1 * u2 -> ur | |
979 | res = u1 * u2 | |
980 | ovf = main_ovf (true) | |
981 | s1 * u2 -> ur | |
982 | res = (U) s1 * u2 | |
983 | ovf = (s1 < 0 && u2) || main_ovf (true) | |
984 | u1 * u2 -> sr | |
985 | res = (S) (u1 * u2) | |
986 | ovf = res < 0 || main_ovf (true) | |
987 | s1 * u2 -> sr | |
988 | res = (S) ((U) s1 * u2) | |
989 | ovf = (S) u2 >= 0 ? main_ovf (false) | |
990 | : (s1 != 0 && (s1 != -1 || u2 != (U) res)) | |
991 | s1 * s2 -> ur | |
992 | t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1) | |
993 | t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2) | |
994 | res = t1 * t2 | |
995 | ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */ | |
996 | ||
997 | if (uns0_p && !uns1_p) | |
998 | { | |
999 | /* Multiplication is commutative, if operand signedness differs, | |
1000 | canonicalize to the first operand being signed and second | |
1001 | unsigned to simplify following code. */ | |
1002 | rtx tem = op1; | |
1003 | op1 = op0; | |
1004 | op0 = tem; | |
1005 | tree t = arg1; | |
1006 | arg1 = arg0; | |
1007 | arg0 = t; | |
1008 | uns0_p = 0; | |
1009 | uns1_p = 1; | |
1010 | } | |
1011 | ||
1012 | int pos_neg0 = get_range_pos_neg (arg0); | |
1013 | int pos_neg1 = get_range_pos_neg (arg1); | |
1014 | ||
1015 | /* s1 * u2 -> ur */ | |
1016 | if (!uns0_p && uns1_p && unsr_p) | |
1017 | { | |
1018 | switch (pos_neg0) | |
1019 | { | |
1020 | case 1: | |
1021 | /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */ | |
1022 | goto do_main; | |
1023 | case 2: | |
1024 | /* If s1 is negative, avoid the main code, just multiply and | |
1025 | signal overflow if op1 is not 0. */ | |
1026 | struct separate_ops ops; | |
1027 | ops.code = MULT_EXPR; | |
1028 | ops.type = TREE_TYPE (arg1); | |
1029 | ops.op0 = make_tree (ops.type, op0); | |
1030 | ops.op1 = make_tree (ops.type, op1); | |
1031 | ops.op2 = NULL_TREE; | |
1032 | ops.location = loc; | |
1033 | res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1034 | emit_cmp_and_jump_insns (op1, const0_rtx, EQ, NULL_RTX, mode, | |
1035 | false, done_label, PROB_VERY_LIKELY); | |
1036 | goto do_error_label; | |
1037 | case 3: | |
1038 | rtx_code_label *do_main_label; | |
1039 | do_main_label = gen_label_rtx (); | |
1040 | emit_cmp_and_jump_insns (op0, const0_rtx, GE, NULL_RTX, mode, | |
1041 | false, do_main_label, PROB_VERY_LIKELY); | |
1042 | emit_cmp_and_jump_insns (op1, const0_rtx, EQ, NULL_RTX, mode, | |
1043 | false, do_main_label, PROB_VERY_LIKELY); | |
1044 | write_complex_part (target, const1_rtx, true); | |
1045 | emit_label (do_main_label); | |
1046 | goto do_main; | |
1047 | default: | |
1048 | gcc_unreachable (); | |
1049 | } | |
1050 | } | |
1051 | ||
1052 | /* u1 * u2 -> sr */ | |
1053 | if (uns0_p && uns1_p && !unsr_p) | |
1054 | { | |
1055 | uns = true; | |
1056 | /* Rest of handling of this case after res is computed. */ | |
1057 | goto do_main; | |
1058 | } | |
1059 | ||
1060 | /* s1 * u2 -> sr */ | |
1061 | if (!uns0_p && uns1_p && !unsr_p) | |
1062 | { | |
1063 | switch (pos_neg1) | |
1064 | { | |
1065 | case 1: | |
1066 | goto do_main; | |
1067 | case 2: | |
1068 | /* If (S) u2 is negative (i.e. u2 is larger than maximum of S, | |
1069 | avoid the main code, just multiply and signal overflow | |
1070 | unless 0 * u2 or -1 * ((U) Smin). */ | |
1071 | struct separate_ops ops; | |
1072 | ops.code = MULT_EXPR; | |
1073 | ops.type = TREE_TYPE (arg1); | |
1074 | ops.op0 = make_tree (ops.type, op0); | |
1075 | ops.op1 = make_tree (ops.type, op1); | |
1076 | ops.op2 = NULL_TREE; | |
1077 | ops.location = loc; | |
1078 | res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1079 | emit_cmp_and_jump_insns (op0, const0_rtx, EQ, NULL_RTX, mode, | |
1080 | false, done_label, PROB_VERY_LIKELY); | |
1081 | emit_cmp_and_jump_insns (op0, constm1_rtx, NE, NULL_RTX, mode, | |
1082 | false, do_error, PROB_VERY_UNLIKELY); | |
1083 | int prec; | |
1084 | prec = GET_MODE_PRECISION (mode); | |
1085 | rtx sgn; | |
1086 | sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode); | |
1087 | emit_cmp_and_jump_insns (op1, sgn, EQ, NULL_RTX, mode, | |
1088 | false, done_label, PROB_VERY_LIKELY); | |
1089 | goto do_error_label; | |
1090 | case 3: | |
1091 | /* Rest of handling of this case after res is computed. */ | |
1092 | goto do_main; | |
1093 | default: | |
1094 | gcc_unreachable (); | |
1095 | } | |
1096 | } | |
137559b2 | 1097 | |
0c93c8a9 | 1098 | /* s1 * s2 -> ur */ |
1099 | if (!uns0_p && !uns1_p && unsr_p) | |
1100 | { | |
1101 | rtx tem, tem2; | |
1102 | switch (pos_neg0 | pos_neg1) | |
1103 | { | |
1104 | case 1: /* Both operands known to be non-negative. */ | |
1105 | goto do_main; | |
1106 | case 2: /* Both operands known to be negative. */ | |
1107 | op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false); | |
1108 | op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false); | |
1109 | /* Avoid looking at arg0/arg1 ranges, as we've changed | |
1110 | the arguments. */ | |
1111 | arg0 = error_mark_node; | |
1112 | arg1 = error_mark_node; | |
1113 | goto do_main; | |
1114 | case 3: | |
1115 | if ((pos_neg0 ^ pos_neg1) == 3) | |
1116 | { | |
1117 | /* If one operand is known to be negative and the other | |
1118 | non-negative, this overflows always, unless the non-negative | |
1119 | one is 0. Just do normal multiply and set overflow | |
1120 | unless one of the operands is 0. */ | |
1121 | struct separate_ops ops; | |
1122 | ops.code = MULT_EXPR; | |
1123 | ops.type | |
1124 | = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), | |
1125 | 1); | |
1126 | ops.op0 = make_tree (ops.type, op0); | |
1127 | ops.op1 = make_tree (ops.type, op1); | |
1128 | ops.op2 = NULL_TREE; | |
1129 | ops.location = loc; | |
1130 | res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1131 | tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false, | |
1132 | OPTAB_LIB_WIDEN); | |
1133 | emit_cmp_and_jump_insns (tem, const0_rtx, EQ, NULL_RTX, mode, | |
1134 | false, done_label, PROB_VERY_LIKELY); | |
1135 | goto do_error_label; | |
1136 | } | |
1137 | /* The general case, do all the needed comparisons at runtime. */ | |
1138 | rtx_code_label *do_main_label, *after_negate_label; | |
1139 | rtx rop0, rop1; | |
1140 | rop0 = gen_reg_rtx (mode); | |
1141 | rop1 = gen_reg_rtx (mode); | |
1142 | emit_move_insn (rop0, op0); | |
1143 | emit_move_insn (rop1, op1); | |
1144 | op0 = rop0; | |
1145 | op1 = rop1; | |
1146 | do_main_label = gen_label_rtx (); | |
1147 | after_negate_label = gen_label_rtx (); | |
1148 | tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false, | |
1149 | OPTAB_LIB_WIDEN); | |
1150 | emit_cmp_and_jump_insns (tem, const0_rtx, GE, NULL_RTX, mode, false, | |
1151 | after_negate_label, PROB_VERY_LIKELY); | |
1152 | /* Both arguments negative here, negate them and continue with | |
1153 | normal unsigned overflow checking multiplication. */ | |
1154 | emit_move_insn (op0, expand_unop (mode, neg_optab, op0, | |
1155 | NULL_RTX, false)); | |
1156 | emit_move_insn (op1, expand_unop (mode, neg_optab, op1, | |
1157 | NULL_RTX, false)); | |
1158 | /* Avoid looking at arg0/arg1 ranges, as we might have changed | |
1159 | the arguments. */ | |
1160 | arg0 = error_mark_node; | |
1161 | arg1 = error_mark_node; | |
1162 | emit_jump (do_main_label); | |
1163 | emit_label (after_negate_label); | |
1164 | tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false, | |
1165 | OPTAB_LIB_WIDEN); | |
1166 | emit_cmp_and_jump_insns (tem2, const0_rtx, GE, NULL_RTX, mode, false, | |
1167 | do_main_label, PROB_VERY_LIKELY); | |
1168 | /* One argument is negative here, the other positive. This | |
1169 | overflows always, unless one of the arguments is 0. But | |
1170 | if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1 | |
1171 | is, thus we can keep do_main code oring in overflow as is. */ | |
1172 | emit_cmp_and_jump_insns (tem, const0_rtx, EQ, NULL_RTX, mode, false, | |
1173 | do_main_label, PROB_VERY_LIKELY); | |
1174 | write_complex_part (target, const1_rtx, true); | |
1175 | emit_label (do_main_label); | |
1176 | goto do_main; | |
1177 | default: | |
1178 | gcc_unreachable (); | |
1179 | } | |
1180 | } | |
1181 | ||
1182 | do_main: | |
1183 | type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns); | |
1184 | sign = uns ? UNSIGNED : SIGNED; | |
1185 | icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode); | |
137559b2 | 1186 | if (icode != CODE_FOR_nothing) |
1187 | { | |
1188 | struct expand_operand ops[4]; | |
f8f996ed | 1189 | rtx_insn *last = get_last_insn (); |
137559b2 | 1190 | |
1191 | res = gen_reg_rtx (mode); | |
1192 | create_output_operand (&ops[0], res, mode); | |
1193 | create_input_operand (&ops[1], op0, mode); | |
1194 | create_input_operand (&ops[2], op1, mode); | |
1195 | create_fixed_operand (&ops[3], do_error); | |
1196 | if (maybe_expand_insn (icode, 4, ops)) | |
1197 | { | |
1198 | last = get_last_insn (); | |
f26d8580 | 1199 | if (profile_status_for_fn (cfun) != PROFILE_ABSENT |
137559b2 | 1200 | && JUMP_P (last) |
1201 | && any_condjump_p (last) | |
1202 | && !find_reg_note (last, REG_BR_PROB, 0)) | |
1203 | add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); | |
1204 | emit_jump (done_label); | |
1205 | } | |
1206 | else | |
1207 | { | |
1208 | delete_insns_since (last); | |
1209 | icode = CODE_FOR_nothing; | |
1210 | } | |
1211 | } | |
1212 | ||
1213 | if (icode == CODE_FOR_nothing) | |
1214 | { | |
1215 | struct separate_ops ops; | |
0c93c8a9 | 1216 | int prec = GET_MODE_PRECISION (mode); |
1217 | machine_mode hmode = mode_for_size (prec / 2, MODE_INT, 1); | |
1218 | ops.op0 = make_tree (type, op0); | |
1219 | ops.op1 = make_tree (type, op1); | |
137559b2 | 1220 | ops.op2 = NULL_TREE; |
0c93c8a9 | 1221 | ops.location = loc; |
137559b2 | 1222 | if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode |
1223 | && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode))) | |
1224 | { | |
3754d046 | 1225 | machine_mode wmode = GET_MODE_2XWIDER_MODE (mode); |
137559b2 | 1226 | ops.code = WIDEN_MULT_EXPR; |
1227 | ops.type | |
0c93c8a9 | 1228 | = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns); |
137559b2 | 1229 | |
1230 | res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL); | |
0c93c8a9 | 1231 | rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec, |
1232 | NULL_RTX, uns); | |
137559b2 | 1233 | hipart = gen_lowpart (mode, hipart); |
1234 | res = gen_lowpart (mode, res); | |
0c93c8a9 | 1235 | if (uns) |
1236 | /* For the unsigned multiplication, there was overflow if | |
1237 | HIPART is non-zero. */ | |
1238 | emit_cmp_and_jump_insns (hipart, const0_rtx, EQ, NULL_RTX, mode, | |
1239 | false, done_label, PROB_VERY_LIKELY); | |
1240 | else | |
1241 | { | |
1242 | rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1, | |
1243 | NULL_RTX, 0); | |
1244 | /* RES is low half of the double width result, HIPART | |
1245 | the high half. There was overflow if | |
1246 | HIPART is different from RES < 0 ? -1 : 0. */ | |
1247 | emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode, | |
1248 | false, done_label, PROB_VERY_LIKELY); | |
1249 | } | |
137559b2 | 1250 | } |
0c93c8a9 | 1251 | else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec) |
1ee16627 | 1252 | { |
f8f996ed | 1253 | rtx_code_label *large_op0 = gen_label_rtx (); |
1254 | rtx_code_label *small_op0_large_op1 = gen_label_rtx (); | |
1255 | rtx_code_label *one_small_one_large = gen_label_rtx (); | |
1256 | rtx_code_label *both_ops_large = gen_label_rtx (); | |
0c93c8a9 | 1257 | rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx (); |
1258 | rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx (); | |
f8f996ed | 1259 | rtx_code_label *do_overflow = gen_label_rtx (); |
0c93c8a9 | 1260 | rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx (); |
1ee16627 | 1261 | |
265815a2 | 1262 | unsigned int hprec = GET_MODE_PRECISION (hmode); |
1ee16627 | 1263 | rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec, |
0c93c8a9 | 1264 | NULL_RTX, uns); |
1ee16627 | 1265 | hipart0 = gen_lowpart (hmode, hipart0); |
1266 | rtx lopart0 = gen_lowpart (hmode, op0); | |
0c93c8a9 | 1267 | rtx signbit0 = const0_rtx; |
1268 | if (!uns) | |
1269 | signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1, | |
1270 | NULL_RTX, 0); | |
1ee16627 | 1271 | rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec, |
0c93c8a9 | 1272 | NULL_RTX, uns); |
1ee16627 | 1273 | hipart1 = gen_lowpart (hmode, hipart1); |
1274 | rtx lopart1 = gen_lowpart (hmode, op1); | |
0c93c8a9 | 1275 | rtx signbit1 = const0_rtx; |
1276 | if (!uns) | |
1277 | signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1, | |
1278 | NULL_RTX, 0); | |
1ee16627 | 1279 | |
1280 | res = gen_reg_rtx (mode); | |
1281 | ||
1282 | /* True if op0 resp. op1 are known to be in the range of | |
1283 | halfstype. */ | |
1284 | bool op0_small_p = false; | |
1285 | bool op1_small_p = false; | |
1286 | /* True if op0 resp. op1 are known to have all zeros or all ones | |
1287 | in the upper half of bits, but are not known to be | |
1288 | op{0,1}_small_p. */ | |
1289 | bool op0_medium_p = false; | |
1290 | bool op1_medium_p = false; | |
1291 | /* -1 if op{0,1} is known to be negative, 0 if it is known to be | |
1292 | nonnegative, 1 if unknown. */ | |
1293 | int op0_sign = 1; | |
1294 | int op1_sign = 1; | |
1295 | ||
0c93c8a9 | 1296 | if (pos_neg0 == 1) |
1297 | op0_sign = 0; | |
1298 | else if (pos_neg0 == 2) | |
1299 | op0_sign = -1; | |
1300 | if (pos_neg1 == 1) | |
1301 | op1_sign = 0; | |
1302 | else if (pos_neg1 == 2) | |
1303 | op1_sign = -1; | |
1304 | ||
1305 | unsigned int mprec0 = prec; | |
1306 | if (arg0 != error_mark_node) | |
1307 | mprec0 = get_min_precision (arg0, sign); | |
1308 | if (mprec0 <= hprec) | |
1309 | op0_small_p = true; | |
1310 | else if (!uns && mprec0 <= hprec + 1) | |
1311 | op0_medium_p = true; | |
1312 | unsigned int mprec1 = prec; | |
1313 | if (arg1 != error_mark_node) | |
1314 | mprec1 = get_min_precision (arg1, sign); | |
1315 | if (mprec1 <= hprec) | |
1316 | op1_small_p = true; | |
1317 | else if (!uns && mprec1 <= hprec + 1) | |
1318 | op1_medium_p = true; | |
1ee16627 | 1319 | |
1320 | int smaller_sign = 1; | |
1321 | int larger_sign = 1; | |
1322 | if (op0_small_p) | |
1323 | { | |
1324 | smaller_sign = op0_sign; | |
1325 | larger_sign = op1_sign; | |
1326 | } | |
1327 | else if (op1_small_p) | |
1328 | { | |
1329 | smaller_sign = op1_sign; | |
1330 | larger_sign = op0_sign; | |
1331 | } | |
1332 | else if (op0_sign == op1_sign) | |
1333 | { | |
1334 | smaller_sign = op0_sign; | |
1335 | larger_sign = op0_sign; | |
1336 | } | |
1337 | ||
1338 | if (!op0_small_p) | |
1339 | emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode, | |
1340 | false, large_op0, PROB_UNLIKELY); | |
1341 | ||
1342 | if (!op1_small_p) | |
1343 | emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, | |
1344 | false, small_op0_large_op1, | |
1345 | PROB_UNLIKELY); | |
1346 | ||
0c93c8a9 | 1347 | /* If both op0 and op1 are sign (!uns) or zero (uns) extended from |
1348 | hmode to mode, the multiplication will never overflow. We can | |
1349 | do just one hmode x hmode => mode widening multiplication. */ | |
1350 | rtx lopart0s = lopart0, lopart1s = lopart1; | |
1ee16627 | 1351 | if (GET_CODE (lopart0) == SUBREG) |
1352 | { | |
0c93c8a9 | 1353 | lopart0s = shallow_copy_rtx (lopart0); |
1354 | SUBREG_PROMOTED_VAR_P (lopart0s) = 1; | |
1355 | SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED); | |
1ee16627 | 1356 | } |
1357 | if (GET_CODE (lopart1) == SUBREG) | |
1358 | { | |
0c93c8a9 | 1359 | lopart1s = shallow_copy_rtx (lopart1); |
1360 | SUBREG_PROMOTED_VAR_P (lopart1s) = 1; | |
1361 | SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED); | |
1ee16627 | 1362 | } |
0c93c8a9 | 1363 | tree halfstype = build_nonstandard_integer_type (hprec, uns); |
1364 | ops.op0 = make_tree (halfstype, lopart0s); | |
1365 | ops.op1 = make_tree (halfstype, lopart1s); | |
1ee16627 | 1366 | ops.code = WIDEN_MULT_EXPR; |
0c93c8a9 | 1367 | ops.type = type; |
1ee16627 | 1368 | rtx thisres |
1369 | = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1370 | emit_move_insn (res, thisres); | |
1371 | emit_jump (done_label); | |
1372 | ||
1373 | emit_label (small_op0_large_op1); | |
1374 | ||
0c93c8a9 | 1375 | /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode, |
1376 | but op1 is not, just swap the arguments and handle it as op1 | |
1377 | sign/zero extended, op0 not. */ | |
1ee16627 | 1378 | rtx larger = gen_reg_rtx (mode); |
1379 | rtx hipart = gen_reg_rtx (hmode); | |
1380 | rtx lopart = gen_reg_rtx (hmode); | |
1381 | emit_move_insn (larger, op1); | |
1382 | emit_move_insn (hipart, hipart1); | |
1383 | emit_move_insn (lopart, lopart0); | |
1384 | emit_jump (one_small_one_large); | |
1385 | ||
1386 | emit_label (large_op0); | |
1387 | ||
1388 | if (!op1_small_p) | |
1389 | emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, | |
1390 | false, both_ops_large, PROB_UNLIKELY); | |
1391 | ||
0c93c8a9 | 1392 | /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode, |
1393 | but op0 is not, prepare larger, hipart and lopart pseudos and | |
1394 | handle it together with small_op0_large_op1. */ | |
1ee16627 | 1395 | emit_move_insn (larger, op0); |
1396 | emit_move_insn (hipart, hipart0); | |
1397 | emit_move_insn (lopart, lopart1); | |
1398 | ||
1399 | emit_label (one_small_one_large); | |
1400 | ||
1401 | /* lopart is the low part of the operand that is sign extended | |
1402 | to mode, larger is the the other operand, hipart is the | |
1403 | high part of larger and lopart0 and lopart1 are the low parts | |
1404 | of both operands. | |
1405 | We perform lopart0 * lopart1 and lopart * hipart widening | |
1406 | multiplications. */ | |
1407 | tree halfutype = build_nonstandard_integer_type (hprec, 1); | |
1408 | ops.op0 = make_tree (halfutype, lopart0); | |
1409 | ops.op1 = make_tree (halfutype, lopart1); | |
1410 | rtx lo0xlo1 | |
1411 | = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1412 | ||
1413 | ops.op0 = make_tree (halfutype, lopart); | |
1414 | ops.op1 = make_tree (halfutype, hipart); | |
1415 | rtx loxhi = gen_reg_rtx (mode); | |
1416 | rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1417 | emit_move_insn (loxhi, tem); | |
1418 | ||
0c93c8a9 | 1419 | if (!uns) |
1420 | { | |
1421 | /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */ | |
1422 | if (larger_sign == 0) | |
1423 | emit_jump (after_hipart_neg); | |
1424 | else if (larger_sign != -1) | |
1425 | emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, | |
1426 | hmode, false, after_hipart_neg, | |
1427 | PROB_EVEN); | |
1428 | ||
1429 | tem = convert_modes (mode, hmode, lopart, 1); | |
1430 | tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1); | |
1431 | tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX, | |
1432 | 1, OPTAB_DIRECT); | |
1433 | emit_move_insn (loxhi, tem); | |
1434 | ||
1435 | emit_label (after_hipart_neg); | |
1436 | ||
1437 | /* if (lopart < 0) loxhi -= larger; */ | |
1438 | if (smaller_sign == 0) | |
1439 | emit_jump (after_lopart_neg); | |
1440 | else if (smaller_sign != -1) | |
1441 | emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, | |
1442 | hmode, false, after_lopart_neg, | |
1443 | PROB_EVEN); | |
1444 | ||
1445 | tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX, | |
1446 | 1, OPTAB_DIRECT); | |
1447 | emit_move_insn (loxhi, tem); | |
1448 | ||
1449 | emit_label (after_lopart_neg); | |
1450 | } | |
1ee16627 | 1451 | |
1452 | /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */ | |
1453 | tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1); | |
1454 | tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX, | |
1455 | 1, OPTAB_DIRECT); | |
1456 | emit_move_insn (loxhi, tem); | |
1457 | ||
1458 | /* if (loxhi >> (bitsize / 2) | |
0c93c8a9 | 1459 | == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns) |
1460 | if (loxhi >> (bitsize / 2) == 0 (if uns). */ | |
1ee16627 | 1461 | rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec, |
1462 | NULL_RTX, 0); | |
1463 | hipartloxhi = gen_lowpart (hmode, hipartloxhi); | |
0c93c8a9 | 1464 | rtx signbitloxhi = const0_rtx; |
1465 | if (!uns) | |
1466 | signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, | |
1467 | gen_lowpart (hmode, loxhi), | |
1468 | hprec - 1, NULL_RTX, 0); | |
1ee16627 | 1469 | |
1470 | emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX, | |
1471 | hmode, false, do_overflow, | |
1472 | PROB_VERY_UNLIKELY); | |
1473 | ||
1474 | /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */ | |
1475 | rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec, | |
1476 | NULL_RTX, 1); | |
1477 | tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1); | |
1478 | ||
1479 | tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res, | |
1480 | 1, OPTAB_DIRECT); | |
1481 | if (tem != res) | |
1482 | emit_move_insn (res, tem); | |
1483 | emit_jump (done_label); | |
1484 | ||
1485 | emit_label (both_ops_large); | |
1486 | ||
0c93c8a9 | 1487 | /* If both operands are large (not sign (!uns) or zero (uns) |
1488 | extended from hmode), then perform the full multiplication | |
1489 | which will be the result of the operation. | |
1490 | The only cases which don't overflow are for signed multiplication | |
1491 | some cases where both hipart0 and highpart1 are 0 or -1. | |
1492 | For unsigned multiplication when high parts are both non-zero | |
1493 | this overflows always. */ | |
1ee16627 | 1494 | ops.code = MULT_EXPR; |
0c93c8a9 | 1495 | ops.op0 = make_tree (type, op0); |
1496 | ops.op1 = make_tree (type, op1); | |
1ee16627 | 1497 | tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); |
1498 | emit_move_insn (res, tem); | |
1499 | ||
0c93c8a9 | 1500 | if (!uns) |
1ee16627 | 1501 | { |
0c93c8a9 | 1502 | if (!op0_medium_p) |
1503 | { | |
1504 | tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx, | |
1505 | NULL_RTX, 1, OPTAB_DIRECT); | |
1506 | emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, | |
1507 | hmode, true, do_error, | |
1508 | PROB_VERY_UNLIKELY); | |
1509 | } | |
1ee16627 | 1510 | |
0c93c8a9 | 1511 | if (!op1_medium_p) |
1512 | { | |
1513 | tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx, | |
1514 | NULL_RTX, 1, OPTAB_DIRECT); | |
1515 | emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, | |
1516 | hmode, true, do_error, | |
1517 | PROB_VERY_UNLIKELY); | |
1518 | } | |
1ee16627 | 1519 | |
0c93c8a9 | 1520 | /* At this point hipart{0,1} are both in [-1, 0]. If they are |
1521 | the same, overflow happened if res is negative, if they are | |
1522 | different, overflow happened if res is positive. */ | |
1523 | if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign) | |
1524 | emit_jump (hipart_different); | |
1525 | else if (op0_sign == 1 || op1_sign == 1) | |
1526 | emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode, | |
1527 | true, hipart_different, PROB_EVEN); | |
1ee16627 | 1528 | |
0c93c8a9 | 1529 | emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, |
1530 | false, do_error, PROB_VERY_UNLIKELY); | |
1531 | emit_jump (done_label); | |
1ee16627 | 1532 | |
0c93c8a9 | 1533 | emit_label (hipart_different); |
1534 | ||
1535 | emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, | |
1536 | false, do_error, PROB_VERY_UNLIKELY); | |
1537 | emit_jump (done_label); | |
1538 | } | |
1ee16627 | 1539 | |
1540 | emit_label (do_overflow); | |
1541 | ||
1542 | /* Overflow, do full multiplication and fallthru into do_error. */ | |
0c93c8a9 | 1543 | ops.op0 = make_tree (type, op0); |
1544 | ops.op1 = make_tree (type, op1); | |
1ee16627 | 1545 | tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); |
1546 | emit_move_insn (res, tem); | |
1547 | } | |
137559b2 | 1548 | else |
1549 | { | |
0c93c8a9 | 1550 | gcc_assert (!is_ubsan); |
137559b2 | 1551 | ops.code = MULT_EXPR; |
0c93c8a9 | 1552 | ops.type = type; |
137559b2 | 1553 | res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); |
1554 | emit_jump (done_label); | |
1555 | } | |
1556 | } | |
1557 | ||
0c93c8a9 | 1558 | do_error_label: |
137559b2 | 1559 | emit_label (do_error); |
0c93c8a9 | 1560 | if (is_ubsan) |
1561 | { | |
1562 | /* Expand the ubsan builtin call. */ | |
1563 | push_temp_slots (); | |
1564 | fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0), | |
1565 | arg0, arg1); | |
1566 | expand_normal (fn); | |
1567 | pop_temp_slots (); | |
1568 | do_pending_stack_adjust (); | |
1569 | } | |
1570 | else if (lhs) | |
1571 | write_complex_part (target, const1_rtx, true); | |
137559b2 | 1572 | |
1573 | /* We're done. */ | |
1574 | emit_label (done_label); | |
1575 | ||
0c93c8a9 | 1576 | /* u1 * u2 -> sr */ |
1577 | if (uns0_p && uns1_p && !unsr_p) | |
1578 | { | |
1579 | rtx_code_label *all_done_label = gen_label_rtx (); | |
1580 | emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, | |
1581 | false, all_done_label, PROB_VERY_LIKELY); | |
1582 | write_complex_part (target, const1_rtx, true); | |
1583 | emit_label (all_done_label); | |
1584 | } | |
1585 | ||
1586 | /* s1 * u2 -> sr */ | |
1587 | if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3) | |
1588 | { | |
1589 | rtx_code_label *all_done_label = gen_label_rtx (); | |
1590 | rtx_code_label *set_noovf = gen_label_rtx (); | |
1591 | emit_cmp_and_jump_insns (op1, const0_rtx, GE, NULL_RTX, mode, | |
1592 | false, all_done_label, PROB_VERY_LIKELY); | |
1593 | write_complex_part (target, const1_rtx, true); | |
1594 | emit_cmp_and_jump_insns (op0, const0_rtx, EQ, NULL_RTX, mode, | |
1595 | false, set_noovf, PROB_VERY_LIKELY); | |
1596 | emit_cmp_and_jump_insns (op0, constm1_rtx, NE, NULL_RTX, mode, | |
1597 | false, all_done_label, PROB_VERY_UNLIKELY); | |
1598 | emit_cmp_and_jump_insns (op1, res, NE, NULL_RTX, mode, | |
1599 | false, all_done_label, PROB_VERY_UNLIKELY); | |
1600 | emit_label (set_noovf); | |
1601 | write_complex_part (target, const0_rtx, true); | |
1602 | emit_label (all_done_label); | |
1603 | } | |
1604 | ||
137559b2 | 1605 | if (lhs) |
0c93c8a9 | 1606 | { |
1607 | if (is_ubsan) | |
920d99b4 | 1608 | expand_ubsan_result_store (target, res); |
0c93c8a9 | 1609 | else |
1610 | expand_arith_overflow_result_store (lhs, target, mode, res); | |
1611 | } | |
137559b2 | 1612 | } |
1613 | ||
1614 | /* Expand UBSAN_CHECK_ADD call STMT. */ | |
1615 | ||
1616 | static void | |
1a91d914 | 1617 | expand_UBSAN_CHECK_ADD (gcall *stmt) |
137559b2 | 1618 | { |
0c93c8a9 | 1619 | location_t loc = gimple_location (stmt); |
1620 | tree lhs = gimple_call_lhs (stmt); | |
1621 | tree arg0 = gimple_call_arg (stmt, 0); | |
1622 | tree arg1 = gimple_call_arg (stmt, 1); | |
1623 | expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1, | |
1624 | false, false, false, true); | |
137559b2 | 1625 | } |
1626 | ||
1627 | /* Expand UBSAN_CHECK_SUB call STMT. */ | |
1628 | ||
1629 | static void | |
1a91d914 | 1630 | expand_UBSAN_CHECK_SUB (gcall *stmt) |
137559b2 | 1631 | { |
0c93c8a9 | 1632 | location_t loc = gimple_location (stmt); |
1633 | tree lhs = gimple_call_lhs (stmt); | |
1634 | tree arg0 = gimple_call_arg (stmt, 0); | |
1635 | tree arg1 = gimple_call_arg (stmt, 1); | |
1636 | if (integer_zerop (arg0)) | |
1637 | expand_neg_overflow (loc, lhs, arg1, true); | |
137559b2 | 1638 | else |
0c93c8a9 | 1639 | expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1, |
1640 | false, false, false, true); | |
137559b2 | 1641 | } |
1642 | ||
1643 | /* Expand UBSAN_CHECK_MUL call STMT. */ | |
1644 | ||
1645 | static void | |
1a91d914 | 1646 | expand_UBSAN_CHECK_MUL (gcall *stmt) |
137559b2 | 1647 | { |
0c93c8a9 | 1648 | location_t loc = gimple_location (stmt); |
1649 | tree lhs = gimple_call_lhs (stmt); | |
1650 | tree arg0 = gimple_call_arg (stmt, 0); | |
1651 | tree arg1 = gimple_call_arg (stmt, 1); | |
1652 | expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true); | |
1653 | } | |
1654 | ||
1655 | /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */ | |
1656 | ||
1657 | static void | |
1658 | expand_arith_overflow (enum tree_code code, gimple stmt) | |
1659 | { | |
1660 | tree lhs = gimple_call_lhs (stmt); | |
1661 | if (lhs == NULL_TREE) | |
1662 | return; | |
1663 | tree arg0 = gimple_call_arg (stmt, 0); | |
1664 | tree arg1 = gimple_call_arg (stmt, 1); | |
1665 | tree type = TREE_TYPE (TREE_TYPE (lhs)); | |
1666 | int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0)); | |
1667 | int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1)); | |
1668 | int unsr_p = TYPE_UNSIGNED (type); | |
1669 | int prec0 = TYPE_PRECISION (TREE_TYPE (arg0)); | |
1670 | int prec1 = TYPE_PRECISION (TREE_TYPE (arg1)); | |
1671 | int precres = TYPE_PRECISION (type); | |
1672 | location_t loc = gimple_location (stmt); | |
1673 | if (!uns0_p && get_range_pos_neg (arg0) == 1) | |
1674 | uns0_p = true; | |
1675 | if (!uns1_p && get_range_pos_neg (arg1) == 1) | |
1676 | uns1_p = true; | |
1677 | int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED); | |
1678 | prec0 = MIN (prec0, pr); | |
1679 | pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED); | |
1680 | prec1 = MIN (prec1, pr); | |
1681 | ||
1682 | /* If uns0_p && uns1_p, precop is minimum needed precision | |
1683 | of unsigned type to hold the exact result, otherwise | |
1684 | precop is minimum needed precision of signed type to | |
1685 | hold the exact result. */ | |
1686 | int precop; | |
1687 | if (code == MULT_EXPR) | |
1688 | precop = prec0 + prec1 + (uns0_p != uns1_p); | |
1689 | else | |
1690 | { | |
1691 | if (uns0_p == uns1_p) | |
1692 | precop = MAX (prec0, prec1) + 1; | |
1693 | else if (uns0_p) | |
1694 | precop = MAX (prec0 + 1, prec1) + 1; | |
1695 | else | |
1696 | precop = MAX (prec0, prec1 + 1) + 1; | |
1697 | } | |
1698 | int orig_precres = precres; | |
1699 | ||
1700 | do | |
1701 | { | |
1702 | if ((uns0_p && uns1_p) | |
1703 | ? ((precop + !unsr_p) <= precres | |
1704 | /* u1 - u2 -> ur can overflow, no matter what precision | |
1705 | the result has. */ | |
1706 | && (code != MINUS_EXPR || !unsr_p)) | |
1707 | : (!unsr_p && precop <= precres)) | |
1708 | { | |
1709 | /* The infinity precision result will always fit into result. */ | |
1710 | rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
1711 | write_complex_part (target, const0_rtx, true); | |
1712 | enum machine_mode mode = TYPE_MODE (type); | |
1713 | struct separate_ops ops; | |
1714 | ops.code = code; | |
1715 | ops.type = type; | |
1716 | ops.op0 = fold_convert_loc (loc, type, arg0); | |
1717 | ops.op1 = fold_convert_loc (loc, type, arg1); | |
1718 | ops.op2 = NULL_TREE; | |
1719 | ops.location = loc; | |
1720 | rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | |
1721 | expand_arith_overflow_result_store (lhs, target, mode, tem); | |
1722 | return; | |
1723 | } | |
1724 | ||
1725 | #ifdef WORD_REGISTER_OPERATIONS | |
1726 | /* For sub-word operations, if target doesn't have them, start | |
1727 | with precres widening right away, otherwise do it only | |
1728 | if the most simple cases can't be used. */ | |
1729 | if (orig_precres == precres && precres < BITS_PER_WORD) | |
1730 | ; | |
1731 | else | |
1732 | #endif | |
1733 | if ((uns0_p && uns1_p && unsr_p && prec0 <= precres && prec1 <= precres) | |
1734 | || ((!uns0_p || !uns1_p) && !unsr_p | |
1735 | && prec0 + uns0_p <= precres | |
1736 | && prec1 + uns1_p <= precres)) | |
1737 | { | |
1738 | arg0 = fold_convert_loc (loc, type, arg0); | |
1739 | arg1 = fold_convert_loc (loc, type, arg1); | |
1740 | switch (code) | |
1741 | { | |
1742 | case MINUS_EXPR: | |
1743 | if (integer_zerop (arg0) && !unsr_p) | |
1744 | expand_neg_overflow (loc, lhs, arg1, false); | |
1745 | /* FALLTHRU */ | |
1746 | case PLUS_EXPR: | |
1747 | expand_addsub_overflow (loc, code, lhs, arg0, arg1, | |
1748 | unsr_p, unsr_p, unsr_p, false); | |
1749 | return; | |
1750 | case MULT_EXPR: | |
1751 | expand_mul_overflow (loc, lhs, arg0, arg1, | |
1752 | unsr_p, unsr_p, unsr_p, false); | |
1753 | return; | |
1754 | default: | |
1755 | gcc_unreachable (); | |
1756 | } | |
1757 | } | |
1758 | ||
1759 | /* For sub-word operations, retry with a wider type first. */ | |
1760 | if (orig_precres == precres && precop <= BITS_PER_WORD) | |
1761 | { | |
1762 | #ifdef WORD_REGISTER_OPERATIONS | |
1763 | int p = BITS_PER_WORD; | |
1764 | #else | |
1765 | int p = precop; | |
1766 | #endif | |
1767 | enum machine_mode m = smallest_mode_for_size (p, MODE_INT); | |
1768 | tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m), | |
1769 | uns0_p && uns1_p | |
1770 | && unsr_p); | |
1771 | p = TYPE_PRECISION (optype); | |
1772 | if (p > precres) | |
1773 | { | |
1774 | precres = p; | |
1775 | unsr_p = TYPE_UNSIGNED (optype); | |
1776 | type = optype; | |
1777 | continue; | |
1778 | } | |
1779 | } | |
1780 | ||
1781 | if (prec0 <= precres && prec1 <= precres) | |
1782 | { | |
1783 | tree types[2]; | |
1784 | if (unsr_p) | |
1785 | { | |
1786 | types[0] = build_nonstandard_integer_type (precres, 0); | |
1787 | types[1] = type; | |
1788 | } | |
1789 | else | |
1790 | { | |
1791 | types[0] = type; | |
1792 | types[1] = build_nonstandard_integer_type (precres, 1); | |
1793 | } | |
1794 | arg0 = fold_convert_loc (loc, types[uns0_p], arg0); | |
1795 | arg1 = fold_convert_loc (loc, types[uns1_p], arg1); | |
1796 | if (code != MULT_EXPR) | |
1797 | expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p, | |
1798 | uns0_p, uns1_p, false); | |
1799 | else | |
1800 | expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p, | |
1801 | uns0_p, uns1_p, false); | |
1802 | return; | |
1803 | } | |
1804 | ||
1805 | /* Retry with a wider type. */ | |
1806 | if (orig_precres == precres) | |
1807 | { | |
1808 | int p = MAX (prec0, prec1); | |
1809 | enum machine_mode m = smallest_mode_for_size (p, MODE_INT); | |
1810 | tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m), | |
1811 | uns0_p && uns1_p | |
1812 | && unsr_p); | |
1813 | p = TYPE_PRECISION (optype); | |
1814 | if (p > precres) | |
1815 | { | |
1816 | precres = p; | |
1817 | unsr_p = TYPE_UNSIGNED (optype); | |
1818 | type = optype; | |
1819 | continue; | |
1820 | } | |
1821 | } | |
1822 | ||
1823 | gcc_unreachable (); | |
1824 | } | |
1825 | while (1); | |
1826 | } | |
1827 | ||
1828 | /* Expand ADD_OVERFLOW STMT. */ | |
1829 | ||
1830 | static void | |
1a91d914 | 1831 | expand_ADD_OVERFLOW (gcall *stmt) |
0c93c8a9 | 1832 | { |
1833 | expand_arith_overflow (PLUS_EXPR, stmt); | |
1834 | } | |
1835 | ||
1836 | /* Expand SUB_OVERFLOW STMT. */ | |
1837 | ||
1838 | static void | |
1a91d914 | 1839 | expand_SUB_OVERFLOW (gcall *stmt) |
0c93c8a9 | 1840 | { |
1841 | expand_arith_overflow (MINUS_EXPR, stmt); | |
1842 | } | |
1843 | ||
1844 | /* Expand MUL_OVERFLOW STMT. */ | |
1845 | ||
1846 | static void | |
1a91d914 | 1847 | expand_MUL_OVERFLOW (gcall *stmt) |
0c93c8a9 | 1848 | { |
1849 | expand_arith_overflow (MULT_EXPR, stmt); | |
137559b2 | 1850 | } |
1851 | ||
c71d3c24 | 1852 | /* This should get folded in tree-vectorizer.c. */ |
1853 | ||
1854 | static void | |
1a91d914 | 1855 | expand_LOOP_VECTORIZED (gcall *stmt ATTRIBUTE_UNUSED) |
c71d3c24 | 1856 | { |
1857 | gcc_unreachable (); | |
1858 | } | |
1859 | ||
1860 | static void | |
1a91d914 | 1861 | expand_MASK_LOAD (gcall *stmt) |
c71d3c24 | 1862 | { |
1863 | struct expand_operand ops[3]; | |
1864 | tree type, lhs, rhs, maskt; | |
1865 | rtx mem, target, mask; | |
1866 | ||
1867 | maskt = gimple_call_arg (stmt, 2); | |
1868 | lhs = gimple_call_lhs (stmt); | |
4e459157 | 1869 | if (lhs == NULL_TREE) |
1870 | return; | |
c71d3c24 | 1871 | type = TREE_TYPE (lhs); |
1872 | rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), | |
1873 | gimple_call_arg (stmt, 1)); | |
1874 | ||
1875 | mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
1876 | gcc_assert (MEM_P (mem)); | |
1877 | mask = expand_normal (maskt); | |
1878 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
1879 | create_output_operand (&ops[0], target, TYPE_MODE (type)); | |
1880 | create_fixed_operand (&ops[1], mem); | |
1881 | create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt))); | |
1882 | expand_insn (optab_handler (maskload_optab, TYPE_MODE (type)), 3, ops); | |
1883 | } | |
1884 | ||
1885 | static void | |
1a91d914 | 1886 | expand_MASK_STORE (gcall *stmt) |
c71d3c24 | 1887 | { |
1888 | struct expand_operand ops[3]; | |
1889 | tree type, lhs, rhs, maskt; | |
1890 | rtx mem, reg, mask; | |
1891 | ||
1892 | maskt = gimple_call_arg (stmt, 2); | |
1893 | rhs = gimple_call_arg (stmt, 3); | |
1894 | type = TREE_TYPE (rhs); | |
1895 | lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), | |
1896 | gimple_call_arg (stmt, 1)); | |
1897 | ||
1898 | mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
1899 | gcc_assert (MEM_P (mem)); | |
1900 | mask = expand_normal (maskt); | |
1901 | reg = expand_normal (rhs); | |
1902 | create_fixed_operand (&ops[0], mem); | |
1903 | create_input_operand (&ops[1], reg, TYPE_MODE (type)); | |
1904 | create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt))); | |
1905 | expand_insn (optab_handler (maskstore_optab, TYPE_MODE (type)), 3, ops); | |
1906 | } | |
1907 | ||
b2c0e0b7 | 1908 | static void |
1a91d914 | 1909 | expand_ABNORMAL_DISPATCHER (gcall *) |
b2c0e0b7 | 1910 | { |
1911 | } | |
1912 | ||
c83059be | 1913 | static void |
1a91d914 | 1914 | expand_BUILTIN_EXPECT (gcall *stmt) |
c83059be | 1915 | { |
1916 | /* When guessing was done, the hints should be already stripped away. */ | |
1917 | gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ()); | |
1918 | ||
1919 | rtx target; | |
1920 | tree lhs = gimple_call_lhs (stmt); | |
1921 | if (lhs) | |
1922 | target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | |
1923 | else | |
1924 | target = const0_rtx; | |
1925 | rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL); | |
1926 | if (lhs && val != target) | |
1927 | emit_move_insn (target, val); | |
1928 | } | |
1929 | ||
fb049fba | 1930 | /* Routines to expand each internal function, indexed by function number. |
1931 | Each routine has the prototype: | |
1932 | ||
1a91d914 | 1933 | expand_<NAME> (gcall *stmt) |
fb049fba | 1934 | |
1935 | where STMT is the statement that performs the call. */ | |
1a91d914 | 1936 | static void (*const internal_fn_expanders[]) (gcall *) = { |
64d5d3e8 | 1937 | #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE, |
fb049fba | 1938 | #include "internal-fn.def" |
1939 | #undef DEF_INTERNAL_FN | |
1940 | 0 | |
1941 | }; | |
1942 | ||
1943 | /* Expand STMT, which is a call to internal function FN. */ | |
1944 | ||
1945 | void | |
1a91d914 | 1946 | expand_internal_call (gcall *stmt) |
fb049fba | 1947 | { |
1948 | internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt); | |
1949 | } |